From 952ad17e3e9a72e8802973f82ce770ede7c923e0 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Fri, 6 Jan 2023 10:38:22 -0600
Subject: [PATCH 001/256] initial commit of claw monitor

---
 .../nircam_monitors/claw_monitor.py           | 64 +++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 jwql/instrument_monitors/nircam_monitors/claw_monitor.py

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
new file mode 100644
index 000000000..c11ca57f9
--- /dev/null
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -0,0 +1,64 @@
+#! /usr/bin/env python
+
+"""This module contains code for the claw monitor.
+
+Author
+------
+    - Ben Sunnquist
+
+Use
+---
+    This module can be used from the command line as such:
+
+    ::
+
+        python claw_monitor.py
+"""
+
+import logging
+import os
+
+from astropy.io import fits
+from astropy.time import Time
+from astropy.visualization import ZScaleInterval
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import numpy as np
+
+from jwql.utils import monitor_utils
+from jwql.utils.logging_functions import log_info, log_fail
+
+
+class ClawMonitor():
+    """Class for executing the claw monitor.
+    """
+
+    def __init__(self):
+        """Initialize an instance of the ``ClawMonitor`` class.
+        """
+
+    def process(self, file_list):
+        """The main method for processing.  See module docstrings for further details.
+        """
+
+    #@log_fail  # ben uncomment
+    #@log_info  # ben uncomment
+    def run(self):
+        """The main method.  See module docstrings for further details."""
+
+        logging.info('Begin logging for claw_monitor')
+
+
+        logging.info('Claw Monitor completed successfully.')
+
+
+if __name__ == '__main__':
+
+    module = os.path.basename(__file__).strip('.py')
+    #start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # ben uncomment
+
+    monitor = ClawMonitor()
+    monitor.run()
+
+    #monitor_utils.update_monitor_table(module, start_time, log_file)   # ben uncomment

From 402d7638651cb12ffda85680e55d649d78f02bcf Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Fri, 6 Jan 2023 15:41:52 -0600
Subject: [PATCH 002/256] added mast query, initial stacking and plotting

---
 .../nircam_monitors/claw_monitor.py           | 97 +++++++++++++++++--
 1 file changed, 91 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index c11ca57f9..db829c14f 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -21,6 +21,7 @@
 from astropy.io import fits
 from astropy.time import Time
 from astropy.visualization import ZScaleInterval
+from astroquery.mast import Mast
 import matplotlib
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
@@ -38,17 +39,101 @@ def __init__(self):
         """Initialize an instance of the ``ClawMonitor`` class.
         """
 
-    def process(self, file_list):
+    def process(self):
         """The main method for processing.  See module docstrings for further details.
         """
 
-    #@log_fail  # ben uncomment
-    #@log_info  # ben uncomment
+        # Get detector order and plot settings, depending on the wavelength channel
+        if self.wv == 'SW':
+            detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']  # in on-sky order, don't change order
+            cols, rows = 5, 2
+            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,1,1,.1])
+            fig = plt.figure(figsize=(40, 20))
+            cbar_fs = 20
+            fs = 30
+        else:
+            detectors_to_run = ['NRCALONG', 'NRCBLONG']
+            cols, rows = 3, 1
+            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,.1])
+            fig = plt.figure(figsize=(20, 10))
+            cbar_fs = 15
+            fs = 25
+        
+        # Make source-masked, median-stack of each detector's images, and add them to the plot
+        for det in detectors_to_run:
+            files = self.files[self.detectors == det]
+            print(det)
+            print(files)
+            print('------')
+
+    def query_mast(self):
+        """Query MAST for new nircam full-frame imaging data.
+
+        Returns
+        -------
+        t : astropy.table.table.Table
+            A table summarizing the new nircam imaging data.
+        """
+
+        server = "https://mast.stsci.edu"
+        JwstObs = Mast()
+        JwstObs._portal_api_connection.MAST_REQUEST_URL = server + "/portal_jwst/Mashup/Mashup.asmx/invoke"
+        JwstObs._portal_api_connection.MAST_DOWNLOAD_URL = server + "/jwst/api/v0.1/download/file"
+        JwstObs._portal_api_connection.COLUMNS_CONFIG_URL = server + "/portal_jwst/Mashup/Mashup.asmx/columnsconfig"
+        JwstObs._portal_api_connection.MAST_BUNDLE_URL = server + "/jwst/api/v0.1/download/bundle"
+        service = 'Mast.Jwst.Filtered.Nircam'
+        FIELDS = ['filename','program', 'observtn','category','instrume', 'productLevel', 'filter', 
+                  'pupil', 'subarray', 'detector','datamodl','date_beg_mjd', 'effexptm']
+        params = {"columns":",".join(FIELDS),
+                "filters":[
+                {"paramName":"pupil","values":['CLEAR','F162M','F164N','F323N','F405N','F466N','F470N']},
+                {"paramName":"exp_type","values":['NRC_IMAGE']},
+                {"paramName":"datamodl", "values":['ImageModel']},  # exclude calints, which are cubemodel
+                {"paramName":"productLevel", "values":['2b']},  # i.e. cal.fits
+                {"paramName":"subarray", "values":['FULL']},
+                ]
+                }
+        t = JwstObs.service_request(service, params)
+        t = t[t['date_beg_mjd']>self.query_start_mjd]
+        t.sort('date_beg_mjd')
+        filetypes = np.array([row['filename'].split('_')[-1].replace('.fits','') for row in t])
+        t = t[filetypes=='cal']  # only want cal.fits files, no e.g. i2d.fits
+
+        return t
+
+    #@log_fail  # todo uncomment
+    #@log_info  # todo uncomment
     def run(self):
         """The main method.  See module docstrings for further details."""
 
         logging.info('Begin logging for claw_monitor')
-
+        self.output_dir = '/Users/bsunnquist/Downloads/'  # todo change this to os.path.join(get_config()['outputs'], 'claw_monitor')
+        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
+
+        # Query MAST for new imaging data from the last 3 days
+        self.query_end_mjd = Time.now().mjd
+        self.query_start_mjd = self.query_end_mjd - 3
+        print(self.query_end_mjd, self.query_start_mjd)
+        t = self.query_mast()
+        print(t)
+
+        # Create observation-level median stacks for each filter/pupil combo, in pixel-space
+        combos = np.array(sorted(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t]))
+        t['combos'] = combos
+        for combo in np.unique(combos)[0:2]:  # todo take off 0:2
+            tt = t[t['combos']==combo]
+            if 'long' in tt['filename'][0]:
+                self.wv = 'LW'
+            else:
+                self.wv = 'SW'
+            self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
+            self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
+            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            self.detectors = np.array(tt['detector'])
+            if not os.path.exists(self.outfile):
+                self.process()
+            else:
+                print('{} already exists'.format(self.outfile))
 
         logging.info('Claw Monitor completed successfully.')
 
@@ -56,9 +141,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    #start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # ben uncomment
+    #start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # todo uncomment
 
     monitor = ClawMonitor()
     monitor.run()
 
-    #monitor_utils.update_monitor_table(module, start_time, log_file)   # ben uncomment
+    #monitor_utils.update_monitor_table(module, start_time, log_file)   # todo uncomment

From 0ab25da8e3d4ed49205e334d2cf6d7b98e291af5 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Tue, 10 Jan 2023 13:30:13 -0600
Subject: [PATCH 003/256] stacks and plots now working on test case

---
 .../nircam_monitors/claw_monitor.py           | 80 +++++++++++++++++--
 1 file changed, 74 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index db829c14f..eb0d993a0 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -18,7 +18,9 @@
 import logging
 import os
 
+from astropy.convolution import Gaussian2DKernel, convolve
 from astropy.io import fits
+from astropy.stats import gaussian_fwhm_to_sigma
 from astropy.time import Time
 from astropy.visualization import ZScaleInterval
 from astroquery.mast import Mast
@@ -26,6 +28,7 @@
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import numpy as np
+from photutils import detect_sources, detect_threshold
 
 from jwql.utils import monitor_utils
 from jwql.utils.logging_functions import log_info, log_fail
@@ -59,12 +62,73 @@ def process(self):
             cbar_fs = 15
             fs = 25
         
-        # Make source-masked, median-stack of each detector's images, and add them to the plot
-        for det in detectors_to_run:
+        # Make source-masked, median-stack of each detector's images
+        print(self.outfile)
+        print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
+        found_scale = False
+        for i,det in enumerate(detectors_to_run):
             files = self.files[self.detectors == det]
+            # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
+            files = [f for f in files if os.path.exists(f)][0:2]  # todo change index value?
+            stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
             print(files)
             print('------')
+            for n,f in enumerate(files):
+                # Get pointing and other info from first image
+                if n == 0:
+                    h = fits.open(f)
+                    obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
+                    obs_start_mjd = h[0].header['EXPSTART']
+                    targname, ra_v1, dec_v1, pa_v3 = h[0].header['TARGPROP'], h[1].header['RA_V1'], h[1].header['DEC_V1'], h[1].header['PA_V3']
+                    h.close()
+
+                # Make source segmap, and add the masked data to the stack
+                data = fits.getdata(f, 'SCI')
+                threshold = detect_threshold(data, 1.25)
+                sigma = 3.0 * gaussian_fwhm_to_sigma  # FWHM = 3.
+                kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
+                kernel.normalize()
+                data_conv = convolve(data, kernel)
+                segmap = detect_sources(data_conv, threshold, npixels=3)
+                segmap = segmap.data
+                stack[n] = np.ma.masked_array(data, mask=segmap!=0)
+
+            # Make the normalized skyflat for this detector
+            skyflat = np.ma.median(stack, axis=0)
+            skyflat = skyflat.filled(fill_value=np.nan)
+            skyflat = skyflat / np.nanmedian(skyflat)
+            skyflat[~np.isfinite(skyflat)] = 1  # fill missing values
+
+            # Add the skyflat for this detector to the plot
+            if (self.wv=='SW') & (i>3):  # skip colobar axis
+                idx = i+1
+            else:
+                idx = i
+            ax = fig.add_subplot(grid[idx])
+            if len(skyflat[skyflat!=1])==0:
+                ax.set_title('N/A', fontsize=fs)
+                ax.imshow(skyflat, cmap='coolwarm', vmin=999, vmax=999, origin='lower')
+            elif (len(skyflat[skyflat!=1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
+                z = ZScaleInterval()
+                vmin, vmax = z.get_limits(skyflat)
+                found_scale = True
+                ax.set_title(det, fontsize=fs)
+                im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
+            else:
+                ax.set_title(det, fontsize=fs)
+                im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
+            ax.axes.get_xaxis().set_ticks([])
+            ax.axes.get_yaxis().set_ticks([])
+        
+        # Add colobar, save figure if any detector stacks exist
+        if found_scale:
+            cax = fig.add_subplot(grid[0:rows, cols-1:cols])
+            cbar = fig.colorbar(im, cax=cax, orientation='vertical')
+            cbar.ax.tick_params(labelsize=cbar_fs)
+            fig.savefig(self.outfile, dpi=100, bbox_inches='tight')
+        fig.clf()
+        plt.close()
 
     def query_mast(self):
         """Query MAST for new nircam full-frame imaging data.
@@ -94,7 +158,7 @@ def query_mast(self):
                 ]
                 }
         t = JwstObs.service_request(service, params)
-        t = t[t['date_beg_mjd']>self.query_start_mjd]
+        t = t[(t['date_beg_mjd']>self.query_start_mjd) & (t['date_beg_mjd']<self.query_end_mjd)]
         t.sort('date_beg_mjd')
         filetypes = np.array([row['filename'].split('_')[-1].replace('.fits','') for row in t])
         t = t[filetypes=='cal']  # only want cal.fits files, no e.g. i2d.fits
@@ -107,21 +171,24 @@ def run(self):
         """The main method.  See module docstrings for further details."""
 
         logging.info('Begin logging for claw_monitor')
-        self.output_dir = '/Users/bsunnquist/Downloads/'  # todo change this to os.path.join(get_config()['outputs'], 'claw_monitor')
+        self.output_dir = '/Users/bsunnquist/Documents/nircam/claw_monitor_testing/'  # todo change this to os.path.join(get_config()['outputs'], 'claw_monitor')
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
 
         # Query MAST for new imaging data from the last 3 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 3
-        print(self.query_end_mjd, self.query_start_mjd)
+        self.query_end_mjd, self.query_start_mjd = 59878.986, 59878.934  # todo remove these test datess
+        print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
         print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
-        combos = np.array(sorted(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t]))
+        combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
+        print(np.unique(combos))
         t['combos'] = combos
         for combo in np.unique(combos)[0:2]:  # todo take off 0:2
             tt = t[t['combos']==combo]
+            print(tt)
             if 'long' in tt['filename'][0]:
                 self.wv = 'LW'
             else:
@@ -129,6 +196,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
             self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 self.process()

From 1a87dd260766ddfa36be39323984e300b878e8ab Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 11 Jan 2023 16:22:30 -0600
Subject: [PATCH 004/256] adding pointing and other info to plots

---
 jwql/instrument_monitors/nircam_monitors/claw_monitor.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index eb0d993a0..ab1c10bfc 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -30,8 +30,8 @@
 import numpy as np
 from photutils import detect_sources, detect_threshold
 
-from jwql.utils import monitor_utils
-from jwql.utils.logging_functions import log_info, log_fail
+#from jwql.utils import monitor_utils  # todo uncomment
+#from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
 
 
 class ClawMonitor():
@@ -59,8 +59,8 @@ def process(self):
             cols, rows = 3, 1
             grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,.1])
             fig = plt.figure(figsize=(20, 10))
-            cbar_fs = 15
-            fs = 25
+            cbar_fs = 10
+            fs = 20
         
         # Make source-masked, median-stack of each detector's images
         print(self.outfile)
@@ -123,6 +123,7 @@ def process(self):
         
         # Add colobar, save figure if any detector stacks exist
         if found_scale:
+            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
             cax = fig.add_subplot(grid[0:rows, cols-1:cols])
             cbar = fig.colorbar(im, cax=cax, orientation='vertical')
             cbar.ax.tick_params(labelsize=cbar_fs)

From 2f48901925e7f0d0e77845e514a4a13865a75da6 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Wed, 15 Feb 2023 15:59:01 -0500
Subject: [PATCH 005/256] Adding dependabot.yml

---
 .github/dependabot.yml | 7 +++++++
 1 file changed, 7 insertions(+)
 create mode 100644 .github/dependabot.yml

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..c6e9e36a2
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,7 @@
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "monthly"
\ No newline at end of file

From f12fa1739ec825987095455924ee8a6927a4bc7a Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 14 Mar 2023 10:58:00 -0400
Subject: [PATCH 006/256] fix data going into filesystem database

---
 jwql/jwql_monitors/monitor_filesystem.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index afdd603ff..6940619cd 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -649,11 +649,13 @@ def update_characteristics_database(char_info):
 
     # Add data to filesystem_instrument table
     for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:
+        filter_list = [e[0] for e in obs['nircam']]
+        value_list= [e[1] for e in obs['nircam']]
         new_record = {}
         new_record['date'] = now
         new_record['instrument'] = instrument
-        new_record['filter_pupil'] = list(char_info[instrument].keys())
-        new_record['obs_per_filter_pupil'] = list(char_info[instrument].values())
+        new_record['filter_pupil'] = filter_list
+        new_record['obs_per_filter_pupil'] = value_list
         engine.execute(FilesystemCharacteristics.__table__.insert(), new_record)
         session.commit()
 

From aa53a267b42976d755e78b6456d881fc501cc2d8 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 14 Mar 2023 11:30:35 -0400
Subject: [PATCH 007/256] Fix sorting of log table

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 2cd0ba04a..eb28a952e 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -301,13 +301,14 @@ def dashboard_monitor_tracking(self):
         if not pd.isnull(self.delta_t):
             data = data[(data['start_time'] >= self.date - self.delta_t) & (data['start_time'] <= self.date)]
 
+        # Sort the data by start_time before translating into strings
+        data.sort_values(by='start_time', ascending=False, inplace=True)
+
+        # Now translate times to strings
         data['start_time'] = data['start_time'].map(lambda x: x.strftime('%m-%d-%Y %H:%M:%S'))
         data['end_time'] = data['end_time'].map(lambda x: x.strftime('%m-%d-%Y %H:%M:%S'))
-        # data = data.drop(columns='affected_tables')
-        table_values = data.sort_values(by='start_time', ascending=False).values
-        table_columns = data.columns.values
 
-        return table_columns, table_values
+        return data.columns.values, data.values
 
     def make_panel(self, x_value, top, instrument, title, x_axis_label):
         """Make tab panel for tablulated figure.

From db362dbe8d1498e7dd673fc8f71ddcdfedd1b10b Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 16 Mar 2023 09:49:45 -0400
Subject: [PATCH 008/256] Make database entries robust against unknown file
 suffixes

---
 jwql/jwql_monitors/monitor_filesystem.py  | 21 ++++++++--
 jwql/website/apps/jwql/bokeh_dashboard.py | 50 +++++++++++++++++++++++
 2 files changed, 67 insertions(+), 4 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 6940619cd..9b7fbe5d7 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -46,6 +46,7 @@
 from bokeh.palettes import Category20_20 as palette
 from bokeh.plotting import figure, output_file, save
 import numpy as np
+from sqlalchemy.exc import DataError
 
 from jwql.database.database_interface import engine
 from jwql.database.database_interface import session
@@ -218,9 +219,11 @@ def get_area_stats(central_storage_dict):
     """
     logging.info('Gathering stats for central storage area')
 
-    arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
+    #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
+    arealist = ['outputs']
     counteddirs = []
 
+    skipped_files = []
     sums = 0  # to be used to count 'all'
     for area in arealist:
 
@@ -263,6 +266,8 @@ def get_area_stats(central_storage_dict):
                             if exists:
                                 filesize = os.path.getsize(file_path)
                                 sums += filesize
+                            else:
+                                skipped_files.append(filename)
             use = sums / (1024 ** 4)
         else:
             for dirpath, _, files in os.walk(fullpath):
@@ -274,10 +279,12 @@ def get_area_stats(central_storage_dict):
                         filesize = os.path.getsize(file_path)
                         used += filesize
                         sums += filesize
+                    else:
+                        skipped_files.append(filename)
             use = used / (1024 ** 4)
         central_storage_dict[area]['used'] = use
 
-    return central_storage_dict
+    return central_storage_dict, skipped_files
 
 
 def get_observation_characteristics():
@@ -690,8 +697,14 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
             new_record['filetype'] = filetype
             new_record['count'] = instrument_results_dict[instrument][filetype]['count']
             new_record['size'] = instrument_results_dict[instrument][filetype]['size']
-            engine.execute(FilesystemInstrument.__table__.insert(), new_record)
-            session.commit()
+
+            # Protect against updated enum options that have not been propagated to
+            # the table definition
+            try:
+                engine.execute(FilesystemInstrument.__table__.insert(), new_record)
+                session.commit()
+            except DataError as e:
+                logging.error(e)
 
     # Add data to central_storage table
     arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index eb28a952e..74849e495 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -152,6 +152,56 @@ def __init__(self, delta_t=None):
         now = dt.now()
         self.date = pd.Timestamp('{}-{}-{}'.format(now.year, now.month, now.day))
 
+    def dashboard_central_store_data_volume(self):
+        """Create trending plot of data volume for various areas on central store
+
+        Returns
+        -------
+        tabs : bokeh.models.widgets.widget.Widget
+            A figure with tabs for each central store area
+        """
+        # Plot total data volume and available disk space versus time
+        results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all()
+
+        # Initialize plot
+        dates, total_sizes, availables = zip(*results)
+        plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save',
+                      x_axis_type='datetime',
+                      title='Central Store stats',
+                      x_axis_label='Date',
+                      y_axis_label='Size TB')
+
+        plot.line(dates, total_sizes, legend='Total size', line_color='red')
+        plot.circle(dates, total_sizes, color='red')
+        plot.line(dates, availables, legend='Free', line_color='blue')
+        plot.circle(dates, availables, color='blue')
+
+        # This part of the plot should cycle through areas and plot area used values vs. date
+        arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
+        colors = itertools.cycle(palette)
+        for area, color in zip(arealist, colors):
+
+            # Query for used sizes
+            results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area)
+
+            # Group by date
+            if results:
+                results_dict = defaultdict(int)
+                for date, value in results:
+                    results_dict[date] += value
+
+                # Parse results so they can be easily plotted
+                dates = list(results_dict.keys())
+                values = list(results_dict.values())
+
+                # Plot the results
+                plot.line(dates, values, legend='{} files'.format(area), line_color=color)
+                plot.circle(dates, values, color=color)
+
+        session.close()
+        return plot
+
+
     def dashboard_filetype_bar_chart(self):
         """Build bar chart of files based off of type
 

From d9be4b564bcf456eb83d94ad5c0d1875071337ca Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 17 Mar 2023 12:17:04 -0400
Subject: [PATCH 009/256] Tweak plots. Ready for testing on server

---
 jwql/jwql_monitors/monitor_filesystem.py  |  19 +-
 jwql/website/apps/jwql/bokeh_dashboard.py | 203 ++++++++++++++++++----
 jwql/website/apps/jwql/views.py           |   6 +-
 3 files changed, 188 insertions(+), 40 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 9b7fbe5d7..39d38af24 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -68,7 +68,9 @@
 PUBLIC_FILESYSTEM = os.path.join(FILESYSTEM, 'public')
 CENTRAL = SETTINGS['jwql_dir']
 OUTPUTS = SETTINGS['outputs']
-
+PREVIEW_IMAGES = SETTINGS['preview_image_filesystem']
+THUMBNAILS = SETTINGS['thumbnail_filesystem']
+LOGS = SETTINGS['log_dir']
 
 def files_per_filter():
     """Querying MAST (rather than looping through the filesystem), determine how
@@ -219,23 +221,24 @@ def get_area_stats(central_storage_dict):
     """
     logging.info('Gathering stats for central storage area')
 
+    areas = {'outputs': OUTPUTS,
+             'logs': LOGS,
+             'preview_images': PREVIEW_IMAGES,
+             'thumbnails': THUMBNAILS,
+             'all': CENTRAL}
+
     #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
-    arealist = ['outputs']
     counteddirs = []
 
-    skipped_files = []
     sums = 0  # to be used to count 'all'
-    for area in arealist:
+    for area in areas:
 
         used = 0
         # initialize area in dictionary
         if area not in central_storage_dict:
             central_storage_dict[area] = {}
 
-        if area == 'all':
-            fullpath = CENTRAL
-        else:
-            fullpath = os.path.join(CENTRAL, area)
+        fullpath = areas[area]
 
         logging.info('\tSearching directory {}'.format(fullpath))
         counteddirs.append(fullpath)
diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 74849e495..43ba11614 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -32,6 +32,7 @@
 from datetime import datetime as dt
 from math import pi
 from operator import itemgetter
+import os
 
 from bokeh.layouts import column
 from bokeh.models import Axis, ColumnDataSource, DatetimeTickFormatter, OpenURL, TapTool
@@ -43,6 +44,7 @@
 from sqlalchemy import func, and_
 
 import jwql.database.database_interface as di
+from jwql.database.database_interface import CentralStore
 from jwql.utils.constants import ANOMALY_CHOICES_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT
 from jwql.utils.utils import get_base_url, get_config
 from jwql.website.apps.jwql.data_containers import build_table
@@ -152,54 +154,192 @@ def __init__(self, delta_t=None):
         now = dt.now()
         self.date = pd.Timestamp('{}-{}-{}'.format(now.year, now.month, now.day))
 
-    def dashboard_central_store_data_volume(self):
-        """Create trending plot of data volume for various areas on central store
+    def dashboard_disk_usage(self):
+        """Create trending plot of data volume for various disks. Here we are plotting
+        the results of a "df -hk" call for top-level directories. The results (i.e. on
+        central store) may contain contributions from non-JWQL files, since JWQL can
+        share disks with other projects. These plots are useful for tracking disk usage
+        and requesting more disk space if needed.
 
         Returns
         -------
         tabs : bokeh.models.widgets.widget.Widget
-            A figure with tabs for each central store area
+            Set of tabs containing plots of the used and available disk space
         """
+        # There are two main disks that we want to show usage for. The central store
+        # area, and the disk that is internal to the server. Use the logs entry to
+        # get the central store information, and the preview_image entry to get
+        # server disk information.
+        config = get_config()
+
+        log_data = di.session.query(CentralStore.date, CentralStore.size, CentralStore.available) \
+            .filter(CentralStore.area == 'logs') \
+            .all()
+
+        # Convert to dataframe
+        log_data = pd.DataFrame(log_data)
+
+        preview_data = di.session.query(CentralStore.date, CentralStore.size, CentralStore.available) \
+            .filter(CentralStore.area == 'preview_images') \
+            .all()
+
+        # Convert to dataframe
+        preview_data = pd.DataFrame(preview_data)
+
+        # If the user is requesting a certain time range, cut down the entries
+        if not pd.isnull(self.delta_t):
+            log_data = log_data[(log_data['date'] >= self.date - self.delta_t) & (log_data['date'] <= self.date)]
+            preview_data = preview_data[(preview_data['date'] >= self.date - self.delta_t) & (preview_data['date'] <= self.date)]
+
+        log_results = {'dirname': os.path.abspath(os.path.join(config['log_dir'], '../')),
+                       'results': log_data,
+                       'shortname': 'Central Store'
+                       }
+
+        preview_results = {'dirname': os.path.abspath(os.path.join(config['preview_image_filesystem'], '../')),
+                           'results': preview_data,
+                           'shortname': 'Server'
+                           }
+
         # Plot total data volume and available disk space versus time
-        results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all()
+        plots = {}
+        tabs = []
+        for data in [preview_results, log_results]:
+
+            # Calculate the size of the data
+            data['results']['used'] = data['results']['size'] - data['results']['available']
+            source = ColumnDataSource(data['results'])
+
+            # Initialize plot
+            plots[data['shortname']] = figure(tools='pan,hover,box_zoom,wheel_zoom,reset,save',
+                                              x_axis_type='datetime',
+                                              title=f"Available & Used Storage on {data['shortname']}",
+                                              x_axis_label='Date',
+                                              y_axis_label='Size TB')
+
+            plots[data['shortname']].line(x='date', y='available', source=source, legend_label='Available', line_color='red',line_width=3)
+            plots[data['shortname']].circle(x='date', y='available', source=source,color='red', size=10)
+            plots[data['shortname']].line(x='date', y='used', source=source, legend_label='Used', line_color='blue', line_width=3)
+            plots[data['shortname']].circle(x='date', y='used', source=source,color='blue', size=10)
+
+            plots[data['shortname']].xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
+                                                       days=["%d %B %Y"],
+                                                       months=["%d %B %Y"],
+                                                       years=["%d %B %Y"],
+                                                       )
+            plots[data['shortname']].xaxis.major_label_orientation = pi / 4
+            plots[data['shortname']].legend.location = 'top_left'
+
+            hover_tool = HoverTool(tooltips=[('Available:', '@available'),
+                                             ('Used:', '@used'),
+                                             ('Date:', '@time{%d %b %Y}')
+                                             ])
+            hover_tool.formatters = {'@time': 'datetime'}
+            plots[data['shortname']].tools.append(hover_tool)
+            tabs.append(Panel(child=plots[data['shortname']], title=f"{data['shortname']} Storage"))
+
+        tabs = Tabs(tabs=tabs)
+
+        session.close()
+        return tabs
+
+
+    def dashboard_central_store_data_volume(self):
+        """Create trending plot of data volume for various JWQL-related areas on disk.
+        These plots show data volumes calculated by walking over subdirectories/files in
+        the JWQL-specific directories. So these plots may not include the total used
+        disk volume, in the cases where JWQL is sharing a disk with other projects. These
+        plots are useful for monitoring the total volume of e.g. our preview images.
 
+        Returns
+        -------
+        tabs : bokeh.models.widgets.widget.Widget
+            A figure with tabs for each central store area
+        """
         # Initialize plot
-        dates, total_sizes, availables = zip(*results)
         plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save',
                       x_axis_type='datetime',
-                      title='Central Store stats',
+                      title='JWQL directory size',
                       x_axis_label='Date',
                       y_axis_label='Size TB')
 
-        plot.line(dates, total_sizes, legend='Total size', line_color='red')
-        plot.circle(dates, total_sizes, color='red')
-        plot.line(dates, availables, legend='Free', line_color='blue')
-        plot.circle(dates, availables, color='blue')
-
         # This part of the plot should cycle through areas and plot area used values vs. date
-        arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
-        colors = itertools.cycle(palette)
+        #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
+        arealist = ['logs', 'outputs', 'preview_images', 'thumbnails']
+        colors = ['black', 'blue', 'red', 'green']
         for area, color in zip(arealist, colors):
 
             # Query for used sizes
-            results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area)
+            results = di.session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area).all()
 
-            # Group by date
             if results:
-                results_dict = defaultdict(int)
-                for date, value in results:
-                    results_dict[date] += value
+                # Convert to dataframe
+                results = pd.DataFrame(results)
 
-                # Parse results so they can be easily plotted
-                dates = list(results_dict.keys())
-                values = list(results_dict.values())
+                if not pd.isnull(self.delta_t):
+                    results = results[(results['date'] >= self.date - self.delta_t) & (results['date'] <= self.date)]
 
                 # Plot the results
-                plot.line(dates, values, legend='{} files'.format(area), line_color=color)
-                plot.circle(dates, values, color=color)
+                source = ColumnDataSource(results)
+                plot.line(x='date', y='used', source=source, line_color=color, legend_label=area, line_width=3)
+                plot.circle(x='date', y='used', source=source, color=color, size=10)
+
+                hover_tool = HoverTool(tooltips=[('Used:', '@used'),
+                                                 ('Date:', '@time{%d %b %Y}')
+                                                 ])
+                hover_tool.formatters = {'@time': 'datetime'}
+                plot.tools.append(hover_tool)
+
+        plot.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
+                                                     days=["%d %B %Y"],
+                                                     months=["%d %B %Y"],
+                                                     years=["%d %B %Y"],
+                                                     )
+        plot.xaxis.major_label_orientation = pi / 4
+        plot.legend.location = 'top_left'
 
-        session.close()
-        return plot
+        # Put the "all" plot in a separate figure because it will be larger than all the pieces, which would
+        # throw off the y range if it were in a single plot
+        cen_store_plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save',
+                                x_axis_type='datetime',
+                                title='JWQL central store directory, total data volume',
+                                x_axis_label='Date',
+                                y_axis_label='Size TB')
+
+        cen_store_results = di.session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == 'all').all()
+
+        # Group by date
+        if cen_store_results:
+
+            # Convert to dataframe
+            cen_store_results = pd.DataFrame(cen_store_results)
+
+            if not pd.isnull(self.delta_t):
+                cen_store_results = cen_store_results[(cen_store_results['date'] >= self.date - self.delta_t) & (cen_store_results['date'] <= self.date)]
+
+            # Group by date
+            cen_store_source = ColumnDataSource(cen_store_results)
+
+            # Plot the results
+            legend_str = 'File volume'
+            cen_store_plot.line(x='date', y='used', source=cen_store_source, legend_label=legend_str, line_color='blue', line_width=3)
+            cen_store_plot.circle(x='date', y='used', source=cen_store_source, color='blue', size=10)
+            cen_store_plot.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
+                                                                   days=["%d %B %Y"],
+                                                                   months=["%d %B %Y"],
+                                                                   years=["%d %B %Y"],
+                                                                   )
+            cen_store_plot.xaxis.major_label_orientation = pi / 4
+            cen_store_plot.legend.location = 'top_left'
+
+            hover_tool = HoverTool(tooltips=[('Used:', '@used'),
+                                             ('Date:', '@time{%d %b %Y}')
+                                             ])
+            hover_tool.formatters = {'@time': 'datetime'}
+            cen_store_plot.tools.append(hover_tool)
+
+        di.session.close()
+        return plot, cen_store_plot
 
 
     def dashboard_filetype_bar_chart(self):
@@ -217,9 +357,10 @@ def dashboard_filetype_bar_chart(self):
 
         # Keep only the rows containing the most recent timestamp
         data = data[data['date'] == data['date'].max()]
+        date_string = data['date'].max().strftime("%d %b %Y")
 
         # Set title and figures list to make panels
-        title = 'Files per Filetype by Instrument'
+        title = f'Files per Filetype by Instrument {date_string}'
         figures = []
 
         # For unique instrument values, loop through data
@@ -242,12 +383,12 @@ def dashboard_instrument_pie_chart(self):
         plot : bokeh.plotting.figure
             Pie chart figure
         """
-
         # Replace with jwql.website.apps.jwql.data_containers.build_table
         data = build_table('filesystem_instrument')
 
         # Keep only the rows containing the most recent timestamp
         data = data[data['date'] == data['date'].max()]
+        date_string = data['date'].max().strftime("%d %b %Y")
 
         try:
             file_counts = {'nircam': data[data.instrument == 'nircam']['count'].sum(),
@@ -265,12 +406,12 @@ def dashboard_instrument_pie_chart(self):
         data = pd.Series(file_counts).reset_index(name='value').rename(columns={'index': 'instrument'})
         data['angle'] = data['value'] / data['value'].sum() * 2 * pi
         data['color'] = ['#F8B195', '#F67280', '#C06C84', '#6C5B7B', '#355C7D']
-        plot = figure(title="Number of Files Per Instrument", toolbar_location=None,
+        plot = figure(title=f"Number of Files Per Instrument {date_string}", toolbar_location=None,
                       tools="hover,tap", tooltips="@instrument: @value", x_range=(-0.5, 1.0))
 
         plot.wedge(x=0, y=1, radius=0.4,
                    start_angle=cumsum('angle', include_zero=True), end_angle=cumsum('angle'),
-                   line_color="white", color='color', legend='instrument', source=data)
+                   line_color="white", color='color', legend_label='instrument', source=data)
 
         url = "{}/@instrument".format(get_base_url())
         taptool = plot.select(type=TapTool)
@@ -308,8 +449,8 @@ def dashboard_files_per_day(self):
         # Show date and used and available storage together
 
         p2 = figure(title="Available & Used Storage", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=1700, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]')
-        p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend='Available Storage')
-        p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend='Used Storage')
+        p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend_label='Available Storage')
+        p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend_label='Used Storage')
         p2.scatter(x='date', y='available', source=source, color='#C85108', size=10)
         p2.scatter(x='date', y='used', source=source, color='#C85108', size=10)
         disable_scientific_notation(p2)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index e739ac956..453e3e9e8 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -477,12 +477,16 @@ def dashboard(request):
     db = get_dashboard_components(request)
     pie_graph = db.dashboard_instrument_pie_chart()
     files_graph = db.dashboard_files_per_day()
+    useage_graph = db.dashboard_disk_usage()
+    directories_usage_graph, central_store_usage_graph = db.dashboard_central_store_data_volume()
     filetype_bar = db.dashboard_filetype_bar_chart()
     table_columns, table_values = db.dashboard_monitor_tracking()
     grating_plot = db.dashboard_exposure_count_by_filter()
     anomaly_plot = db.dashboard_anomaly_per_instrument()
 
-    plot = layout([[files_graph], [pie_graph, filetype_bar],
+    plot = layout([[files_graph, usage_graph],
+                   [directories_usage_graph, central_store_usage_graph],
+                   [pie_graph, filetype_bar],
                    [grating_plot, anomaly_plot]], sizing_mode='stretch_width')
     script, div = components(plot)
 

From 994ff3cfdfd7ee2124ee8174301672457e055624 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 20 Mar 2023 15:11:16 -0400
Subject: [PATCH 010/256] Remove unused skipped_files

---
 jwql/jwql_monitors/monitor_filesystem.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 39d38af24..255822d91 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -269,8 +269,6 @@ def get_area_stats(central_storage_dict):
                             if exists:
                                 filesize = os.path.getsize(file_path)
                                 sums += filesize
-                            else:
-                                skipped_files.append(filename)
             use = sums / (1024 ** 4)
         else:
             for dirpath, _, files in os.walk(fullpath):
@@ -282,12 +280,10 @@ def get_area_stats(central_storage_dict):
                         filesize = os.path.getsize(file_path)
                         used += filesize
                         sums += filesize
-                    else:
-                        skipped_files.append(filename)
             use = used / (1024 ** 4)
         central_storage_dict[area]['used'] = use
 
-    return central_storage_dict, skipped_files
+    return central_storage_dict
 
 
 def get_observation_characteristics():

From bd1ce38037647a4ce876a2605dfb271cb384db18 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 20 Mar 2023 20:46:16 -0400
Subject: [PATCH 011/256] Make subdir lists consistent

---
 jwql/jwql_monitors/monitor_filesystem.py | 11 ++++-------
 jwql/utils/constants.py                  |  3 +++
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 255822d91..7d33f4058 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -55,7 +55,8 @@
 from jwql.database.database_interface import CentralStore
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
-from jwql.utils.constants import FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import FILESYSYEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.utils import filename_parser
 from jwql.utils.utils import get_config
 from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table
@@ -227,7 +228,6 @@ def get_area_stats(central_storage_dict):
              'thumbnails': THUMBNAILS,
              'all': CENTRAL}
 
-    #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
     counteddirs = []
 
     sums = 0  # to be used to count 'all'
@@ -522,8 +522,6 @@ def plot_central_store_dirs():
     # Plot system stats vs. date
     results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all()
 
-    arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
-
     # Initialize plot
     dates, total_sizes, availables = zip(*results)
     plot = figure(
@@ -540,7 +538,7 @@ def plot_central_store_dirs():
     plot.circle(dates, availables, color='blue')
 
     # This part of the plot should cycle through areas and plot area used values vs. date
-    for area, color in zip(arealist, colors):
+    for area, color in zip(FILESYSYEM_MONITOR_SUBDIRS, colors):
 
         # Query for used sizes
         results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area)
@@ -706,8 +704,7 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
                 logging.error(e)
 
     # Add data to central_storage table
-    arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
-    for area in arealist:
+    for area in FILESYSYEM_MONITOR_SUBDIRS:
         new_record = {}
         new_record['date'] = central_storage_dict['date']
         new_record['area'] = area
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index ada1dc312..df64f69ba 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -228,6 +228,9 @@
                   'nirspec': ['NRS_AUTOFLAT', 'NRS_LAMP'],
                   'fgs': ['FGS_INTFLAT']}
 
+# output subdirectories to keep track of via the filesytem monitor
+FILESYSYEM_MONITOR_SUBDIRS = ['logs', 'outputs', 'preview_images', 'thumbnails', 'all']
+
 FILTERS_PER_INSTRUMENT = {'fgs': [],
                           'miri': ['F1000W', 'F1130W', 'F1280W', 'OPAQUE', 'F2300C', 'F560W', 'P750L',
                                    'F1500W', 'F2550W', 'F770W', 'FLENS', 'FND', 'F2100W', 'F1800W',

From 0e32ee1190da9b3fb44367e94fd66cc987a718dd Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 09:19:46 -0400
Subject: [PATCH 012/256] fix incorrect var name

---
 jwql/jwql_monitors/monitor_filesystem.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 7d33f4058..e153c7983 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -653,8 +653,8 @@ def update_characteristics_database(char_info):
 
     # Add data to filesystem_instrument table
     for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:
-        filter_list = [e[0] for e in obs['nircam']]
-        value_list= [e[1] for e in obs['nircam']]
+        filter_list = [e[0] for e in char_info['nircam']]
+        value_list= [e[1] for e in char_info['nircam']]
         new_record = {}
         new_record['date'] = now
         new_record['instrument'] = instrument

From fc13047ac4d11fc1f0fb26500ae7c71d028006db Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 12:25:42 -0400
Subject: [PATCH 013/256] missing import

---
 jwql/jwql_monitors/monitor_filesystem.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index e153c7983..9956104a8 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -50,7 +50,7 @@
 
 from jwql.database.database_interface import engine
 from jwql.database.database_interface import session
-from jwql.database.database_interface import FilesystemGeneral
+from jwql.database.database_interface import FilesystemCharacteristics, FilesystemGeneral
 from jwql.database.database_interface import FilesystemInstrument
 from jwql.database.database_interface import CentralStore
 from jwql.utils.logging_functions import log_info, log_fail

From 2e02f82ac41cffb3b794102109e15a9b0eb00158 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 15:24:05 -0400
Subject: [PATCH 014/256] Fix table ordering. Resize plots

---
 jwql/website/apps/jwql/bokeh_dashboard.py       | 15 +++++++++------
 jwql/website/apps/jwql/templates/dashboard.html |  8 ++++----
 jwql/website/apps/jwql/views.py                 |  2 +-
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 43ba11614..8a58d4ad9 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -35,7 +35,7 @@
 import os
 
 from bokeh.layouts import column
-from bokeh.models import Axis, ColumnDataSource, DatetimeTickFormatter, OpenURL, TapTool
+from bokeh.models import Axis, ColumnDataSource, DatetimeTickFormatter, HoverTool, OpenURL, TapTool
 from bokeh.models.widgets import Panel, Tabs
 from bokeh.plotting import figure
 from bokeh.transform import cumsum
@@ -212,6 +212,7 @@ def dashboard_disk_usage(self):
 
             # Initialize plot
             plots[data['shortname']] = figure(tools='pan,hover,box_zoom,wheel_zoom,reset,save',
+                                              plot_width=800,
                                               x_axis_type='datetime',
                                               title=f"Available & Used Storage on {data['shortname']}",
                                               x_axis_label='Date',
@@ -240,7 +241,7 @@ def dashboard_disk_usage(self):
 
         tabs = Tabs(tabs=tabs)
 
-        session.close()
+        di.session.close()
         return tabs
 
 
@@ -258,6 +259,7 @@ def dashboard_central_store_data_volume(self):
         """
         # Initialize plot
         plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save',
+                      plot_width=800,
                       x_axis_type='datetime',
                       title='JWQL directory size',
                       x_axis_label='Date',
@@ -301,6 +303,7 @@ def dashboard_central_store_data_volume(self):
         # Put the "all" plot in a separate figure because it will be larger than all the pieces, which would
         # throw off the y range if it were in a single plot
         cen_store_plot = figure(tools='pan,box_zoom,wheel_zoom,reset,save',
+                                plot_width=800,
                                 x_axis_type='datetime',
                                 title='JWQL central store directory, total data volume',
                                 x_axis_label='Date',
@@ -439,7 +442,7 @@ def dashboard_files_per_day(self):
         date_times = [pd.to_datetime(datetime).date() for datetime in source['date'].values]
         source['datestr'] = [date_time.strftime("%Y-%m-%d") for date_time in date_times]
 
-        p1 = figure(title="Number of Files in Filesystem", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=1700, x_axis_label='Date', y_axis_label='Number of Files Added')
+        p1 = figure(title="Number of Files in Filesystem", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Number of Files Added')
         p1.line(x='date', y='total_file_count', source=source, color='#6C5B7B', line_dash='dashed', line_width=3)
         p1.scatter(x='date', y='total_file_count', source=source, color='#C85108', size=10)
         disable_scientific_notation(p1)
@@ -448,7 +451,7 @@ def dashboard_files_per_day(self):
         # Create separate tooltip for storage plot.
         # Show date and used and available storage together
 
-        p2 = figure(title="Available & Used Storage", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=1700, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]')
+        p2 = figure(title="Available & Used Storage", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]')
         p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend_label='Available Storage')
         p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend_label='Used Storage')
         p2.scatter(x='date', y='available', source=source, color='#C85108', size=10)
@@ -552,8 +555,8 @@ def dashboard_exposure_count_by_filter(self):
         # This is a loop over instruments
         for i in range(len(data)):
             instrument = data.loc[i]['instrument']
-            filterpupil = data.loc[i]['filter_pupil']
-            num_obs = data.loc[i]['obs_per_filter_pupil']
+            filterpupil = np.array(data.loc[i]['filter_pupil'])
+            num_obs = np.array(data.loc[i]['obs_per_filter_pupil'])
 
             # Sort by num_obs in order to make the plot more readable
             idx = np.argsort(num_obs)
diff --git a/jwql/website/apps/jwql/templates/dashboard.html b/jwql/website/apps/jwql/templates/dashboard.html
index e81c8c7ec..d03ed7283 100644
--- a/jwql/website/apps/jwql/templates/dashboard.html
+++ b/jwql/website/apps/jwql/templates/dashboard.html
@@ -13,7 +13,7 @@ <h1 style="text-align: center">JWQL Dashboard</h1>
         <hr>
 
         <p>
-            The JWQL dashboard provides users with a "high level" status of our systems and tools. Here we provide some analytics and 
+            The JWQL dashboard provides users with a "high level" status of our systems and tools. Here we provide some analytics and
             statistics for the JWQL filesystem as well as the status of the instrument monitors. The figures below are interactive and some provide
             more detailed information if you hover over them and will take you to other landing pages within the application if clicked.
         </p>
@@ -25,7 +25,7 @@ <h4>Select Time Delta</h4>
                 {% for dt in time_deltas %}
                     <option value='{{ dt }}'>{{ dt }}</option>
                 {% endfor %}
-            </select> 
+            </select>
 
             <button name=time_delta_value class="btn btn-primary" type="submit"><span class="fas fa-search"></span></button>
 
@@ -34,7 +34,7 @@ <h4>Select Time Delta</h4>
             <body>
                 {{ div | safe }}
                 {{ script | safe }}
-                
+
                 <br>
                 <h3>JWQL Monitor Status</h3>
                 <hr>
@@ -87,7 +87,7 @@ <h3>JWQL Monitor Status</h3>
                 <script>
                     $(document).ready( function () {
                         var table = $('#jwqltable').DataTable({
-                            "order": [[ 2, "desc" ]],
+                            "order": [],
                             "createdRow": function( row, data, dataIndex ) {
                             if ( data[4] == "FAILURE" ) {
                                 $(row).addClass('red');
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 453e3e9e8..827771c90 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -484,7 +484,7 @@ def dashboard(request):
     grating_plot = db.dashboard_exposure_count_by_filter()
     anomaly_plot = db.dashboard_anomaly_per_instrument()
 
-    plot = layout([[files_graph, usage_graph],
+    plot = layout([[files_graph, useage_graph],
                    [directories_usage_graph, central_store_usage_graph],
                    [pie_graph, filetype_bar],
                    [grating_plot, anomaly_plot]], sizing_mode='stretch_width')

From d0b7e05a15fa4d9312be0ab46e2dce8eedbf40e6 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 16:04:21 -0400
Subject: [PATCH 015/256] loc to iloc

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 8a58d4ad9..63324d176 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -554,9 +554,9 @@ def dashboard_exposure_count_by_filter(self):
         figures = []
         # This is a loop over instruments
         for i in range(len(data)):
-            instrument = data.loc[i]['instrument']
-            filterpupil = np.array(data.loc[i]['filter_pupil'])
-            num_obs = np.array(data.loc[i]['obs_per_filter_pupil'])
+            instrument = data.iloc[i]['instrument']
+            filterpupil = np.array(data.iloc[i]['filter_pupil'])
+            num_obs = np.array(data.iloc[i]['obs_per_filter_pupil'])
 
             # Sort by num_obs in order to make the plot more readable
             idx = np.argsort(num_obs)

From 26e9066a11178ca61ba12e6354e437e8f053c9a7 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 16:38:52 -0400
Subject: [PATCH 016/256] Fix failing filter plots

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 63324d176..03172f25c 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -571,29 +571,29 @@ def dashboard_exposure_count_by_filter(self):
             for filt, val in zip(filterpupil, num_obs):
                 data_dict[filt] = val
 
-            data = pd.Series(data_dict).reset_index(name='value').rename(columns={'index': 'filter'})
+            inst_data = pd.Series(data_dict).reset_index(name='value').rename(columns={'index': 'filter'})
 
             if instrument != 'nircam':
                 # Calculate the angle covered by each filter
-                data['angle'] = data['value']/data['value'].sum() * 2 * np.pi
+                inst_data['angle'] = inst_data['value'] / inst_data['value'].sum() * 2 * np.pi
 
                 # Keep all wedges the same color, except for those that are a very
                 # small fraction, and will be covered in the second pie chart. Make
                 # those wedges grey in the primary pie chart.
-                data['colors'] = ['#c85108'] * len(data)
-                data.loc[data['value'] < 0.5, 'colors'] = '#bec4d4'
+                inst_data['colors'] = ['#c85108'] * len(inst_data)
+                inst_data.loc[inst_data['value'] < 0.5, 'colors'] = '#bec4d4'
 
                 # Make a dataframe containing only the filters that are used in less
                 # than some threshold percentage of observations
-                small = data.loc[data['value'] <0.5].copy()
+                small = inst_data.loc[inst_data['value'] < 0.5].copy()
 
                 # Recompute the angles for these, and make them all the same color.
                 small['angle'] = small['value'] / small['value'].sum() * 2 * np.pi
                 small['colors'] = ['#bec4d4'] * len(small)
 
                 # Create two pie charts
-                pie_fig = create_filter_based_pie_chart("Percentage of observations using filter/pupil combinations: All Filters", data)
-                small_pie_fig = create_filter_based_pie_chart("Low Percentage Filters (gray wedges from above)", sw_small)
+                pie_fig = create_filter_based_pie_chart("Percentage of observations using filter/pupil combinations: All Filters", inst_data)
+                small_pie_fig = create_filter_based_pie_chart("Low Percentage Filters (gray wedges from above)", small)
 
                 # Place the pie charts in a column/Panel, and append to the figure
                 colplots = column(pie_fig, small_pie_fig)
@@ -618,19 +618,19 @@ def dashboard_exposure_count_by_filter(self):
                             channel.append('LW')
                     else:
                         channel.append('SW')
-                data['channel'] = channel
+                inst_data['channel'] = channel
 
                 # Set the colors. All wedges with a pie chart have the same color.
                 color_options = {'LW': '#c85108', 'SW': '#3d85c6', 'Dark': '#bec4d4'}
                 colors = []
                 for entry in channel:
                     colors.append(color_options[entry])
-                data['colors'] = colors
+                inst_data['colors'] = colors
 
                 # Even though it's not quite correct, create separate charts for SW vs LW. This will
                 # hopefully make them much easier to read
-                sw_data = data.loc[data['channel'] == 'SW'].copy()
-                lw_data = data.loc[data['channel'] == 'LW'].copy()
+                sw_data = inst_data.loc[inst_data['channel'] == 'SW'].copy()
+                lw_data = inst_data.loc[inst_data['channel'] == 'LW'].copy()
 
                 # Recalculate the angles. These won't be strictly correct since SW and LW filters
                 # are not both used exactly 50% of the time, but it's close enough for now.

From 6359b6fdfac002e237bbfc4f5ff49d0ee16b7555 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 16:46:43 -0400
Subject: [PATCH 017/256] more fixes for filter based plots

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 30 +++++++++++------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 03172f25c..b15a45a32 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -746,21 +746,21 @@ def dashboard_exposure_count_by_filter(self):
                 figures.append(tab_sw)
                 figures.append(tab_lw)
 
-            # Add in a placeholder plot for FGS, in order to keep the page looking consistent
-            # from instrument to instrument
-            instrument = 'fgs'
-            data_dict = {}
-            data_dict['None'] = 100.
-            data = pd.Series(data_dict).reset_index(name='value').rename(columns={'index': 'filter'})
-            data['angle'] = 2 * np.pi
-            data['colors'] = ['#c85108']
-            pie_fig = create_filter_based_pie_chart("FGS has no filters", data)
-            small_pie_fig = create_filter_based_pie_chart("FGS has no filters", data)
-
-            # Place the pie charts in a column/Panel, and append to the figure
-            colplots = column(pie_fig, small_pie_fig)
-            tab = Panel(child=colplots, title=f'{instrument}')
-            figures.append(tab)
+        # Add in a placeholder plot for FGS, in order to keep the page looking consistent
+        # from instrument to instrument
+        instrument = 'fgs'
+        data_dict = {}
+        data_dict['None'] = 100.
+        inst_data = pd.Series(data_dict).reset_index(name='value').rename(columns={'index': 'filter'})
+        inst_data['angle'] = 2 * np.pi
+        inst_data['colors'] = ['#c85108']
+        pie_fig = create_filter_based_pie_chart("FGS has no filters", inst_data)
+        small_pie_fig = create_filter_based_pie_chart("FGS has no filters", inst_data)
+
+        # Place the pie charts in a column/Panel, and append to the figure
+        colplots = column(pie_fig, small_pie_fig)
+        tab = Panel(child=colplots, title=f'{instrument}')
+        figures.append(tab)
 
         tabs = Tabs(tabs=figures)
 

From 23db765f3abc461b38857111ea6784b8fd715708 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 16:56:56 -0400
Subject: [PATCH 018/256] update db using all instruments

---
 jwql/jwql_monitors/monitor_filesystem.py  | 4 ++--
 jwql/website/apps/jwql/bokeh_dashboard.py | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 9956104a8..46386a83b 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -653,8 +653,8 @@ def update_characteristics_database(char_info):
 
     # Add data to filesystem_instrument table
     for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:
-        filter_list = [e[0] for e in char_info['nircam']]
-        value_list= [e[1] for e in char_info['nircam']]
+        filter_list = [e[0] for e in char_info[instrument]]
+        value_list= [e[1] for e in char_info[instrument]]
         new_record = {}
         new_record['date'] = now
         new_record['instrument'] = instrument
diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index b15a45a32..14721729c 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -451,7 +451,7 @@ def dashboard_files_per_day(self):
         # Create separate tooltip for storage plot.
         # Show date and used and available storage together
 
-        p2 = figure(title="Available & Used Storage", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]')
+        p2 = figure(title="Available & Used Storage in Filesystem", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]')
         p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend_label='Available Storage')
         p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend_label='Used Storage')
         p2.scatter(x='date', y='available', source=source, color='#C85108', size=10)
@@ -472,6 +472,7 @@ def dashboard_files_per_day(self):
                                                    years=["%d %B %Y"],
                                                    )
         p2.xaxis.major_label_orientation = pi / 4
+        p2.legend.location = 'top_left'
 
         tabs = Tabs(tabs=[tab1, tab2])
 

From 2ad087870ee65575c3e492d54e46dbf1edae8b11 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 17:12:10 -0400
Subject: [PATCH 019/256] Tweak plot colors and line styles

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 14721729c..0e8652692 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -218,10 +218,10 @@ def dashboard_disk_usage(self):
                                               x_axis_label='Date',
                                               y_axis_label='Size TB')
 
-            plots[data['shortname']].line(x='date', y='available', source=source, legend_label='Available', line_color='red',line_width=3)
-            plots[data['shortname']].circle(x='date', y='available', source=source,color='red', size=10)
-            plots[data['shortname']].line(x='date', y='used', source=source, legend_label='Used', line_color='blue', line_width=3)
-            plots[data['shortname']].circle(x='date', y='used', source=source,color='blue', size=10)
+            plots[data['shortname']].line(x='date', y='available', source=source, legend_label='Available', line_dash='dashed', line_color='#C85108', line_width=3)
+            plots[data['shortname']].circle(x='date', y='available', source=source,color='#C85108', size=10)
+            plots[data['shortname']].line(x='date', y='used', source=source, legend_label='Used', line_dash='dashed', line_color='#355C7D', line_width=3)
+            plots[data['shortname']].circle(x='date', y='used', source=source,color='#355C7D', size=10)
 
             plots[data['shortname']].xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
                                                        days=["%d %B %Y"],
@@ -268,7 +268,7 @@ def dashboard_central_store_data_volume(self):
         # This part of the plot should cycle through areas and plot area used values vs. date
         #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
         arealist = ['logs', 'outputs', 'preview_images', 'thumbnails']
-        colors = ['black', 'blue', 'red', 'green']
+        colors = ['#F8B195', '#F67280', '#6C5B7B', '#355C7D']
         for area, color in zip(arealist, colors):
 
             # Query for used sizes
@@ -283,7 +283,7 @@ def dashboard_central_store_data_volume(self):
 
                 # Plot the results
                 source = ColumnDataSource(results)
-                plot.line(x='date', y='used', source=source, line_color=color, legend_label=area, line_width=3)
+                plot.line(x='date', y='used', source=source, line_color=color, line_dash='dashed', legend_label=area, line_width=3)
                 plot.circle(x='date', y='used', source=source, color=color, size=10)
 
                 hover_tool = HoverTool(tooltips=[('Used:', '@used'),
@@ -325,8 +325,8 @@ def dashboard_central_store_data_volume(self):
 
             # Plot the results
             legend_str = 'File volume'
-            cen_store_plot.line(x='date', y='used', source=cen_store_source, legend_label=legend_str, line_color='blue', line_width=3)
-            cen_store_plot.circle(x='date', y='used', source=cen_store_source, color='blue', size=10)
+            cen_store_plot.line(x='date', y='used', source=cen_store_source, legend_label=legend_str, line_dash='dashed', line_color='#355C7D', line_width=3)
+            cen_store_plot.circle(x='date', y='used', source=cen_store_source, color='#355C7D', size=10)
             cen_store_plot.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
                                                                    days=["%d %B %Y"],
                                                                    months=["%d %B %Y"],
@@ -452,10 +452,10 @@ def dashboard_files_per_day(self):
         # Show date and used and available storage together
 
         p2 = figure(title="Available & Used Storage in Filesystem", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]')
-        p2.line(x='date', y='available', source=source, color='#F8B195', line_dash='dashed', line_width=3, legend_label='Available Storage')
+        p2.line(x='date', y='available', source=source, color='#C85108', line_dash='dashed', line_width=3, legend_label='Available Storage')
         p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend_label='Used Storage')
         p2.scatter(x='date', y='available', source=source, color='#C85108', size=10)
-        p2.scatter(x='date', y='used', source=source, color='#C85108', size=10)
+        p2.scatter(x='date', y='used', source=source, color='#355C7D', size=10)
         disable_scientific_notation(p2)
         tab2 = Panel(child=p2, title='Storage')
 

From 9cb863ad3a08ca9de63cdd9a6131138274155c68 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 21 Mar 2023 17:22:21 -0400
Subject: [PATCH 020/256] Make plot labels consistent

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 30 +++++++++++------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 0e8652692..c02fd0956 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -216,17 +216,17 @@ def dashboard_disk_usage(self):
                                               x_axis_type='datetime',
                                               title=f"Available & Used Storage on {data['shortname']}",
                                               x_axis_label='Date',
-                                              y_axis_label='Size TB')
+                                              y_axis_label='Disk Space (TB)')
 
             plots[data['shortname']].line(x='date', y='available', source=source, legend_label='Available', line_dash='dashed', line_color='#C85108', line_width=3)
             plots[data['shortname']].circle(x='date', y='available', source=source,color='#C85108', size=10)
             plots[data['shortname']].line(x='date', y='used', source=source, legend_label='Used', line_dash='dashed', line_color='#355C7D', line_width=3)
             plots[data['shortname']].circle(x='date', y='used', source=source,color='#355C7D', size=10)
 
-            plots[data['shortname']].xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
+            plots[data['shortname']].xaxis.formatter = DatetimeTickFormatter(hours=["%H:%M %d %B %Y"],
                                                        days=["%d %B %Y"],
                                                        months=["%d %B %Y"],
-                                                       years=["%d %B %Y"],
+                                                       years=["%B %Y"],
                                                        )
             plots[data['shortname']].xaxis.major_label_orientation = pi / 4
             plots[data['shortname']].legend.location = 'top_left'
@@ -263,7 +263,7 @@ def dashboard_central_store_data_volume(self):
                       x_axis_type='datetime',
                       title='JWQL directory size',
                       x_axis_label='Date',
-                      y_axis_label='Size TB')
+                      y_axis_label='Disk Space (TB)')
 
         # This part of the plot should cycle through areas and plot area used values vs. date
         #arealist = ['logs', 'outputs', 'test', 'preview_images', 'thumbnails', 'all']
@@ -292,10 +292,10 @@ def dashboard_central_store_data_volume(self):
                 hover_tool.formatters = {'@time': 'datetime'}
                 plot.tools.append(hover_tool)
 
-        plot.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
+        plot.xaxis.formatter = DatetimeTickFormatter(hours=["%H:%M %d %B %Y"],
                                                      days=["%d %B %Y"],
                                                      months=["%d %B %Y"],
-                                                     years=["%d %B %Y"],
+                                                     years=["%B %Y"],
                                                      )
         plot.xaxis.major_label_orientation = pi / 4
         plot.legend.location = 'top_left'
@@ -307,7 +307,7 @@ def dashboard_central_store_data_volume(self):
                                 x_axis_type='datetime',
                                 title='JWQL central store directory, total data volume',
                                 x_axis_label='Date',
-                                y_axis_label='Size TB')
+                                y_axis_label='Disk Space (TB)')
 
         cen_store_results = di.session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == 'all').all()
 
@@ -327,10 +327,10 @@ def dashboard_central_store_data_volume(self):
             legend_str = 'File volume'
             cen_store_plot.line(x='date', y='used', source=cen_store_source, legend_label=legend_str, line_dash='dashed', line_color='#355C7D', line_width=3)
             cen_store_plot.circle(x='date', y='used', source=cen_store_source, color='#355C7D', size=10)
-            cen_store_plot.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
+            cen_store_plot.xaxis.formatter = DatetimeTickFormatter(hours=["%H:%M %d %B %Y"],
                                                                    days=["%d %B %Y"],
                                                                    months=["%d %B %Y"],
-                                                                   years=["%d %B %Y"],
+                                                                   years=["%B %Y"],
                                                                    )
             cen_store_plot.xaxis.major_label_orientation = pi / 4
             cen_store_plot.legend.location = 'top_left'
@@ -442,7 +442,7 @@ def dashboard_files_per_day(self):
         date_times = [pd.to_datetime(datetime).date() for datetime in source['date'].values]
         source['datestr'] = [date_time.strftime("%Y-%m-%d") for date_time in date_times]
 
-        p1 = figure(title="Number of Files in Filesystem", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Number of Files Added')
+        p1 = figure(title="Number of Files in Filesystem (MAST)", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Number of Files Added')
         p1.line(x='date', y='total_file_count', source=source, color='#6C5B7B', line_dash='dashed', line_width=3)
         p1.scatter(x='date', y='total_file_count', source=source, color='#C85108', size=10)
         disable_scientific_notation(p1)
@@ -451,7 +451,7 @@ def dashboard_files_per_day(self):
         # Create separate tooltip for storage plot.
         # Show date and used and available storage together
 
-        p2 = figure(title="Available & Used Storage in Filesystem", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Storage Space [Terabytes?]')
+        p2 = figure(title="Available & Used Storage in Filesystem (MAST)", tools="reset,hover,box_zoom,wheel_zoom", tooltips="@datestr: @total_file_count", plot_width=800, x_axis_label='Date', y_axis_label='Disk Space (TB)')
         p2.line(x='date', y='available', source=source, color='#C85108', line_dash='dashed', line_width=3, legend_label='Available Storage')
         p2.line(x='date', y='used', source=source, color='#355C7D', line_dash='dashed', line_width=3, legend_label='Used Storage')
         p2.scatter(x='date', y='available', source=source, color='#C85108', size=10)
@@ -459,17 +459,17 @@ def dashboard_files_per_day(self):
         disable_scientific_notation(p2)
         tab2 = Panel(child=p2, title='Storage')
 
-        p1.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
+        p1.xaxis.formatter = DatetimeTickFormatter(hours=["%H:%M %d %B %Y"],
                                                    days=["%d %B %Y"],
                                                    months=["%d %B %Y"],
-                                                   years=["%d %B %Y"],
+                                                   years=["%B %Y"],
                                                    )
         p1.xaxis.major_label_orientation = pi / 4
 
-        p2.xaxis.formatter = DatetimeTickFormatter(hours=["%d %B %Y"],
+        p2.xaxis.formatter = DatetimeTickFormatter(hours=["%H:%M %d %B %Y"],
                                                    days=["%d %B %Y"],
                                                    months=["%d %B %Y"],
-                                                   years=["%d %B %Y"],
+                                                   years=["%B %Y"],
                                                    )
         p2.xaxis.major_label_orientation = pi / 4
         p2.legend.location = 'top_left'

From fb202072acdb41105e9acac37ab81b775e6f611b Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 22 Mar 2023 15:11:21 -0400
Subject: [PATCH 021/256] Turn off plot making at time of monitor run

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index c02fd0956..415b19c43 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -45,7 +45,7 @@
 
 import jwql.database.database_interface as di
 from jwql.database.database_interface import CentralStore
-from jwql.utils.constants import ANOMALY_CHOICES_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT
+from jwql.utils.constants import ANOMALY_CHOICES_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.utils import get_base_url, get_config
 from jwql.website.apps.jwql.data_containers import build_table
 
@@ -363,7 +363,6 @@ def dashboard_filetype_bar_chart(self):
         date_string = data['date'].max().strftime("%d %b %Y")
 
         # Set title and figures list to make panels
-        title = f'Files per Filetype by Instrument {date_string}'
         figures = []
 
         # For unique instrument values, loop through data
@@ -372,6 +371,7 @@ def dashboard_filetype_bar_chart(self):
         for instrument in data.instrument.unique():
             index = data["instrument"] == instrument
             inst_only = data[index].sort_values('filetype')
+            title = f'{JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()]} files per Filetype: {date_string}'
             figures.append(self.make_panel(inst_only['filetype'], inst_only['count'], instrument, title, 'File Type'))
 
         tabs = Tabs(tabs=figures)

From dd0d2eefbf8c9b221ce9e3f812d0ba533634c55e Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 28 Mar 2023 16:57:55 -0400
Subject: [PATCH 022/256] Tweaks

---
 jwql/jwql_monitors/monitor_filesystem.py  | 36 +++++++++++++----------
 jwql/website/apps/jwql/bokeh_dashboard.py |  8 ++---
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 46386a83b..98e2f1837 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -406,9 +406,10 @@ def monitor_filesystem():
     # Add data to database tables
     update_database(general_results_dict, instrument_results_dict, central_storage_dict)
     update_characteristics_database(characteristics)
+    update_central_store_database(central_storage_dict)
 
     # Create the plots
-    plot_filesystem_stats()
+    #plot_filesystem_stats()
 
 
 def plot_by_filetype(plot_type, instrument):
@@ -638,6 +639,25 @@ def plot_total_file_counts():
 
     return plot
 
+def update_central_store_database(central_storage_dict):
+    """Updates the ``CentralStore`` database table with info on disk space
+
+    Parameters
+    ----------
+    central_storage_dict : dict
+        A dictionary for the ``central_storage`` database table
+    """
+    for area in FILESYSYEM_MONITOR_SUBDIRS:
+        new_record = {}
+        new_record['date'] = central_storage_dict['date']
+        new_record['area'] = area
+        new_record['size'] = central_storage_dict[area]['size']
+        new_record['used'] = central_storage_dict[area]['used']
+        new_record['available'] = central_storage_dict[area]['available']
+        engine.execute(CentralStore.__table__.insert(), new_record)
+        session.commit()
+    session.close()
+
 
 def update_characteristics_database(char_info):
     """Updates the ``filesystem_characteristics`` database table.
@@ -676,9 +696,6 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
         A dictionary for the ``filesystem_general`` database table
     instrument_results_dict : dict
         A dictionary for the ``filesystem_instrument`` database table
-    central_storage_dict : dict
-        A dictionary for the ``central_storage`` database table
-
     """
     logging.info('\tUpdating the database')
 
@@ -703,17 +720,6 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
             except DataError as e:
                 logging.error(e)
 
-    # Add data to central_storage table
-    for area in FILESYSYEM_MONITOR_SUBDIRS:
-        new_record = {}
-        new_record['date'] = central_storage_dict['date']
-        new_record['area'] = area
-        new_record['size'] = central_storage_dict[area]['size']
-        new_record['used'] = central_storage_dict[area]['used']
-        new_record['available'] = central_storage_dict[area]['available']
-        engine.execute(CentralStore.__table__.insert(), new_record)
-        session.commit()
-
     session.close()
 
 
diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 415b19c43..932b71cbd 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -286,10 +286,10 @@ def dashboard_central_store_data_volume(self):
                 plot.line(x='date', y='used', source=source, line_color=color, line_dash='dashed', legend_label=area, line_width=3)
                 plot.circle(x='date', y='used', source=source, color=color, size=10)
 
-                hover_tool = HoverTool(tooltips=[('Used:', '@used'),
-                                                 ('Date:', '@time{%d %b %Y}')
+                hover_tool = HoverTool(tooltips=[('Used:', f'@used TB'),
+                                                 ('Date:', '@date{%d %b %Y}')
                                                  ])
-                hover_tool.formatters = {'@time': 'datetime'}
+                hover_tool.formatters = {'@date': 'datetime'}
                 plot.tools.append(hover_tool)
 
         plot.xaxis.formatter = DatetimeTickFormatter(hours=["%H:%M %d %B %Y"],
@@ -335,7 +335,7 @@ def dashboard_central_store_data_volume(self):
             cen_store_plot.xaxis.major_label_orientation = pi / 4
             cen_store_plot.legend.location = 'top_left'
 
-            hover_tool = HoverTool(tooltips=[('Used:', '@used'),
+            hover_tool = HoverTool(tooltips=[('Used:', f'@used TB'),
                                              ('Date:', '@time{%d %b %Y}')
                                              ])
             hover_tool.formatters = {'@time': 'datetime'}

From 63e44bf3d588cc473e70bf97ecaa08606cb63cac Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 31 Mar 2023 16:34:07 -0400
Subject: [PATCH 023/256] update constants with team feedback

---
 jwql/utils/constants.py         | 76 +++++++++++++++++++--------------
 jwql/website/apps/jwql/forms.py | 26 ++++++++++-
 jwql/website/apps/jwql/views.py | 13 ++++--
 3 files changed, 77 insertions(+), 38 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 0e0a9cd69..432344cea 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -11,6 +11,7 @@
     - Mike Engesser
     - Maria Pena-Guerrero
     - Rachel Cooper
+    - Brad Sappington
 
 Use
 ---
@@ -88,18 +89,18 @@
     # additional anomalies:
     'other': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec']}
 # anomalies that shouldn't be 'titleized'
-special_cases = ['Dominant_MSA_Leakage','MRS_Glow','MRS_Zipper','LRS_Contamination']
+special_cases = ['Dominant_MSA_Leakage', 'MRS_Glow', 'MRS_Zipper', 'LRS_Contamination']
 
 # Defines the possible anomalies to flag through the web app
 ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) if anomaly not in special_cases
-                        else (anomaly, anomaly.replace('_',' '))
-                        for anomaly in ANOMALIES_PER_INSTRUMENT]
+                   else (anomaly, anomaly.replace('_', ' '))
+                   for anomaly in ANOMALIES_PER_INSTRUMENT]
 
 ANOMALY_CHOICES_FGS = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT
                        if 'fgs' in ANOMALIES_PER_INSTRUMENT[anomaly]]
 
 ANOMALY_CHOICES_MIRI = [(anomaly, inflection.titleize(anomaly)) if anomaly not in special_cases
-                        else (anomaly, anomaly.replace('_',' '))
+                        else (anomaly, anomaly.replace('_', ' '))
                         for anomaly in ANOMALIES_PER_INSTRUMENT
                         if 'miri' in ANOMALIES_PER_INSTRUMENT[anomaly]]
 
@@ -110,9 +111,9 @@
                           if 'niriss' in ANOMALIES_PER_INSTRUMENT[anomaly]]
 
 ANOMALY_CHOICES_NIRSPEC = [(anomaly, inflection.titleize(anomaly)) if anomaly not in special_cases
-                            else (anomaly, anomaly.replace('_',' '))
-                            for anomaly in ANOMALIES_PER_INSTRUMENT
-                            if 'nirspec' in ANOMALIES_PER_INSTRUMENT[anomaly]]
+                           else (anomaly, anomaly.replace('_', ' '))
+                           for anomaly in ANOMALIES_PER_INSTRUMENT
+                           if 'nirspec' in ANOMALIES_PER_INSTRUMENT[anomaly]]
 
 ANOMALY_CHOICES_PER_INSTRUMENT = {'fgs': ANOMALY_CHOICES_FGS,
                                   'miri': ANOMALY_CHOICES_MIRI,
@@ -121,18 +122,12 @@
                                   'nirspec': ANOMALY_CHOICES_NIRSPEC
                                   }
 
-APERTURES_PER_INSTRUMENT = {'NIRCAM': ['NRCA1_FULL', 'NRCA2_FULL', 'NRCA3_FULL', 'NRCA4_FULL',
-                                       'NRCA5_FULL', 'NRCB1_FULL', 'NRCB2_FULL', 'NRCB3_FULL',
-                                       'NRCB4_FULL', 'NRCB5_FULL'],
-                            'NIRISS': ['NIS_CEN', 'NIS_SOSSFULL', 'NIS_AMIFULL', 'NIS_AMI1',
-                                       'NIS_SUBSTRIP256', 'NIS_SUBSTRIP96',
-                                       'NIS_SUB64', 'NIS_SUB128', 'NIS_SUB256'],
-                            'NIRSPEC': ['NRS_FULL_MSA', 'NRS_FULL_IFU', 'NRS_S200A1_SLIT', 'NRS_S200A2_SLIT',
+APERTURES_PER_INSTRUMENT = {'nircam': [],  # NIRCAM aperture redundant, can just use Subarray + Detector
+                            'niriss': [],  # NIRISS preferred subarray only
+                            'nirspec': ['NRS_FULL_MSA', 'NRS_FULL_IFU', 'NRS_S200A1_SLIT', 'NRS_S200A2_SLIT',
                                         'NRS_S400A1_SLIT', 'NRS_S1600A1_SLIT', 'NRS_S200B1_SLIT'],
-                            'MIRI': ['MIRIM_SUB64', 'MIRIM_SUB128', 'MIRIM_SUB256', 'MIRIM_MASK1140',
-                                     'MIRIM_MASK1065', 'MIRIM_MASK1550', 'MIRIM_MASKLYOT',
-                                     'MIRIM_BRIGHTSKY', 'MIRIM_SLITLESSPRISM'],
-                            'FGS': ['FGS1_FULL', 'FGS2_FULL']}
+                            'miri': [],   # MIRI preferred subarray only
+                            'fgs': ['FGS1_FULL', 'FGS2_FULL']}
 
 # Observing templates used for ASIC tuning. MAST query results that
 # have one of these templates will be ignored
@@ -235,16 +230,12 @@
                   'fgs': ['FGS_INTFLAT']}
 
 FILTERS_PER_INSTRUMENT = {'fgs': [],
-                          'miri': ['F1000W', 'F1130W', 'F1280W', 'OPAQUE', 'F2300C', 'F560W', 'P750L',
-                                   'F1500W', 'F2550W', 'F770W', 'FLENS', 'FND', 'F2100W', 'F1800W',
-                                   'F1550C', 'F1140C', 'F2550WR', 'F1065C'],
-                          'nircam': ['F070W', 'F090W', 'F115W', 'F140M', 'F150W', 'F150W2', 'F182M',
-                                     'F187N', 'F200W', 'F210M', 'F212N', 'F250M', 'F277W', 'F300M',
-                                     'F322W2', 'F335M', 'F356W', 'F360M', 'F410M', 'F430M', 'F444W',
-                                     'F460M', 'F480M'],
-                          'niriss': ['CLEAR', 'F380M', 'F480M', 'GR150R', 'F430M', 'GR150C', 'F444W',
-                                     'F356W', 'F277W'],
-                          'nirspec': ['F290LP', 'F170LP', 'OPAQUE', 'F100LP', 'F070LP', 'F140X', 'CLEAR', 'F110W']}
+                          'miri': ['F560W', 'F770W', 'F1000W', 'F1065C', 'F1130W', 'F1140C', 'F1280W', 'F1500W',
+                                   'F1550C', 'F1800W', 'F2100W', 'F2300C', 'F2550W', 'F2550WR', 'FLENS', 'FND', 'OPAQUE', 'P750L'],
+                          'nircam': ['F070W', 'F090W', 'F115W', 'F140M', 'F150W', 'F150W2', 'F182M', 'F187N', 'F200W', 'F210M', 'F212N',
+                                     'WLP4', 'F277W', 'F356W', 'F444W', 'F300M', 'F335M', 'F360M', 'F410M', 'F430M', 'F460M', 'F480M', 'F250M', 'F322W2'],
+                          'niriss': ['F090W', 'F115W', 'F140M', 'F150W', 'F200W', 'F277W', 'F356W', 'F380M', 'F430M', 'F444W', 'F480M', 'GR150C', 'GR150R'],
+                          'nirspec': ['CLEAR', 'F070LP', 'F100LP', 'F110W', 'F140X', 'F170LP', 'F290LP', 'OPAQUE', 'P750L']}
 
 FOUR_AMP_SUBARRAYS = ['WFSS128R', 'WFSS64R']
 
@@ -269,7 +260,7 @@
                           'nircam': [],
                           'niriss': [],
                           'nirspec': ['G140M', 'G235M', 'G395M', 'G140H',
-                                      'G235H', 'G395H', 'PRISM']
+                                      'G235H', 'G395H', 'PRISM', 'MIRROR']
                           }
 
 # Filename extensions for guider data
@@ -353,15 +344,15 @@
                                 },
                          'CCC': {'CLOSED': (398.0376386, 0.173703628),
                                  'OPEN': (504.0482685, 0.328112274)
-                                },
+                                 },
                          'GW14': {'SHORT': (626.9411005, 0.116034024),
                                   'MEDIUM': (342.8685233, 0.127123169),
                                   'LONG': (408.8339259, 0.117079193)
-                                 },
+                                  },
                          'GW23': {'SHORT': (619.7948107, 0.215417336),
                                   'MEDIUM': (373.1697309, 0.204314122),
                                   'LONG': (441.6632325, 0.349161169)
-                                 }
+                                  }
                          }
 
 # Suffix for msa files
@@ -424,6 +415,13 @@
 # The complete name will have "_{instrument.lower}.txt" added to the end of this.
 PREVIEW_IMAGE_LISTFILE = 'preview_image_inventory'
 
+PUPILS_PER_INSTRUMENT = {'nircam': ['CLEAR', 'FLAT', 'F162M', 'F164N', 'GDHS0', 'GDHS60', 'MASKBAR', 'MASKIPR', 'MASKRND',
+                                    'PINHOLES', 'WLM8', 'WLP8', 'F323N', 'F405N', 'F466N', 'F470N', 'GRISMC', 'GRISMR', 'GRISMV2', 'GRISMV3'],
+                         'niriss': ['CLEARP', 'F090W', 'F115W', 'F140M', 'F150W', 'F158M', 'F200W', 'GR700XD', 'NRM'],
+                         'nirspec': [],
+                         'miri': [],
+                         'fgs': []}
+
 
 # Keep keys defined via class as they are used many places with potential mispellings
 class QUERY_CONFIG_KEYS:
@@ -436,7 +434,9 @@ class QUERY_CONFIG_KEYS:
     FILTERS = "FILTERS"
     GRATINGS = "GRATINGS"
     INSTRUMENTS = "INSTRUMENTS"
+    PUPILS = "PUPILS"
     READ_PATTS = "READ_PATTS"
+    SUBARRAYS = "SUBARRAYS"
     THUMBNAILS = "THUMBNAILS"
 
 
@@ -451,7 +451,9 @@ class QUERY_CONFIG_KEYS:
     QUERY_CONFIG_KEYS.FILTERS: {},
     QUERY_CONFIG_KEYS.GRATINGS: {},
     QUERY_CONFIG_KEYS.INSTRUMENTS: [],
+    QUERY_CONFIG_KEYS.PUPILS: {},
     QUERY_CONFIG_KEYS.READ_PATTS: {},
+    QUERY_CONFIG_KEYS.SUBARRAYS: {},
     QUERY_CONFIG_KEYS.THUMBNAILS: []
     }
 
@@ -493,6 +495,16 @@ class QUERY_CONFIG_KEYS:
 
 SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256']
 
+SUBARRAYS_PER_INSTRUMENT = {'nircam': ['FULL', 'FULLP', 'SUB640', 'SUB320', 'SUB160', 'SUB400P', 'SUB160P', 'SUB64P',
+                                       'SUB32TATS', 'SUB640A210R', 'SUB640ASWB', 'SUB320A335R', 'SUB320A430R', 'SUB320ALWB',
+                                       'SUBGRISM256', 'SUBGRISM128', 'SUBGRISM64', 'SUB32TATSGRISM'],
+                            'niriss': ['FULL', 'SUBSTRIP96', 'SUBSTRIP256', 'SUB80', 'SUB64', 'SUB128', 'SUB256',
+                                       'WFSS64R', 'WFSS128R', 'WFSS64C', 'WFSS128C', 'SUBAMPCAL', 'SUBTAAMI', 'SUBTASOSS'],
+                            'nirspec': [],
+                            'miri': ['MIRIM_BRIGHTSKY', 'MIRIM_FULL', 'MIRIM_MASK1065', 'MIRIM_MASK1140',
+                                     'MIRIM_MASK1550', 'MIRIM_MASKLYOT', 'MIRIM_SLITLESSPRISM', 'MIRIM_SUB64', 'MIRIM_SUB128', 'MIRIM_SUB256'],
+                            'fgs': []}
+
 # Filename suffixes that need to include the association value in the suffix in
 # order to identify the preview image file. This should only be crf and crfints,
 # since those are essentially level 2 files that are output by the level 3 pipeline.
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index ee97e96ce..f2fea0b38 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -61,7 +61,7 @@ def view_function(request):
 
 from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, ANOMALIES_PER_INSTRUMENT, APERTURES_PER_INSTRUMENT, DETECTOR_PER_INSTRUMENT, EXP_TYPE_PER_INSTRUMENT,
                                   FILTERS_PER_INSTRUMENT, GENERIC_SUFFIX_TYPES, GRATING_PER_INSTRUMENT, GUIDER_FILENAME_TYPE, JWST_INSTRUMENT_NAMES_MIXEDCASE,
-                                  JWST_INSTRUMENT_NAMES_SHORTHAND, READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES)
+                                  JWST_INSTRUMENT_NAMES_SHORTHAND, READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES, SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT)
 from jwql.utils.utils import (get_config, get_rootnames_for_instrument_proposal, filename_parser, query_format)
 
 from wtforms import SubmitField, StringField
@@ -93,9 +93,11 @@ class JwqlQueryForm(BaseForm):
         params[instrument]['readpatt_list'] = []
         params[instrument]['exptype_list'] = []
         params[instrument]['grating_list'] = []
+        params[instrument]['subarray_list'] = []
+        params[instrument]['pupil_list'] = []
         params[instrument]['anomalies_list'] = []
         # Generate dynamic lists of apertures to use in forms
-        for aperture in APERTURES_PER_INSTRUMENT[instrument.upper()]:
+        for aperture in APERTURES_PER_INSTRUMENT[instrument.lower()]:
             params[instrument]['aperture_list'].append([query_format(aperture), query_format(aperture)])
         # Generate dynamic lists of filters to use in forms
         for filt in FILTERS_PER_INSTRUMENT[instrument]:
@@ -117,6 +119,14 @@ class JwqlQueryForm(BaseForm):
         for grating in GRATING_PER_INSTRUMENT[instrument]:
             grating = query_format(grating)
             params[instrument]['grating_list'].append([grating, grating])
+        # Generate dynamic lists of subarray options to use in forms
+        for subarray in SUBARRAYS_PER_INSTRUMENT[instrument]:
+            subarray = query_format(subarray)
+            params[instrument]['subarray_list'].append([subarray, subarray])
+        # Generate dynamic lists of pupil options to use in forms
+        for pupil in PUPILS_PER_INSTRUMENT[instrument]:
+            pupil = query_format(pupil)
+            params[instrument]['pupil_list'].append([pupil, pupil])
         # Generate dynamic lists of anomalies to use in forms
         for anomaly in ANOMALIES_PER_INSTRUMENT.keys():
             if instrument in ANOMALIES_PER_INSTRUMENT[anomaly]:
@@ -172,6 +182,18 @@ class JwqlQueryForm(BaseForm):
     nircam_grating = forms.MultipleChoiceField(required=False, choices=params['nircam']['grating_list'], widget=forms.CheckboxSelectMultiple)
     fgs_grating = forms.MultipleChoiceField(required=False, choices=params['fgs']['grating_list'], widget=forms.CheckboxSelectMultiple)
 
+    miri_subarray = forms.MultipleChoiceField(required=False, choices=params['miri']['subarray_list'], widget=forms.CheckboxSelectMultiple)
+    nirspec_subarray = forms.MultipleChoiceField(required=False, choices=params['nirspec']['subarray_list'], widget=forms.CheckboxSelectMultiple)
+    niriss_subarray = forms.MultipleChoiceField(required=False, choices=params['niriss']['subarray_list'], widget=forms.CheckboxSelectMultiple)
+    nircam_subarray = forms.MultipleChoiceField(required=False, choices=params['nircam']['subarray_list'], widget=forms.CheckboxSelectMultiple)
+    fgs_subarray = forms.MultipleChoiceField(required=False, choices=params['fgs']['subarray_list'], widget=forms.CheckboxSelectMultiple)
+
+    miri_pupil = forms.MultipleChoiceField(required=False, choices=params['miri']['pupil_list'], widget=forms.CheckboxSelectMultiple)
+    nirspec_pupil = forms.MultipleChoiceField(required=False, choices=params['nirspec']['pupil_list'], widget=forms.CheckboxSelectMultiple)
+    niriss_pupil = forms.MultipleChoiceField(required=False, choices=params['niriss']['pupil_list'], widget=forms.CheckboxSelectMultiple)
+    nircam_pupil = forms.MultipleChoiceField(required=False, choices=params['nircam']['pupil_list'], widget=forms.CheckboxSelectMultiple)
+    fgs_pupil = forms.MultipleChoiceField(required=False, choices=params['fgs']['pupil_list'], widget=forms.CheckboxSelectMultiple)
+
     def clean_inst(self):
 
         inst = self.cleaned_data['instrument']
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 6ac214bfa..827722371 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -42,7 +42,6 @@
     placed in the ``jwql`` directory.
 """
 
-from copy import deepcopy
 import csv
 import datetime
 import json
@@ -60,7 +59,7 @@
 from jwql.database.database_interface import load_connection
 from jwql.utils import monitor_utils
 from jwql.utils.interactive_preview_image import InteractivePreviewImg
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, URL_DICT, THUMBNAIL_FILTER_LOOK, QUERY_CONFIG_TEMPLATE, QUERY_CONFIG_KEYS
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, URL_DICT, QUERY_CONFIG_TEMPLATE, QUERY_CONFIG_KEYS
 from jwql.utils.utils import filename_parser, get_base_url, get_config, get_rootnames_for_instrument_proposal, query_unformat
 
 from .data_containers import build_table
@@ -82,7 +81,7 @@
 from .forms import JwqlQueryForm
 from .forms import FileSearchForm
 if not os.environ.get("READTHEDOCS"):
-    from .models import Observation, Proposal, RootFileInfo
+    from .models import Observation, RootFileInfo
 from astropy.io import fits
 from astropy.time import Time
 import astropy.units as u
@@ -104,9 +103,11 @@ def jwql_query(request):
                 query_configs[instrument]['exptypes'] = [query_unformat(i) for i in form.cleaned_data['{}_exptype'.format(instrument)]]
                 query_configs[instrument]['readpatts'] = [query_unformat(i) for i in form.cleaned_data['{}_readpatt'.format(instrument)]]
                 query_configs[instrument]['gratings'] = [query_unformat(i) for i in form.cleaned_data['{}_grating'.format(instrument)]]
+                query_configs[instrument]['subarrays'] = [query_unformat(i) for i in form.cleaned_data['{}_subarray'.format(instrument)]]
+                query_configs[instrument]['pupils'] = [query_unformat(i) for i in form.cleaned_data['{}_pupil'.format(instrument)]]
                 query_configs[instrument]['anomalies'] = [query_unformat(i) for i in form.cleaned_data['{}_anomalies'.format(instrument)]]
 
-            all_filters, all_apers, all_detectors, all_exptypes, all_readpatts, all_gratings, all_anomalies = {}, {}, {}, {}, {}, {}, {}
+            all_filters, all_apers, all_detectors, all_exptypes, all_readpatts, all_gratings, all_subarrays, all_pupils, all_anomalies = {}, {}, {}, {}, {}, {}, {}, {}, {}
             for instrument in query_configs:
                 all_filters[instrument] = query_configs[instrument]['filters']
                 all_apers[instrument] = query_configs[instrument]['apertures']
@@ -114,6 +115,8 @@ def jwql_query(request):
                 all_exptypes[instrument] = query_configs[instrument]['exptypes']
                 all_readpatts[instrument] = query_configs[instrument]['readpatts']
                 all_gratings[instrument] = query_configs[instrument]['gratings']
+                all_subarrays[instrument] = query_configs[instrument]['subarrays']
+                all_pupils[instrument] = query_configs[instrument]['pupils']
                 all_anomalies[instrument] = query_configs[instrument]['anomalies']
 
             parameters = QUERY_CONFIG_TEMPLATE.copy()
@@ -125,6 +128,8 @@ def jwql_query(request):
             parameters[QUERY_CONFIG_KEYS.EXP_TYPES] = all_exptypes
             parameters[QUERY_CONFIG_KEYS.READ_PATTS] = all_readpatts
             parameters[QUERY_CONFIG_KEYS.GRATINGS] = all_gratings
+            parameters[QUERY_CONFIG_KEYS.SUBARRAYS] = all_subarrays
+            parameters[QUERY_CONFIG_KEYS.PUPILS] = all_pupils
             parameters[QUERY_CONFIG_KEYS.EXP_TIME_MIN] = str(form.cleaned_data['exp_time_min'])
             parameters[QUERY_CONFIG_KEYS.EXP_TIME_MAX] = str(form.cleaned_data['exp_time_max'])
 

From cac615ebbe4b8858eb913c41fb1219e7c4e04d92 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 31 Mar 2023 16:34:24 -0400
Subject: [PATCH 024/256] update query page with new constants

---
 .../apps/jwql/templates/jwql_query.html       | 244 +++++++++++-------
 1 file changed, 152 insertions(+), 92 deletions(-)

diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index 785d4337a..e58117cf2 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -8,7 +8,7 @@
                                     margin-left: 80px; }
         span#helpBlock.help-block {font-size: LARGER;}
     </style>
-    
+
     <title>Dynamic Query Form - JWQL</title>
 
 
@@ -62,7 +62,7 @@ <h1>Dynamic Query Form</h1>
             Note, however, that it may increase the run time if you search a wider array of images.
         </p>
     </main>
-        
+
     <div class='form-group'>
         <form method="POST" action="." name="jwstqueryform">
             {{ csrf_input }}
@@ -70,7 +70,7 @@ <h1>Dynamic Query Form</h1>
                 <hr class="col-md-12">
                 <div class='instrument-form-group' id='instrument_form_group'>
                     <!-- Initial INSTRUMENT Checkbox -->
-                    <div class="col-sm-2">
+                    <div class="col-md-2">
                         <span id="helpBlock" class="help-block">Instruments</span>
                         <div class="form-control" name='instrument' id='instrument'>
                             {% for option in form.instrument %}
@@ -80,24 +80,25 @@ <h1>Dynamic Query Form</h1>
                             {% endfor %}
                         </div>
                     </div>
-                    
+
                     <!-- FGS Fields -->
                     <div style="display: flex" id="FGS">
-                        <section class="col-sm">
-                            <span id="helpBlock" class="help-block">FGS Apertures</span>
-                            <div class="form-control" class="row-1" id='fgs_aper'>
-                                {% for field in form.fgs_aper %}
-                                <div class="row-1">
+                        <section class="col-md">
+                            <span id="helpBlock" class="help-block">FGS Anomalies</span>
+                            <div class="form-control" id='fgs_anomalies'>
+                                {% for field in form.fgs_anomalies %}
+                                <div class="row-sm-10">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">FGS Detectors</span>
-                            <div class="form-control" id='fgs_detector'>
-                                {% for field in form.fgs_detector %}
+                            <span id="helpBlock" class="help-block">FGS Apertures</span>
+                            <div class="form-control" id='fgs_aper'>
+                                {% for field in form.fgs_aper %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -105,10 +106,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">FGS Read Patterns</span>
-                            <div class="form-control" id='fgs_readpatt'>
-                                {% for field in form.fgs_readpatt %}
+                            <span id="helpBlock" class="help-block">FGS Detectors</span>
+                            <div class="form-control" id='fgs_detector'>
+                                {% for field in form.fgs_detector %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -116,6 +118,7 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">FGS Exposure Types</span>
                             <div class="form-control" id='fgs_exptype'>
@@ -127,10 +130,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">FGS Anomalies</span>
-                            <div class="form-control" id='fgs_anomalies'>
-                                {% for field in form.fgs_anomalies %}
+                            <span id="helpBlock" class="help-block">FGS Read Patterns</span>
+                            <div class="form-control" id='fgs_readpatt'>
+                                {% for field in form.fgs_readpatt %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -142,20 +146,34 @@ <h1>Dynamic Query Form</h1>
 
                     <!-- MIRI Fields -->
                     <div style="display: flex" id="MIRI">
-                        <section class="col-sm">
-                            <span id="helpBlock" class="help-block">MIRI Subarrays</span>
-                            <div class="form-control" id='miri_aper'>
-                                {% for field in form.miri_aper %}
+                        <section class="col-md">
+                            <span id="helpBlock" class="help-block">MIRI Anomalies</span>
+                            <div class="form-control" id='miri_anomalies'>
+                                {% for field in form.miri_anomalies %}
+                                <div class="row-sm-10">
+                                    {{ field }}
+                                </div>
+                                {% endfor %}
+                                <div class="input-group-addon" style='width:60px'></div>
+                            </div>
+                        </section>
+
+                        <section class="col-md">
+                            <span id="helpBlock" class="help-block">MIRI Detectors</span>
+                            <div class="form-control" id='miri_detector'>
+                                {% for field in form.miri_detector %}
                                 <div class="row-2">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
+                                <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Filters</span>
-                            <div class="form-control" id='miri_filt'>
-                                {% for field in form.miri_filt %}
+                            <span id="helpBlock" class="help-block">MIRI Exposure Types</span>
+                            <div class="form-control" id='miri_exptype'>
+                                {% for field in form.miri_exptype %}
                                 <div class="row-2">
                                     {{ field }}
                                 </div>
@@ -163,10 +181,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Detectors</span>
-                            <div class="form-control" id='miri_detector'>
-                                {% for field in form.miri_detector %}
+                            <span id="helpBlock" class="help-block">MIRI Filters</span>
+                            <div class="form-control" id='miri_filt'>
+                                {% for field in form.miri_filt %}
                                 <div class="row-2">
                                     {{ field }}
                                 </div>
@@ -174,6 +193,7 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">MIRI Read Patterns</span>
                             <div class="form-control" id='miri_readpatt'>
@@ -185,21 +205,25 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Exposure Types</span>
-                            <div class="form-control" id='miri_exptype'>
-                                {% for field in form.miri_exptype %}
+                            <span id="helpBlock" class="help-block">MIRI Subarrays</span>
+                            <div class="form-control" id='miri_subarray'>
+                                {% for field in form.miri_subarray %}
                                 <div class="row-2">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
-                                <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+                    </div>
+
+                    <!-- NIRCAM Fields -->
+                    <div style="display: flex" id="NIRCAM">
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Anomalies</span>
-                            <div class="form-control" id='miri_anomalies'>
-                                {% for field in form.miri_anomalies %}
+                            <span id="helpBlock" class="help-block">NIRCam Anomalies</span>
+                            <div class="form-control" id='nircam_anomalies'>
+                                {% for field in form.nircam_anomalies %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -207,36 +231,47 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
-                    </div>
 
-                    <!-- NIRCAM Fields -->
-                    <div style="display: flex" id="NIRCAM">
-                        <section class="col-sm">
-                            <span id="helpBlock" class="help-block">NIRCam Apertures</span>
-                            <div class="form-control" class="row-3" id='nircam_aper'>
-                                {% for field in form.nircam_aper %}
-                                <div class="row-1">
+                        <section class="col-md">
+                            <span id="helpBlock" class="help-block">NIRCam Detectors</span>
+                            <div class="form-control" id='nircam_detector'>
+                                {% for field in form.nircam_detector %}
+                                <div class="row-4">
+                                    {{ field }}
+                                </div>
+                                {% endfor %}
+                                <div class="input-group-addon" style='width:60px'></div>
+                            </div>
+                        </section>
+
+                        <section class="col-md">
+                            <span id="helpBlock" class="help-block">NIRCam Exposure Types</span>
+                            <div class="form-control" id='nircam_exptype'>
+                                {% for field in form.nircam_exptype %}
+                                <div class="row-4">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRCam Filters</span>
                             <div class="form-control" id='nircam_filt'>
                                 {% for field in form.nircam_filt %}
-                                <div class="row-1">
+                                <div class="row-sm-10">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Detectors</span>
-                            <div class="form-control" id='nircam_detector'>
-                                {% for field in form.nircam_detector %}
+                            <span id="helpBlock" class="help-block">NIRCam Pupils</span>
+                            <div class="form-control" id='nircam_pupil'>
+                                {% for field in form.nircam_pupil %}
                                 <div class="row-4">
                                     {{ field }}
                                 </div>
@@ -244,6 +279,7 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRCam Read Patterns</span>
                             <div class="form-control" id='nircam_readpatt'>
@@ -255,10 +291,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Exposure Types</span>
-                            <div class="form-control" id='nircam_exptype'>
-                                {% for field in form.nircam_exptype %}
+                            <span id="helpBlock" class="help-block">NIRCam Subarrays</span>
+                            <div class="form-control" id='nircam_subarray'>
+                                {% for field in form.nircam_subarray %}
                                 <div class="row-4">
                                     {{ field }}
                                 </div>
@@ -266,10 +303,14 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+                    </div>
+
+                    <!-- NIRISS Fields -->
+                    <div style="display: flex" id="NIRISS">
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Anomalies</span>
-                            <div class="form-control" id='nircam_anomalies'>
-                                {% for field in form.nircam_anomalies %}
+                            <span id="helpBlock" class="help-block">NIRISS Anomalies</span>
+                            <div class="form-control" id='niriss_anomalies'>
+                                {% for field in form.niriss_anomalies %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -277,36 +318,47 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
-                    </div>
 
-                    <!-- NIRISS Fields -->
-                    <div style="display: flex" id="NIRISS">
-                        <section class="col-sm">
-                            <span id="helpBlock" class="help-block">NIRISS Apertures</span>
-                            <div class="form-control" class="row-4" id='niriss_aper'>
-                                {% for field in form.niriss_aper %}
-                                <div class="row-1">
+                        <section class="col-md">
+                            <span id="helpBlock" class="help-block">NIRISS Detectors</span>
+                            <div class="form-control" id='niriss_detector'>
+                                {% for field in form.niriss_detector %}
+                                <div class="row-sm-10">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
+                        <section class="col-md">
+                            <span id="helpBlock" class="help-block">NIRISS Exposure Types</span>
+                            <div class="form-control" id='niriss_exptype'>
+                                {% for field in form.niriss_exptype %}
+                                <div class="row-sm-10">
+                                    {{ field }}
+                                </div>
+                                {% endfor %}
+                                <div class="input-group-addon" style='width:60px'></div>
+                            </div>
+                        </section>
+
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRISS Filters</span>
                             <div class="form-control" id='niriss_filt'>
                                 {% for field in form.niriss_filt %}
-                                <div class="row-1">
+                                <div class="row-sm-10">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Read Patterns</span>
-                            <div class="form-control" id='niriss_readpatt'>
-                                {% for field in form.niriss_readpatt %}
+                            <span id="helpBlock" class="help-block">NIRISS Pupils</span>
+                            <div class="form-control" id='niriss_pupil'>
+                                {% for field in form.niriss_pupil %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -314,10 +366,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Exposure Types</span>
-                            <div class="form-control" id='niriss_exptype'>
-                                {% for field in form.niriss_exptype %}
+                            <span id="helpBlock" class="help-block">NIRISS Read Patterns</span>
+                            <div class="form-control" id='niriss_readpatt'>
+                                {% for field in form.niriss_readpatt %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -325,10 +378,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Anomalies</span>
-                            <div class="form-control" id='niriss_anomalies'>
-                                {% for field in form.niriss_anomalies %}
+                            <span id="helpBlock" class="help-block">NIRISS Subarrays</span>
+                            <div class="form-control" id='niriss_subarray'>
+                                {% for field in form.niriss_subarray %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -340,28 +394,30 @@ <h1>Dynamic Query Form</h1>
 
                     <!-- NIRSPEC Fields -->
                     <div style="display: flex" id="NIRSPEC">
-                        <section class="col-sm">
-                            <span id="helpBlock" class="help-block">NIRSpec Apertures</span>
-                            <div class="form-control" id='nirspec_aper'>
-                                {% for field in form.nirspec_aper %}
-                                <div class="row-1">
+                        <section class="col-md">
+                            <span id="helpBlock" class="help-block">NIRSpec Anomalies</span>
+                            <div class="form-control" id='nirspec_anomalies'>
+                                {% for field in form.nirspec_anomalies %}
+                                <div class="row-sm-10">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Filters</span>
-                            <div class="form-control" id='nirspec_filt'>
-                                {% for field in form.nirspec_filt %}
-                                <div class="row-1">
+                            <span id="helpBlock" class="help-block">NIRSpec Apertures</span>
+                            <div class="form-control" id='nirspec_aper'>
+                                {% for field in form.nirspec_aper %}
+                                <div class="row-sm-10">
                                     {{ field }}
                                 </div>
                                 {% endfor %}
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRSpec Detectors</span>
                             <div class="form-control" id='nirspec_detector'>
@@ -373,10 +429,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Gratings</span>
-                            <div class="form-control" id='nirspec_grating'>
-                                {% for field in form.nirspec_grating %}
+                            <span id="helpBlock" class="help-block">NIRSpec Exposure Types</span>
+                            <div class="form-control" id='nirspec_exptype'>
+                                {% for field in form.nirspec_exptype %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -384,10 +441,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Read Patterns</span>
-                            <div class="form-control" id='nirspec_readpatt'>
-                                {% for field in form.nirspec_readpatt %}
+                            <span id="helpBlock" class="help-block">NIRSpec Filters</span>
+                            <div class="form-control" id='nirspec_filt'>
+                                {% for field in form.nirspec_filt %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -395,10 +453,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Exposure Types</span>
-                            <div class="form-control" id='nirspec_exptype'>
-                                {% for field in form.nirspec_exptype %}
+                            <span id="helpBlock" class="help-block">NIRSpec Gratings</span>
+                            <div class="form-control" id='nirspec_grating'>
+                                {% for field in form.nirspec_grating %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>
@@ -406,10 +465,11 @@ <h1>Dynamic Query Form</h1>
                                 <div class="input-group-addon" style='width:60px'></div>
                             </div>
                         </section>
+
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Anomalies</span>
-                            <div class="form-control" id='nirspec_anomalies'>
-                                {% for field in form.nirspec_anomalies %}
+                            <span id="helpBlock" class="help-block">NIRSpec Read Patterns</span>
+                            <div class="form-control" id='nirspec_readpatt'>
+                                {% for field in form.nirspec_readpatt %}
                                 <div class="row-sm-10">
                                     {{ field }}
                                 </div>

From 1a64a99252c03c3431e14714f3032ea09087cc95 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 4 Apr 2023 11:16:53 -0400
Subject: [PATCH 025/256] Fix hover over date values

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 932b71cbd..d4f4ba047 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -233,9 +233,9 @@ def dashboard_disk_usage(self):
 
             hover_tool = HoverTool(tooltips=[('Available:', '@available'),
                                              ('Used:', '@used'),
-                                             ('Date:', '@time{%d %b %Y}')
+                                             ('Date:', '@date{%d %b %Y}')
                                              ])
-            hover_tool.formatters = {'@time': 'datetime'}
+            hover_tool.formatters = {'@date': 'datetime'}
             plots[data['shortname']].tools.append(hover_tool)
             tabs.append(Panel(child=plots[data['shortname']], title=f"{data['shortname']} Storage"))
 
@@ -336,9 +336,9 @@ def dashboard_central_store_data_volume(self):
             cen_store_plot.legend.location = 'top_left'
 
             hover_tool = HoverTool(tooltips=[('Used:', f'@used TB'),
-                                             ('Date:', '@time{%d %b %Y}')
+                                             ('Date:', '@date{%d %b %Y}')
                                              ])
-            hover_tool.formatters = {'@time': 'datetime'}
+            hover_tool.formatters = {'@date': 'datetime'}
             cen_store_plot.tools.append(hover_tool)
 
         di.session.close()

From 8465020c2df0d0e94cdd0a189ca071ca243b563a Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 4 Apr 2023 13:14:32 -0400
Subject: [PATCH 026/256] separate hover tools

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index d4f4ba047..f97d7c9ba 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -203,6 +203,7 @@ def dashboard_disk_usage(self):
 
         # Plot total data volume and available disk space versus time
         plots = {}
+        hover_tool = {}
         tabs = []
         for data in [preview_results, log_results]:
 
@@ -224,19 +225,19 @@ def dashboard_disk_usage(self):
             plots[data['shortname']].circle(x='date', y='used', source=source,color='#355C7D', size=10)
 
             plots[data['shortname']].xaxis.formatter = DatetimeTickFormatter(hours=["%H:%M %d %B %Y"],
-                                                       days=["%d %B %Y"],
-                                                       months=["%d %B %Y"],
-                                                       years=["%B %Y"],
-                                                       )
+                                                                             days=["%d %B %Y"],
+                                                                             months=["%d %B %Y"],
+                                                                             years=["%B %Y"],
+                                                                             )
             plots[data['shortname']].xaxis.major_label_orientation = pi / 4
             plots[data['shortname']].legend.location = 'top_left'
 
-            hover_tool = HoverTool(tooltips=[('Available:', '@available'),
-                                             ('Used:', '@used'),
-                                             ('Date:', '@date{%d %b %Y}')
-                                             ])
-            hover_tool.formatters = {'@date': 'datetime'}
-            plots[data['shortname']].tools.append(hover_tool)
+            hover_tool[data['shortname']] = HoverTool(tooltips=[('Available:', '@available'),
+                                                                ('Used:', '@used'),
+                                                                ('Date:', '@date{%d %b %Y}')
+                                                                ])
+            hover_tool[data['shortname']].formatters = {'@date': 'datetime'}
+            plots[data['shortname']].tools.append(hover_tool[data['shortname']])
             tabs.append(Panel(child=plots[data['shortname']], title=f"{data['shortname']} Storage"))
 
         tabs = Tabs(tabs=tabs)

From a9f3b9ba07dd7a623c0451ee47b29370f5fb043a Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 4 Apr 2023 13:29:59 -0400
Subject: [PATCH 027/256] Remove second hover tool

---
 jwql/website/apps/jwql/bokeh_dashboard.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index f97d7c9ba..d9dec4a12 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -212,7 +212,7 @@ def dashboard_disk_usage(self):
             source = ColumnDataSource(data['results'])
 
             # Initialize plot
-            plots[data['shortname']] = figure(tools='pan,hover,box_zoom,wheel_zoom,reset,save',
+            plots[data['shortname']] = figure(tools='pan,box_zoom,wheel_zoom,reset,save',
                                               plot_width=800,
                                               x_axis_type='datetime',
                                               title=f"Available & Used Storage on {data['shortname']}",

From a61f52726e3872e05025ff159324d581429cb1c5 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Tue, 4 Apr 2023 16:40:06 -0400
Subject: [PATCH 028/256] Remove yaml and bokeh code

---
 .../monitor_pages/monitor_readnoise_bokeh.py  | 160 ---------------
 .../yaml/monitor_readnoise_interface.yaml     | 189 ------------------
 2 files changed, 349 deletions(-)
 delete mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
 delete mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
deleted file mode 100644
index 8c8be095e..000000000
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ /dev/null
@@ -1,160 +0,0 @@
-"""This module contains code for the readnoise monitor Bokeh plots.
-
-Author
-------
-
-    - Ben Sunnquist
-
-Use
----
-
-    This module can be used from the command line as such:
-
-    ::
-
-        from jwql.website.apps.jwql import monitor_pages
-        monitor_template = monitor_pages.ReadnoiseMonitor()
-        monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL')
-"""
-
-from datetime import datetime, timedelta
-import os
-
-import numpy as np
-
-from jwql.bokeh_templating import BokehTemplate
-from jwql.database.database_interface import session
-from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
-class ReadnoiseMonitor(BokehTemplate):
-
-    # Combine the input parameters into a single property because we
-    # do not want to invoke the setter unless all are updated
-    @property
-    def input_parameters(self):
-        return (self._instrument, self._aperture)
-
-    @input_parameters.setter
-    def input_parameters(self, info):
-        self._instrument, self._aperture = info
-        self.pre_init()
-        self.post_init()
-
-    def identify_tables(self):
-        """Determine which database tables to use for the given instrument"""
-
-        mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()]
-        self.stats_table = eval('{}ReadnoiseStats'.format(mixed_case_name))
-
-    def load_data(self):
-        """Query the database tables to get all of the relevant readnoise data"""
-
-        # Determine which database tables are needed based on instrument
-        self.identify_tables()
-
-        # Query database for all data in readnoise stats with a matching aperture,
-        # and sort the data by exposure start time.
-        self.query_results = session.query(self.stats_table) \
-            .filter(self.stats_table.aperture == self._aperture) \
-            .order_by(self.stats_table.expstart) \
-            .all()
-
-        session.close()
-
-    def pre_init(self):
-
-        # Start with default values for instrument and aperture because
-        # BokehTemplate's __init__ method does not allow input arguments
-        try:
-            dummy_instrument = self._instrument
-            dummy_aperture = self._aperture
-        except AttributeError:
-            self._instrument = 'NIRCam'
-            self._aperture = ''
-
-        self._embed = True
-        self.format_string = None
-        self.interface_file = os.path.join(SCRIPT_DIR, 'yaml', 'monitor_readnoise_interface.yaml')
-
-    def post_init(self):
-
-        # Load the readnoise data
-        self.load_data()
-
-        # Update the mean readnoise figures
-        self.update_mean_readnoise_figures()
-
-        # Update the readnoise difference image and histogram
-        self.update_readnoise_diff_plots()
-
-    def update_mean_readnoise_figures(self):
-        """Updates the mean readnoise bokeh plots"""
-
-        # Get the dark exposures info
-        filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.query_results]
-        expstarts_iso = np.array([result.expstart for result in self.query_results])
-        expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso])
-        nints = [result.nints for result in self.query_results]
-        ngroups = [result.ngroups for result in self.query_results]
-
-        # Update the mean readnoise figures for all amps
-        for amp in ['1', '2', '3', '4']:
-            readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.query_results])
-            self.refs['mean_readnoise_source_amp{}'.format(amp)].data = {'time': expstarts,
-                                                                         'time_iso': expstarts_iso,
-                                                                         'mean_rn': readnoise_vals,
-                                                                         'filename': filenames,
-                                                                         'nints': nints,
-                                                                         'ngroups': ngroups}
-            self.refs['mean_readnoise_figure_amp{}'.format(amp)].title.text = 'Amp {}'.format(amp)
-            self.refs['mean_readnoise_figure_amp{}'.format(amp)].hover.tooltips = [('file', '@filename'),
-                                                                                   ('time', '@time_iso'),
-                                                                                   ('nints', '@nints'),
-                                                                                   ('ngroups', '@ngroups'),
-                                                                                   ('readnoise', '@mean_rn')]
-
-            # Update plot limits if data exists
-            if len(readnoise_vals) != 0:
-                self.refs['mean_readnoise_xr_amp{}'.format(amp)].start = expstarts.min() - timedelta(days=3)
-                self.refs['mean_readnoise_xr_amp{}'.format(amp)].end = expstarts.max() + timedelta(days=3)
-                min_val, max_val = min(x for x in readnoise_vals if x is not None), max(x for x in readnoise_vals if x is not None)
-                if min_val == max_val:
-                    self.refs['mean_readnoise_yr_amp{}'.format(amp)].start = min_val - 1
-                    self.refs['mean_readnoise_yr_amp{}'.format(amp)].end = max_val + 1
-                else:
-                    offset = (max_val - min_val) * .1
-                    self.refs['mean_readnoise_yr_amp{}'.format(amp)].start = min_val - offset
-                    self.refs['mean_readnoise_yr_amp{}'.format(amp)].end = max_val + offset
-
-    def update_readnoise_diff_plots(self):
-        """Updates the readnoise difference image and histogram"""
-
-        # Update the readnoise difference image and histogram, if data exists
-        if len(self.query_results) != 0:
-            # Get the most recent data; the entries were sorted by time when
-            # loading the database, so the last entry will always be the most recent.
-            diff_image_png = self.query_results[-1].readnoise_diff_image
-            diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:]))
-            diff_image_n = np.array(self.query_results[-1].diff_image_n)
-            diff_image_bin_centers = np.array(self.query_results[-1].diff_image_bin_centers)
-
-            # Update the readnoise difference image and histogram
-            self.refs['readnoise_diff_image'].image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left")
-            self.refs['diff_hist_source'].data = {'n': diff_image_n,
-                                                  'bin_centers': diff_image_bin_centers}
-            self.refs['diff_hist_xr'].start = diff_image_bin_centers.min()
-            self.refs['diff_hist_xr'].end = diff_image_bin_centers.max()
-            self.refs['diff_hist_yr'].start = diff_image_n.min()
-            self.refs['diff_hist_yr'].end = diff_image_n.max() + diff_image_n.max() * 0.05
-
-        # Update the readnoise difference image style
-        self.refs['readnoise_diff_image'].xaxis.visible = False
-        self.refs['readnoise_diff_image'].yaxis.visible = False
-        self.refs['readnoise_diff_image'].xgrid.grid_line_color = None
-        self.refs['readnoise_diff_image'].ygrid.grid_line_color = None
-        self.refs['readnoise_diff_image'].title.text_font_size = '22px'
-        self.refs['readnoise_diff_image'].title.align = 'center'
diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml
deleted file mode 100644
index 4d2fd8a78..000000000
--- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_readnoise_interface.yaml
+++ /dev/null
@@ -1,189 +0,0 @@
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Mean Readnoise vs Time Figures Amp1
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- !ColumnDataSource: &mean_readnoise_source_amp1
-    ref: "mean_readnoise_source_amp1"
-    data:
-        time: []
-        time_iso: []
-        mean_rn: []
-        filename: []
-        nints: []
-        ngroups: []
-- !Range1d: &mean_readnoise_xr_amp1
-    ref: "mean_readnoise_xr_amp1"
-    start: 0
-    end: 1
-    bounds: 'auto'
-- !Range1d: &mean_readnoise_yr_amp1
-    ref: "mean_readnoise_yr_amp1"
-    start: 0
-    end: 10
-    bounds: 'auto'
-- !Figure: &mean_readnoise_figure_amp1
-    ref: "mean_readnoise_figure_amp1"
-    title: "Amp 1"
-    x_axis_label: "Date"
-    x_axis_type: "datetime"
-    y_axis_label: "Mean readnoise [DN]"
-    x_range: *mean_readnoise_xr_amp1
-    y_range: *mean_readnoise_yr_amp1
-    height: 800
-    width: 800
-    tools: "hover, wheel_zoom, pan, reset"
-    elements:
-        - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp1}
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Mean Readnoise vs Time Figures Amp2
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- !ColumnDataSource: &mean_readnoise_source_amp2
-    ref: "mean_readnoise_source_amp2"
-    data:
-        time: []
-        time_iso: []
-        mean_rn: []
-        filename: []
-        nints: []
-        ngroups: []
-- !Range1d: &mean_readnoise_xr_amp2
-    ref: "mean_readnoise_xr_amp2"
-    start: 0
-    end: 1
-    bounds: 'auto'
-- !Range1d: &mean_readnoise_yr_amp2
-    ref: "mean_readnoise_yr_amp2"
-    start: 0
-    end: 10
-    bounds: 'auto'
-- !Figure: &mean_readnoise_figure_amp2
-    ref: "mean_readnoise_figure_amp2"
-    title: "Amp 2"
-    x_axis_label: "Date"
-    x_axis_type: "datetime"
-    y_axis_label: "Mean readnoise [DN]"
-    x_range: *mean_readnoise_xr_amp2
-    y_range: *mean_readnoise_yr_amp2
-    height: 800
-    width: 800
-    tools: "hover, wheel_zoom, pan, reset"
-    elements:
-        - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp2}
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Mean Readnoise vs Time Figures Amp3
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- !ColumnDataSource: &mean_readnoise_source_amp3
-    ref: "mean_readnoise_source_amp3"
-    data:
-        time: []
-        time_iso: []
-        mean_rn: []
-        filename: []
-        nints: []
-        ngroups: []
-- !Range1d: &mean_readnoise_xr_amp3
-    ref: "mean_readnoise_xr_amp3"
-    start: 0
-    end: 1
-    bounds: 'auto'
-- !Range1d: &mean_readnoise_yr_amp3
-    ref: "mean_readnoise_yr_amp3"
-    start: 0
-    end: 10
-    bounds: 'auto'
-- !Figure: &mean_readnoise_figure_amp3
-    ref: "mean_readnoise_figure_amp3"
-    title: "Amp 3"
-    x_axis_label: "Date"
-    x_axis_type: "datetime"
-    y_axis_label: "Mean readnoise [DN]"
-    x_range: *mean_readnoise_xr_amp3
-    y_range: *mean_readnoise_yr_amp3
-    height: 800
-    width: 800
-    tools: "hover, wheel_zoom, pan, reset"
-    elements:
-        - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp3}
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Mean Readnoise vs Time Figures Amp4
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- !ColumnDataSource: &mean_readnoise_source_amp4
-    ref: "mean_readnoise_source_amp4"
-    data:
-        time: []
-        time_iso: []
-        mean_rn: []
-        filename: []
-        nints: []
-        ngroups: []
-- !Range1d: &mean_readnoise_xr_amp4
-    ref: "mean_readnoise_xr_amp4"
-    start: 0
-    end: 1
-    bounds: 'auto'
-- !Range1d: &mean_readnoise_yr_amp4
-    ref: "mean_readnoise_yr_amp4"
-    start: 0
-    end: 10
-    bounds: 'auto'
-- !Figure: &mean_readnoise_figure_amp4
-    ref: "mean_readnoise_figure_amp4"
-    title: "Amp 4"
-    x_axis_label: "Date"
-    x_axis_type: "datetime"
-    y_axis_label: "Mean readnoise [DN]"
-    x_range: *mean_readnoise_xr_amp4
-    y_range: *mean_readnoise_yr_amp4
-    height: 800
-    width: 800
-    tools: "hover, wheel_zoom, pan, reset"
-    elements:
-        - {'kind': 'circle', 'x': 'time', 'y': 'mean_rn', 'size': 6, 'source': *mean_readnoise_source_amp4}
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Readnoise Difference Image
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- !ColumnDataSource: &diff_source
-    ref: "diff_source"
-    data:
-        dh: [1]
-        dw: [1]
-        image: [[[0,0], [0, 0]]]
-- !Figure: &readnoise_diff_image
-    ref: "readnoise_diff_image"
-    title: 'Readnoise Difference (most recent dark - pipeline reffile)'
-    elements:
-        - {"kind": "image", "image": "image", "x": 0, "y": 0, "dh": 'dh', "dw": 'dh', "source": *diff_source}
-    tools: ""
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-# Readnoise Difference Histogram
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-- !ColumnDataSource: &diff_hist_source
-    ref: "diff_hist_source"
-    data:
-        n: []
-        bin_centers: []
-- !Range1d: &diff_hist_xr
-    ref: "diff_hist_xr"
-    start: 0
-    end: 1
-    bounds: 'auto'
-- !Range1d: &diff_hist_yr
-    ref: "diff_hist_yr"
-    start: 0
-    end: 10
-    bounds: 'auto'
-- !Figure: &readnoise_diff_hist
-    ref: "readnoise_diff_hist"
-    x_axis_label: "Readnoise Difference [DN]"
-    y_axis_label: "Number of Pixels"
-    x_range: *diff_hist_xr
-    y_range: *diff_hist_yr
-    height: 250
-    width: 300
-    tools: "hover, wheel_zoom, pan, reset"
-    elements:
-        - {'kind': 'circle', 'x': 'bin_centers', 'y': 'n', 'size': 4, 'source': *diff_hist_source}
-
-# Document structure
-# - !Document:
-#     - !row:
-#         - *mean_readnoise_figure
\ No newline at end of file

From 9320346cc7c0b4c7c9d9e0ef1d5ee6bcb2e6dd37 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Thu, 6 Apr 2023 16:22:29 -0400
Subject: [PATCH 029/256] Adding bones to read noise monitor plotting

---
 .../monitor_pages/monitor_readnoise_bokeh.py  | 98 +++++++++++++++++++
 1 file changed, 98 insertions(+)
 create mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
new file mode 100644
index 000000000..3a5d22a6a
--- /dev/null
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -0,0 +1,98 @@
+"""This module contains code for the readnoise monitor Bokeh plots.
+
+Author
+------
+
+    - Ben Sunnquist
+
+Use
+---
+
+    This module can be used from the command line as such:
+
+    ::
+
+        from jwql.website.apps.jwql import monitor_pages
+        monitor_template = monitor_pages.ReadnoiseMonitor()
+        monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL')
+"""
+
+from datetime import datetime, timedelta
+import os
+
+import numpy as np
+
+from jwql.bokeh_templating import BokehTemplate
+from jwql.database.database_interface import session
+from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
+
+
+class ReadnoiseMonitorData():
+    """Class to hold bias data to be plotted
+    
+    Parameters
+    ----------
+    
+    instrument : str
+        Instrument name (e.g. nircam)
+    
+    Attributes
+    ----------
+    
+    instrument : str
+        Instrument name (e.g. nircam)
+    aperture : str
+        Aperture name (e.g. apername)
+    
+    """
+
+    def __init__(self, instrument, aperture):
+        self.instrument = instrument
+        self.aperture = aperture
+        self.load_data()
+
+
+    def identify_tables(self):
+        """Determine which database tables to use for the given instrument"""
+
+        mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument.lower()]
+        self.stats_table = eval('{}ReadnoiseStats'.format(mixed_case_name))
+
+    def load_data(self):
+        """Query the database tables to get all of the relevant readnoise data"""
+
+        # Determine which database tables are needed based on instrument
+        self.identify_tables()
+
+        # Query database for all data in readnoise stats with a matching aperture,
+        # and sort the data by exposure start time.
+        self.query_results = session.query(self.stats_table) \
+            .filter(self.stats_table.aperture == self.aperture) \
+            .order_by(self.stats_table.expstart) \
+            .all()
+
+        session.close()
+
+class ReadNoisePlots():
+    """Class to make readnoise plots.
+    """
+    def __init__(self, instrument, aperture):
+        self.instrument = instrument
+        self.aperture = aperture
+
+        self.db = ReadnoiseMonitorData(self.instrument, self.aperture)
+
+
+    def read_noise_amp_plots(self):
+        """Make read noise monitor trending plots per amplifier
+        """
+        print('make plot here')
+        # for amp in ['1', '2', '3', '4']:
+        #     expstarts_iso = np.array([result.expstart for result in self.db])
+        #     readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.query_results])
+            
+
+
+
+

From 48c89602c7ca4f2927b58efff5deb50300351c91 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 6 Apr 2023 17:55:27 -0400
Subject: [PATCH 030/256] Update sqlalchemy updates to be compatible with
 sqlalchemy version 2

---
 jwql/jwql_monitors/monitor_filesystem.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 98e2f1837..b5313ed5c 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -654,7 +654,8 @@ def update_central_store_database(central_storage_dict):
         new_record['size'] = central_storage_dict[area]['size']
         new_record['used'] = central_storage_dict[area]['used']
         new_record['available'] = central_storage_dict[area]['available']
-        engine.execute(CentralStore.__table__.insert(), new_record)
+        with engine.begin() as connection:
+            connection.execute(CentralStore.__table__.insert(), new_record)
         session.commit()
     session.close()
 
@@ -680,7 +681,8 @@ def update_characteristics_database(char_info):
         new_record['instrument'] = instrument
         new_record['filter_pupil'] = filter_list
         new_record['obs_per_filter_pupil'] = value_list
-        engine.execute(FilesystemCharacteristics.__table__.insert(), new_record)
+        with engine.begin() as connection:
+            connection.execute(FilesystemCharacteristics.__table__.insert(), new_record)
         session.commit()
 
     session.close()
@@ -699,7 +701,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
     """
     logging.info('\tUpdating the database')
 
-    engine.execute(FilesystemGeneral.__table__.insert(), general_results_dict)
+    with engine.begin() as connection:
+        connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict)
     session.commit()
 
     # Add data to filesystem_instrument table
@@ -715,7 +718,8 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
             # Protect against updated enum options that have not been propagated to
             # the table definition
             try:
-                engine.execute(FilesystemInstrument.__table__.insert(), new_record)
+                with engine.begin() as connection:
+                    connection.execute(FilesystemInstrument.__table__.insert(), new_record)
                 session.commit()
             except DataError as e:
                 logging.error(e)

From f3c5f94e1f7a9ab4f8fcaaad40333d675e5c346f Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 6 Apr 2023 18:16:14 -0400
Subject: [PATCH 031/256] Remove session.close() statements

---
 jwql/jwql_monitors/monitor_filesystem.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index b5313ed5c..95a1e1dd6 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -656,7 +656,6 @@ def update_central_store_database(central_storage_dict):
         new_record['available'] = central_storage_dict[area]['available']
         with engine.begin() as connection:
             connection.execute(CentralStore.__table__.insert(), new_record)
-        session.commit()
     session.close()
 
 
@@ -682,8 +681,8 @@ def update_characteristics_database(char_info):
         new_record['filter_pupil'] = filter_list
         new_record['obs_per_filter_pupil'] = value_list
         with engine.begin() as connection:
-            connection.execute(FilesystemCharacteristics.__table__.insert(), new_record)
-        session.commit()
+            connection.execute(
+                FilesystemCharacteristics.__table__.insert(), new_record)
 
     session.close()
 
@@ -703,7 +702,6 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
 
     with engine.begin() as connection:
         connection.execute(FilesystemGeneral.__table__.insert(), general_results_dict)
-    session.commit()
 
     # Add data to filesystem_instrument table
     for instrument in JWST_INSTRUMENT_NAMES:
@@ -720,7 +718,6 @@ def update_database(general_results_dict, instrument_results_dict, central_stora
             try:
                 with engine.begin() as connection:
                     connection.execute(FilesystemInstrument.__table__.insert(), new_record)
-                session.commit()
             except DataError as e:
                 logging.error(e)
 

From f254bced4a569d4fe6e361624a9b17a0f8781bbf Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 7 Apr 2023 16:23:43 -0400
Subject: [PATCH 032/256] reformatting get_rootnames_from_query

---
 jwql/website/apps/jwql/data_containers.py | 160 ++++++++++------------
 jwql/website/apps/jwql/views.py           |  16 +--
 2 files changed, 79 insertions(+), 97 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 045794789..070cf91d7 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -447,7 +447,7 @@ def get_current_flagged_anomalies(rootfileinfo_set):
     Parameters
     ----------
     rootfileinfo_set : RootFileInfo Queryset
-        A query set of 1 or more RootFileInfos of interest 
+        A query set of 1 or more RootFileInfos of interest
         Must be iterable, even if only one RootFileInfo.
 
     Returns
@@ -1484,9 +1484,8 @@ def get_rootnames_for_proposal(proposal):
     return rootnames.compressed()
 
 
-def get_thumbnails_all_instruments(parameters):
-    """Return a list of thumbnails available in the filesystem for all
-    instruments given requested MAST parameters and queried anomalies.
+def get_rootnames_from_query(parameters):
+    """Return a query_set of RootFileInfo given requested filter parameters.
 
     Parameters
     ----------
@@ -1496,87 +1495,85 @@ def get_thumbnails_all_instruments(parameters):
 
     Returns
     -------
-    thumbnails : list
-        A list of thumbnails available in the filesystem for the
-        given instrument.
+    filtered_rootnames : query_set
+        A query_set of all rootfileinfos filtered from the given parameters
     """
-
-    anomalies = parameters[QUERY_CONFIG_KEYS.ANOMALIES]
-
-    thumbnails_subset = []
-
+    # TODO - This code setup is temporary until the merge to Postgres is complete.
+    # selected_rootfileinfos = RootFileInfo.objects.none()
+    filtered_rootnames = []
+    # Each Query Selection is Instrument specific
     for inst in parameters[QUERY_CONFIG_KEYS.INSTRUMENTS]:
-        # Make sure instruments are of the proper format (e.g. "Nircam")
-        instrument = inst[0].upper() + inst[1:].lower()
-
-        # Query MAST for all rootnames for the instrument
-        service = "Mast.Jwst.Filtered.{}".format(instrument)
-
-        if ((parameters[QUERY_CONFIG_KEYS.APERTURES][inst.lower()] == [])
-                and (parameters[QUERY_CONFIG_KEYS.DETECTORS][inst.lower()] == [])
-                and (parameters[QUERY_CONFIG_KEYS.FILTERS][inst.lower()] == [])
-                and (parameters[QUERY_CONFIG_KEYS.EXP_TYPES][inst.lower()] == [])
-                and (parameters[QUERY_CONFIG_KEYS.READ_PATTS][inst.lower()] == [])):  # noqa: W503
-            params = {"columns": "*", "filters": []}
+        # Make sure instruments are of the proper format for the archive query
+        inst = inst.lower()
+        current_ins_rootfileinfos = RootFileInfo.objects.filter(instrument=JWST_INSTRUMENT_NAMES_MIXEDCASE[inst])
+
+        inst_anomalies = parameters[QUERY_CONFIG_KEYS.ANOMALIES][inst]
+        inst_aperture = parameters[QUERY_CONFIG_KEYS.APERTURES][inst]
+        inst_detector = parameters[QUERY_CONFIG_KEYS.DETECTORS][inst]
+        inst_exp_type = parameters[QUERY_CONFIG_KEYS.EXP_TYPES][inst]
+        inst_filter = parameters[QUERY_CONFIG_KEYS.FILTERS][inst]
+        inst_grating = parameters[QUERY_CONFIG_KEYS.GRATINGS][inst]
+        inst_pupil = parameters[QUERY_CONFIG_KEYS.PUPILS][inst]
+        inst_read_patt = parameters[QUERY_CONFIG_KEYS.READ_PATTS][inst]
+        inst_subarray = parameters[QUERY_CONFIG_KEYS.SUBARRAYS][inst]
+
+        if (inst_aperture != []):
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(aperture__in=inst_aperture)
+        if (inst_detector != []):
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(detector__in=inst_detector)
+        if (inst_exp_type != []):
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(exp_type__in=inst_exp_type)
+        if (inst_filter != []):
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(filter__in=inst_filter)
+        if (inst_grating != []):
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(grating__in=inst_grating)
+        if (inst_pupil != []):
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(pupil__in=inst_pupil)
+        if (inst_read_patt != []):
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(read_patt__in=inst_read_patt)
+        if (inst_subarray != []):
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(subarray__in=inst_subarray)
+
+        # TODO - This uncommented CODE is what we will use while DJANGO is still using the SQLITE3 DB.
+        #             ONCE DB IS MIGRATED TO POSTGRES WE CAN REPLACE THIS CODE WITH THE UNTESTED CODE COMMENTED OUT BELOW
+        #      >>>  START CODE PRE DB MIGRATION HERE  <<<
+        if (inst_anomalies != []):
+            # If the rootfile info has any of the marked anomalies we want it
+            # Make union of marked anomalies from our current query set, then save intersection.
+            anomaly_rootnames = []
+
+            for anomaly in inst_anomalies:
+                anomaly_filter = "anomalies__" + str(anomaly).lower()
+                this_anomaly_rootnames = [name[0] for name in current_ins_rootfileinfos.filter(**{anomaly_filter: True}).values_list("root_name")]
+                anomaly_rootnames.extend(this_anomaly_rootnames)
+
+            filtered_rootnames.extend(anomaly_rootnames)
         else:
-            query_filters = []
-            if (parameters[QUERY_CONFIG_KEYS.APERTURES][inst.lower()] != []):
-                if instrument != "Nircam":
-                    query_filters.append({"paramName": "pps_aper", "values": parameters[QUERY_CONFIG_KEYS.APERTURES][inst.lower()]})
-                if instrument == "Nircam":
-                    query_filters.append({"paramName": "apername", "values": parameters[QUERY_CONFIG_KEYS.APERTURES][inst.lower()]})
-            if (parameters[QUERY_CONFIG_KEYS.DETECTORS][inst.lower()] != []):
-                query_filters.append({"paramName": "detector", "values": parameters[QUERY_CONFIG_KEYS.DETECTORS][inst.lower()]})
-            if (parameters[QUERY_CONFIG_KEYS.FILTERS][inst.lower()] != []):
-                query_filters.append({"paramName": "filter", "values": parameters[QUERY_CONFIG_KEYS.FILTERS][inst.lower()]})
-            if (parameters[QUERY_CONFIG_KEYS.EXP_TYPES][inst.lower()] != []):
-                query_filters.append({"paramName": "exp_type", "values": parameters[QUERY_CONFIG_KEYS.EXP_TYPES][inst.lower()]})
-            if (parameters[QUERY_CONFIG_KEYS.READ_PATTS][inst.lower()] != []):
-                query_filters.append({"paramName": "readpatt", "values": parameters[QUERY_CONFIG_KEYS.READ_PATTS][inst.lower()]})
-            params = {"columns": "*",
-                      "filters": query_filters}
-
-        response = Mast.service_request_async(service, params)
-        results = response[0].json()['data']
+            filtered_rootnames.extend(current_ins_rootfileinfos)
 
-        inst_filenames = [result['filename'].split('.')[0] for result in results]
-        inst_filenames = [filename for filename in inst_filenames if filename.split('_')[-1] not in IGNORED_SUFFIXES]
+    return list(set(filtered_rootnames))
+    #      >>>  END CODE PRE DB MIGRATION HERE  <<<
 
-        # Get list of all thumbnails
-        thumb_inventory = os.path.join(f"{THUMBNAIL_FILESYSTEM}", f"{THUMBNAIL_LISTFILE}_{inst.lower()}.txt")
-        thumbnail_inst_list = retrieve_filelist(thumb_inventory)
+    # TODO - BELOW IS THE OUTLINE OF CODE WE WANT TO USE, HOWEVER THIS CAN'T BE IMPLEMENTED WITH DJANGO RUNNING SQLITE
+    #             ONCE WE MIGRATE TO POSTGRES WE CAN IMPLEMENT THE BELOW FUNCTIONALITY WHICH SHOULD MAKE THIS CODE MOVE A LITTLE FASTER
+    #             NOTE: THIS CODE OUTLINE IS UNTESTED and is only a rough outline!
+    #      >>>  START CODE AFTER DB MIGRATION HERE  <<<
+    #     if (inst_anomalies != []):
+    #         # If the rootfile info has any of the marked anomalies we want it
+    #         # Make union of marked anomalies from our current query set, then save intersection.
+    #         anomaly_rootfileinfos = RootFileInfo.objects.none()
 
-        # Get subset of thumbnail images that match the filenames
-        thumbnails_inst_subset = [os.path.basename(item) for item in thumbnail_inst_list if
-                                  os.path.basename(item).split('_integ')[0] in inst_filenames]
+    #         for anomaly in inst_anomalies:
+    #             anomaly_filter = "anomalies__" + str(anomaly).lower()
+    #             anomaly_rootfileinfos.union(current_ins_rootfileinfos.filter(**{anomaly_filter: True}))
 
-        # Eliminate any duplicates
-        thumbnails_inst_subset = list(set(thumbnails_inst_subset))
-        thumbnails_subset.extend(thumbnails_inst_subset)
+    #         current_ins_rootfileinfos = current_ins_rootfileinfos.intersection(anomaly_rootfileinfos)
 
-    # Determine whether or not queried anomalies are flagged
-    final_subset = []
+    #     # Add this instrument's query set to our return queryset
+    #     selected_rootfileinfos = selected_rootfileinfos.union(current_ins_rootfileinfos)
 
-    if anomalies != {'miri': [], 'nirspec': [], 'niriss': [], 'nircam': [], 'fgs': []}:
-        for thumbnail in thumbnails_subset:
-            components = thumbnail.split('_')
-            file_root = ''.join((components[0], '_', components[1], '_', components[2], '_', components[3]))
-            filerootinfo_set = RootFileInfo.objects.filter(root_name__startswith=file_root)
-            try:
-                instrument = filename_parser(thumbnail)['instrument']
-                thumbnail_anomalies = get_current_flagged_anomalies(filerootinfo_set)
-                if thumbnail_anomalies:
-                    for anomaly in anomalies[instrument.lower()]:
-                        if anomaly.lower() in thumbnail_anomalies:
-                            # thumbnail contains an anomaly selected in the query
-                            final_subset.append(thumbnail)
-            except KeyError:
-                print("Error with thumbnail: ", thumbnail)
-    else:
-        # if no anomalies are flagged, return all thumbnails from query
-        final_subset = thumbnails_subset
-
-    return list(set(final_subset))
+    # return selected_rootfileinfos  # OR RETURN LIST OF ROOTNAMES
+    #      >>>  END CODE AFTER DB MIGRATION HERE  <<<
 
 
 def get_thumbnails_by_instrument(inst):
@@ -2117,16 +2114,13 @@ def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exc
     return data_dict
 
 
-def thumbnails_query_ajax(rootnames, expstarts=None):
+def thumbnails_query_ajax(rootnames):
     """Generate a page that provides data necessary to render the
     ``thumbnails`` template.
 
     Parameters
     ----------
     rootnames : list of strings
-        Rootname of APT proposal to filter
-    expstarts : list
-        Exposure start times from MAST (mjd)
 
     Returns
     -------
@@ -2161,11 +2155,7 @@ def thumbnails_query_ajax(rootnames, expstarts=None):
 
         # Add data to dictionary
         data_dict['file_data'][rootname] = {}
-        try:
-            data_dict['file_data'][rootname]['inst'] = JWST_INSTRUMENT_NAMES_MIXEDCASE[filename_parser(rootname)['instrument']]
-        except KeyError:
-            data_dict['file_data'][rootname]['inst'] = "MIRI"
-            print("Warning: assuming instrument is MIRI")
+        data_dict['file_data'][rootname]['inst'] = JWST_INSTRUMENT_NAMES_MIXEDCASE[filename_parser(rootname)['instrument']]
         data_dict['file_data'][rootname]['filename_dict'] = filename_dict
         data_dict['file_data'][rootname]['available_files'] = available_files
         data_dict['file_data'][rootname]['expstart'] = get_expstart(data_dict['file_data'][rootname]['inst'], rootname)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 827722371..cfadf1ee6 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -72,7 +72,8 @@
 from .data_containers import get_header_info
 from .data_containers import get_image_info
 from .data_containers import get_instrument_looks
-from .data_containers import get_thumbnails_all_instruments
+from .data_containers import get_rootnames_from_query
+from .data_containers import get_thumbnail_by_rootname
 from .data_containers import random_404_page
 from .data_containers import text_scrape
 from .data_containers import thumbnails_ajax
@@ -469,17 +470,8 @@ def archive_thumbnails_query_ajax(request):
     """
 
     parameters = request.session.get("query_config", QUERY_CONFIG_TEMPLATE.copy())
-    # Ensure the instrument is correctly capitalized
-    instruments_list = []
-    for instrument in parameters[QUERY_CONFIG_KEYS.INSTRUMENTS]:
-        instrument = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()]
-        instruments_list.append(instrument)
-
-    # when parameters only contains nirspec as instrument,
-    # thumbnails still end up being all niriss data
-    thumbnails = get_thumbnails_all_instruments(parameters)
-
-    data = thumbnails_query_ajax(thumbnails)
+    filtered_rootnames = get_rootnames_from_query(parameters)
+    data = thumbnails_query_ajax(filtered_rootnames)
     data['thumbnail_sort'] = request.session.get("image_sort", "Ascending")
     data['thumbnail_group'] = request.session.get("image_group", "Exposure")
 

From b6805768767f5a0cf45ebc76d79c55617d566c0e Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 7 Apr 2023 20:34:51 -0400
Subject: [PATCH 033/256] Nest the if statements to protect against empty file
 list

---
 .../jwql/monitor_pages/monitor_bias_bokeh.py  | 34 +++++++++++--------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index 971ebd8d3..12857063d 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -616,22 +616,28 @@ def __init__(self, data):
     def create_figure(self):
         """Create the Bokeh figure
         """
-        if len(self.data['cal_image']) > 0 and os.path.isfile(self.data['cal_image'].iloc[0]):
-            image = read_png(self.data['cal_image'].iloc[0])
-
-            datestr = self.data['expstart_str'].iloc[0]
-
-            # Display the 32-bit RGBA image
-            ydim, xdim = image.shape
-            dim = max(xdim, ydim)
-            self.figure = figure(title=f'Calibrated Zeroth Group of Most Recent Dark: {datestr}', x_range=(0, xdim), y_range=(0, ydim),
-                                 tools='pan,box_zoom,reset,wheel_zoom,save')
-            self.figure.image_rgba(image=[image], x=0, y=0, dw=xdim, dh=ydim)
-            self.figure.xaxis.visible = False
-            self.figure.yaxis.visible = False
+        if len(self.data['cal_image']) > 0:
+            if os.path.isfile(self.data['cal_image'].iloc[0]):
+                image = read_png(self.data['cal_image'].iloc[0])
+
+                datestr = self.data['expstart_str'].iloc[0]
+
+                # Display the 32-bit RGBA image
+                ydim, xdim = image.shape
+                dim = max(xdim, ydim)
+                self.figure = figure(title=f'Calibrated Zeroth Group of Most Recent Dark: {datestr}', x_range=(0, xdim), y_range=(0, ydim),
+                                     tools='pan,box_zoom,reset,wheel_zoom,save')
+                self.figure.image_rgba(image=[image], x=0, y=0, dw=xdim, dh=ydim)
+                self.figure.xaxis.visible = False
+                self.figure.yaxis.visible = False
+            else:
+                # If the listed file is missing, create an empty plot
+                self.figure = PlaceholderPlot('Calibrated Zeroth Group of Most Recent Dark', '', '').plot
+                self.figure.xaxis.visible = False
+                self.figure.yaxis.visible = False
 
         else:
-            # If the given file is missing, create an empty plot
+            # If no file is given, create an empty plot
             self.figure = PlaceholderPlot('Calibrated Zeroth Group of Most Recent Dark', '', '').plot
             self.figure.xaxis.visible = False
             self.figure.yaxis.visible = False

From cee75cd6aaa8c2cdee4c00473ecc0169c618f518 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Mon, 10 Apr 2023 12:09:14 -0400
Subject: [PATCH 034/256] fix expstart_iso bad access

---
 jwql/website/apps/jwql/data_containers.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 75fdb30b6..4d41f828d 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -2158,7 +2158,9 @@ def thumbnails_query_ajax(rootnames):
         data_dict['file_data'][rootname]['inst'] = JWST_INSTRUMENT_NAMES_MIXEDCASE[filename_parser(rootname)['instrument']]
         data_dict['file_data'][rootname]['filename_dict'] = filename_dict
         data_dict['file_data'][rootname]['available_files'] = available_files
-        data_dict['file_data'][rootname]['expstart'] = get_expstart(data_dict['file_data'][rootname]['inst'], rootname)
+        exp_start = get_expstart(data_dict['file_data'][rootname]['inst'], rootname)
+        data_dict['file_data'][rootname]['expstart'] = exp_start
+        data_dict['file_data'][rootname]['expstart_iso'] = Time(exp_start, format='mjd').iso.split('.')[0]
         data_dict['file_data'][rootname]['suffixes'] = []
         data_dict['file_data'][rootname]['prop'] = rootname[2:7]
         for filename in available_files:

From 5540a2594936b9c0d7bd29b3c92569a50de48434 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 10 Apr 2023 14:38:16 -0400
Subject: [PATCH 035/256] Figure tweaks

---
 .../jwql/monitor_pages/monitor_bias_bokeh.py  | 28 +++++++++++--------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index 12857063d..da66e2e99 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -101,7 +101,8 @@ def retrieve_trending_data(self, aperture):
                                           self.stats_table.amp3_odd_med,
                                           self.stats_table.amp4_even_med,
                                           self.stats_table.amp4_odd_med,
-                                          self.stats_table.expstart) \
+                                          self.stats_table.expstart,
+                                          self.stats_table.uncal_filename) \
             .filter(self.stats_table.aperture == aperture) \
             .order_by(self.stats_table.expstart) \
             .all()
@@ -113,7 +114,7 @@ def retrieve_trending_data(self, aperture):
                                                                       'amp2_even_med', 'amp2_odd_med',
                                                                       'amp3_even_med', 'amp3_odd_med',
                                                                       'amp4_even_med', 'amp4_odd_med',
-                                                                      'expstart_str'])
+                                                                      'expstart_str', 'uncal_filename'])
         # Add a column of expstart values that are datetime objects
         format_data = "%Y-%m-%dT%H:%M:%S.%f"
         datetimes = [datetime.strptime(entry, format_data) for entry in self.trending_data['expstart_str']]
@@ -128,12 +129,13 @@ def retrieve_latest_data(self, aperture):
         aperture : str
             Aperture name (e.g. NRCA1_FULL)
         """
-        subq = (session.query(func.max(self.stats_table.expstart).label("max_created")) \
-        .filter(self.stats_table.aperture == aperture) \
+        subq = (session.query(self.stats_table.aperture, func.max(self.stats_table.expstart).label("max_created")) \
+        .group_by(self.stats_table.aperture)
         .subquery()
         )
 
-        query = (session.query(self.stats_table.uncal_filename,
+        query = (session.query(self.stats_table.aperture,
+                               self.stats_table.uncal_filename,
                                self.stats_table.cal_filename,
                                self.stats_table.cal_image,
                                self.stats_table.expstart,
@@ -142,6 +144,7 @@ def retrieve_latest_data(self, aperture):
                                self.stats_table.counts,
                                self.stats_table.bin_centers,
                                self.stats_table.entry_date)
+                     .filter(self.stats_table.aperture == aperture)
                      .order_by(self.stats_table.entry_date) \
                      .join(subq, self.stats_table.expstart == subq.c.max_created)
                      )
@@ -152,10 +155,10 @@ def retrieve_latest_data(self, aperture):
         # Put the returned data in a dataframe. Include only the most recent entry.
         # The query has already filtered to include only entries using the latest
         # expstart value.
-        self.latest_data = pd.DataFrame(latest_data[-1:], columns=['uncal_filename', 'cal_filename',
-                                                                  'cal_image', 'expstart_str', 'collapsed_rows',
-                                                                  'collapsed_columns', 'counts', 'bin_centers',
-                                                                  'entry_date'])
+        self.latest_data = pd.DataFrame(latest_data[-1:], columns=['aperture', 'uncal_filename', 'cal_filename',
+                                                                   'cal_image', 'expstart_str', 'collapsed_rows',
+                                                                   'collapsed_columns', 'counts', 'bin_centers',
+                                                                   'entry_date'])
         # Add a column of expstart values that are datetime objects
         format_data = "%Y-%m-%dT%H:%M:%S.%f"
         datetimes = [datetime.strptime(entry, format_data) for entry in self.latest_data['expstart_str']]
@@ -535,7 +538,7 @@ def create_amp_plot(self, amp_num, amp_data):
 
         if len(amp_data["expstart"]) > 0:
             plot = figure(title=title_str, tools='pan,box_zoom,reset,wheel_zoom,save',
-                      background_fill_color="#fafafa")
+                          background_fill_color="#fafafa")
             source = ColumnDataSource(amp_data)
             even_col = f'amp{amp_num}_even_med'
             odd_col = f'amp{amp_num}_odd_med'
@@ -561,9 +564,10 @@ def create_amp_plot(self, amp_num, amp_data):
             # interpret the format codes as html tags and crash with errors such as:
             # "Encountered unknown tag 'd'. Jinja was looking for the following tags: 'endblock'.
             # The innermost block that needs to be closed is 'block'"
-            hover_tool = HoverTool(tooltips=[('Even col bias:', f'@{even_col}'),
+            hover_tool = HoverTool(tooltips=[('File:', '@uncal_filename'),
+                                             ('Even col bias:', f'@{even_col}'),
                                              ('Odd col bias:', f'@{odd_col}'),
-                                             ('Date:', '@expstart_str')
+                                             ('Date:', '@expstart')
                                              ]
                                    )
             hover_tool.formatters = {'@expstart': 'datetime'}

From bb7f9889f3d60ea8df51c446c4611bd9bdbf2f39 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 10 Apr 2023 15:09:17 -0400
Subject: [PATCH 036/256] Fix date hover values

---
 .../apps/jwql/monitor_pages/monitor_bias_bokeh.py        | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index da66e2e99..7f14c9fb2 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -115,6 +115,9 @@ def retrieve_trending_data(self, aperture):
                                                                       'amp3_even_med', 'amp3_odd_med',
                                                                       'amp4_even_med', 'amp4_odd_med',
                                                                       'expstart_str', 'uncal_filename'])
+        uncal_basename = [os.path.basename(e) for e in self.trending_data['uncal_filename']]
+        self.trending_data['uncal_filename'] = uncal_basename
+
         # Add a column of expstart values that are datetime objects
         format_data = "%Y-%m-%dT%H:%M:%S.%f"
         datetimes = [datetime.strptime(entry, format_data) for entry in self.trending_data['expstart_str']]
@@ -567,10 +570,10 @@ def create_amp_plot(self, amp_num, amp_data):
             hover_tool = HoverTool(tooltips=[('File:', '@uncal_filename'),
                                              ('Even col bias:', f'@{even_col}'),
                                              ('Odd col bias:', f'@{odd_col}'),
-                                             ('Date:', '@expstart')
+                                             ('Date:', '@expstart_str')
                                              ]
                                    )
-            hover_tool.formatters = {'@expstart': 'datetime'}
+            #hover_tool.formatters = {'@expstart': 'datetime'}
             plot.tools.append(hover_tool)
             plot.xaxis.axis_label = x_label
             plot.yaxis.axis_label = y_label
@@ -589,7 +592,7 @@ def create_plots(self):
         # Create one plot per amplifier
         for amp_num in range(1, 5):
             cols_to_use = [col for col in self.data.columns if str(amp_num) in col]
-            cols_to_use.append('expstart')
+            cols_to_use.extend(['expstart', 'expstart_str', 'uncal_filename'])
             subframe = self.data[cols_to_use]
             self.plots[amp_num] = self.create_amp_plot(amp_num, subframe)
 

From 86c8caca998d81e18a2501289cf1d3a008b99e27 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Mon, 10 Apr 2023 15:52:51 -0400
Subject: [PATCH 037/256] new buttons clear and return to form

---
 jwql/website/apps/jwql/templates/jwql_query.html | 16 +++++++++++-----
 .../apps/jwql/templates/query_submit.html        | 14 +++++++++++---
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index e58117cf2..feac6ba96 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -18,11 +18,16 @@
 
     <script type="text/javascript">
         $(document).ready(function () {
-            $("#MIRI").hide();
-            $("#NIRCAM").hide();
-            $("#NIRISS").hide();
-            $("#NIRSPEC").hide();
-            $("#FGS").hide();
+
+            hide_all = function() {
+                $("#MIRI").hide();
+                $("#NIRCAM").hide();
+                $("#NIRISS").hide();
+                $("#NIRSPEC").hide();
+                $("#FGS").hide();
+            }
+
+            hide_all();
 
             $("#instrument").change(function () {
                 if (document.getElementById("id_instrument_1").checked) {
@@ -481,6 +486,7 @@ <h1>Dynamic Query Form</h1>
                     <hr class="col-md-12">
                     <div class='form-group'>
                         <input type="submit" value="Submit" class="btn btn-primary" name="jwstqueryform" />
+                        <input type="reset" value="Clear Form" class="btn btn-primary mx-4" name="jwstqueryform" onclick="hide_all();" />
                     </div>
                 </div>
             </form>
diff --git a/jwql/website/apps/jwql/templates/query_submit.html b/jwql/website/apps/jwql/templates/query_submit.html
index 20d1fccbc..7bac5a290 100644
--- a/jwql/website/apps/jwql/templates/query_submit.html
+++ b/jwql/website/apps/jwql/templates/query_submit.html
@@ -22,12 +22,16 @@ <h3 id='title'>Images of Queried Instruments</h3>
     	<div id='img_show_count'></div>
     	<hr>
 
+        <!-- Link Back to Query Page -->
+        <a role="button" class="btn btn-primary my-2" type="submit" href="/jwql_query/">Return To Query Form</a>
+        <hr>
+
     	<!-- Filter and sort options -->
 		<div class="d-flex flex-row">
 			<div class="mr-4" id="thumbnail-filter"></div>
 			<div class="mx-4" id="thumbnail-sort"></div>
             <div class="mx-4 mr-5 ml-auto" id="group-by-exposure"></div>
-		</div> 
+		</div>
 
 		<hr>
 
@@ -53,9 +57,13 @@ <h3 id='title'>Images of Queried Instruments</h3>
         <script>update_thumbnails_query_page('{{base_url}}', '{{ sort }}', '{{ group }}');</script>
         <a id="no_thumbnails_msg" style='display: none'>No data match the selected criteria.</a>
 
+        <hr>
+        <!-- Link Back to Query Page -->
+        <a role="button" class="btn btn-primary my-2" type="submit" href="/jwql_query/">Return To Query Form</a>
+
 	</main>
 
-    
-   
+
+
 
 {% endblock %}

From a1323aa1f6e8f05844ab6f227e801d346ae68aaf Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Mon, 10 Apr 2023 16:29:17 -0400
Subject: [PATCH 038/256] initial hide of elements to avoid 'peek' effect

---
 jwql/website/apps/jwql/templates/jwql_query.html | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index feac6ba96..47e74eafb 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -87,7 +87,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- FGS Fields -->
-                    <div style="display: flex" id="FGS">
+                    <div style="display: flex" id="FGS" style="display: none;">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">FGS Anomalies</span>
                             <div class="form-control" id='fgs_anomalies'>
@@ -150,7 +150,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- MIRI Fields -->
-                    <div style="display: flex" id="MIRI">
+                    <div style="display: flex" id="MIRI" style="display: none;">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">MIRI Anomalies</span>
                             <div class="form-control" id='miri_anomalies'>
@@ -224,7 +224,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- NIRCAM Fields -->
-                    <div style="display: flex" id="NIRCAM">
+                    <div style="display: flex" id="NIRCAM" style="display: none;">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRCam Anomalies</span>
                             <div class="form-control" id='nircam_anomalies'>
@@ -311,7 +311,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- NIRISS Fields -->
-                    <div style="display: flex" id="NIRISS">
+                    <div style="display: flex" id="NIRISS" style="display: none;">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRISS Anomalies</span>
                             <div class="form-control" id='niriss_anomalies'>
@@ -398,7 +398,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- NIRSPEC Fields -->
-                    <div style="display: flex" id="NIRSPEC">
+                    <div style="display: flex" id="NIRSPEC" style="display: none;">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRSpec Anomalies</span>
                             <div class="form-control" id='nirspec_anomalies'>

From 26d496512516d31770d23affec9136ad1df3ff79 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 11 Apr 2023 09:01:23 -0400
Subject: [PATCH 039/256] fix get_rootnames_from_query return val bug

---
 jwql/website/apps/jwql/data_containers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 4d41f828d..43b1ff2ca 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1549,7 +1549,8 @@ def get_rootnames_from_query(parameters):
 
             filtered_rootnames.extend(anomaly_rootnames)
         else:
-            filtered_rootnames.extend(current_ins_rootfileinfos)
+            rootnames = [name[0] for name in current_ins_rootfileinfos.values_list("root_name")]
+            filtered_rootnames.extend(rootnames)
 
     return list(set(filtered_rootnames))
     #      >>>  END CODE PRE DB MIGRATION HERE  <<<

From 96508c5e1fa91ce3afae58107ab91e14531f4f8f Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 11 Apr 2023 09:02:43 -0400
Subject: [PATCH 040/256] remove hide code, affects back navigation

---
 jwql/website/apps/jwql/templates/jwql_query.html | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index 47e74eafb..feac6ba96 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -87,7 +87,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- FGS Fields -->
-                    <div style="display: flex" id="FGS" style="display: none;">
+                    <div style="display: flex" id="FGS">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">FGS Anomalies</span>
                             <div class="form-control" id='fgs_anomalies'>
@@ -150,7 +150,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- MIRI Fields -->
-                    <div style="display: flex" id="MIRI" style="display: none;">
+                    <div style="display: flex" id="MIRI">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">MIRI Anomalies</span>
                             <div class="form-control" id='miri_anomalies'>
@@ -224,7 +224,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- NIRCAM Fields -->
-                    <div style="display: flex" id="NIRCAM" style="display: none;">
+                    <div style="display: flex" id="NIRCAM">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRCam Anomalies</span>
                             <div class="form-control" id='nircam_anomalies'>
@@ -311,7 +311,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- NIRISS Fields -->
-                    <div style="display: flex" id="NIRISS" style="display: none;">
+                    <div style="display: flex" id="NIRISS">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRISS Anomalies</span>
                             <div class="form-control" id='niriss_anomalies'>
@@ -398,7 +398,7 @@ <h1>Dynamic Query Form</h1>
                     </div>
 
                     <!-- NIRSPEC Fields -->
-                    <div style="display: flex" id="NIRSPEC" style="display: none;">
+                    <div style="display: flex" id="NIRSPEC">
                         <section class="col-md">
                             <span id="helpBlock" class="help-block">NIRSpec Anomalies</span>
                             <div class="form-control" id='nirspec_anomalies'>

From 082ba19598b3365ea62c050c0f3e2405d79c7935 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Thu, 13 Apr 2023 13:17:20 -0400
Subject: [PATCH 041/256] remove usages of deprecated `codecov` package

the `codecov` package is now deprecated and has been yanked from PyPI
---
 setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/setup.py b/setup.py
index 45f081f50..6d30c3958 100644
--- a/setup.py
+++ b/setup.py
@@ -16,7 +16,6 @@
     'astroquery',
     'bandit',
     'bokeh<3',
-    'codecov',
     'crds',
     'cryptography',
     'django',

From 5042f6c9dd3e77483f04b5c12248366a0c21a552 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Thu, 13 Apr 2023 13:18:11 -0400
Subject: [PATCH 042/256] remove `codecov`

---
 environment_python_3_9.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/environment_python_3_9.yml b/environment_python_3_9.yml
index 9918f0ca3..2aadf5067 100644
--- a/environment_python_3_9.yml
+++ b/environment_python_3_9.yml
@@ -26,7 +26,6 @@ dependencies:
   - bokeh=2.4.3
   - beautifulsoup4=4.11.2
   - celery=5.2.7
-  - codecov=2.1.12
   - cryptography=39.0.1
   - django=4.1.7
   - flake8=6.0.0

From d6af4d59f25b24a070ae56a69460ff4afcfb54f1 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Thu, 13 Apr 2023 13:18:28 -0400
Subject: [PATCH 043/256] remove `codecov`

---
 environment_python_3_8.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml
index 2a31ef0ba..df3d3a173 100644
--- a/environment_python_3_8.yml
+++ b/environment_python_3_8.yml
@@ -26,7 +26,6 @@ dependencies:
   - bokeh=2.4.3
   - beautifulsoup4=4.11.2
   - celery=5.2.7
-  - codecov=2.1.12
   - cryptography=39.0.1
   - django=4.1.7
   - flake8=6.0.0

From b804ebdfc5dc6e09fb7ec2618bbdfe20328e609e Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Thu, 13 Apr 2023 13:18:43 -0400
Subject: [PATCH 044/256] remove `codecov`

---
 environment_python_3_10.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/environment_python_3_10.yml b/environment_python_3_10.yml
index 22bad9d5c..68a1fa077 100644
--- a/environment_python_3_10.yml
+++ b/environment_python_3_10.yml
@@ -26,7 +26,6 @@ dependencies:
   - bokeh=2.4.3
   - beautifulsoup4=4.11.2
   - celery=5.2.7
-  - codecov=2.1.12
   - cryptography=39.0.1
   - django=4.1.7
   - flake8=6.0.0

From 216d94947439774492e9f1eb33bd78da1f6ea31a Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Thu, 13 Apr 2023 13:20:13 -0400
Subject: [PATCH 045/256] remove usages of deprecated `codecov` package

the `codecov` package is now deprecated and has been yanked from PyPI
---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 66e027461..d4bc70951 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,6 @@ bandit==1.7.4
 beautifulsoup4==4.11.2
 bokeh==2.4.3
 celery==5.2.7
-codecov==2.1.12
 cryptography==39.0.1
 django==4.1.7
 flake8==6.0.0

From 7f92cc8ec920699c2416a3e0384d6e512162f2ea Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 13 Apr 2023 14:41:24 -0400
Subject: [PATCH 046/256] Pass filter img counts as list

---
 jwql/jwql_monitors/monitor_filesystem.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index d8f101ead..b63d2e182 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -674,11 +674,13 @@ def update_characteristics_database(char_info):
 
     # Add data to filesystem_instrument table
     for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:
+        optics = [e[0] for e in char_info[instrument]]
+        values = [e[1] for e in char_info[instrument]]
         new_record = {}
         new_record['date'] = now
         new_record['instrument'] = instrument
-        new_record['filter_pupil'] = list(char_info[instrument].keys())
-        new_record['obs_per_filter_pupil'] = list(char_info[instrument].values())
+        new_record['filter_pupil'] = optics
+        new_record['obs_per_filter_pupil'] = values
         with engine.begin() as connection:
             connection.execute(
                 FilesystemCharacteristics.__table__.insert(), new_record)

From d24f572cfc3a7f994eeeb2ccf28b110f7a92cb15 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 13 Apr 2023 14:42:48 -0400
Subject: [PATCH 047/256] update docstring

---
 jwql/jwql_monitors/monitor_filesystem.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index b63d2e182..33fa0ced7 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -666,8 +666,10 @@ def update_characteristics_database(char_info):
     Parameters
     ----------
     char_info : dict
-        A nested dictionary of characteristic information. Top level keys are
-        instrument names, and second level keys are filter/pupil strings.
+        A dictionary of characteristic information. Keys are
+        instrument names, and values are lists of tuples. Each tuple is
+        composed of a filter/pupil string and a count for the number of observations
+        using that filter/pupil.
     """
     logging.info('\tUpdating the characteristics database')
     now = datetime.datetime.now()

From 2f06306135469117c23b0a2d147350b8806d55d5 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 13 Apr 2023 14:44:23 -0400
Subject: [PATCH 048/256] update another docstring

---
 jwql/jwql_monitors/monitor_filesystem.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 33fa0ced7..2bf9ef705 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -294,9 +294,9 @@ def get_observation_characteristics():
     Returns
     -------
     n_obs : dict
-        Nested dictionary with instrument names as the top level keys, and
-        filter/pupil values as the second level keys. Values are the number of
-        observations that use the filter/pupil value.
+        Dictionary with instrument names as the top level keys, and lists of 2-tuples
+        as values. Each tuple contains filter/pupil string and the number of
+        observations that use that filter/pupil.
     """
     n_obs = {}
     for instrument in ['nircam', 'niriss', 'nirspec', 'miri']:  # Skip FGS here. It has no filters

From f1d833a8b6a5c98926f9a0c78d1ca521e5886e97 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 14 Apr 2023 09:45:02 -0400
Subject: [PATCH 049/256] Move thumbnail img updates outside initial html
 creation loop

---
 jwql/website/apps/jwql/static/js/jwql.js | 58 +++++++++++++++---------
 1 file changed, 37 insertions(+), 21 deletions(-)

diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 43de8ddab..f4166d475 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -250,25 +250,6 @@ function clean_input_parameter(param_value) {
 }
 
 
-/**
- * Determine what filetype to use for a thumbnail
- * @param {String} thumbnail_dir - The path to the thumbnail directory
- * @param {List} suffixes - A list of available suffixes for the file of interest
- * @param {Integer} i - The index of the thumbnail
- * @param {String} file_root - The rootname of the file corresponding to the thumbnail
- */
-function determine_filetype_for_thumbnail(thumbnail_dir, thumb_filename, i, file_root) {
-
-    // Update the thumbnail filename
-    var img = document.getElementById('thumbnail'+i);
-    if (thumb_filename != 'none') {
-        var jpg_path = thumbnail_dir + parse_filename(file_root).program + '/' + thumb_filename;
-        img.src = jpg_path;
-    }
-
-}
-
-
 /**
  * Determine whether the page is archive or unlooked
  * @param {String} instrument - The instrument of interest
@@ -552,6 +533,33 @@ function unhide_file(detector) {
     update_view_explore_link();
 }
 
+
+/**
+ * Insert thumbnail images inside existing HTML img tags
+ * @param {List} updates - A list of updates to make, as [thumbnail_id, jpg_path].
+ */
+function insert_thumbnail_images(updates) {
+    // Update the thumbnail image source
+    for (var i = 0; i < updates.length; i++) {
+        var thumb_id = updates[i][0];
+        var jpg_path = updates[i][1];
+        set_thumbnail_image_source(thumb_id, jpg_path);
+    }
+}
+
+
+/**
+ * Check for a thumbnail image and add it to an img if it exists
+ * @param {Integer} thumb_id - The ID number for the thumbnail img
+ * @param {String} jpg_filepath - The image to show
+ */
+function set_thumbnail_image_source(thumb_id, jpg_path) {
+    $.get(jpg_path, function() {
+        var img = document.getElementById('thumbnail' + thumb_id);
+        img.src = jpg_path;})
+}
+
+
 /**
  * Parse observation information from a JWST file name.
  * @param {String} filename - The file or group root name to parse
@@ -1255,6 +1263,8 @@ function update_sort_options(data, base_url) {
 function update_thumbnail_array(data) {
 
     // Add content to the thumbnail array div
+    var thumbnail_content = "";
+    var image_updates = [];
     for (var i = 0; i < Object.keys(data.file_data).length; i++) {
         
         // Parse out useful variables
@@ -1295,11 +1305,17 @@ function update_thumbnail_array(data) {
         content += '</div></a></div></div>';
 
         // Add the content to the div
-        $("#thumbnail-array")[0].innerHTML += content;
+        thumbnail_content += content;
 
         // Add the appropriate image to the thumbnail
-        determine_filetype_for_thumbnail('/static/thumbnails/' , file.thumbnail, i, rootname);
+        if (file.thumbnail != 'none') {
+            var jpg_path = '/static/thumbnails/' + parse_filename(rootname).program +
+                           '/' + file.thumbnail;
+            image_updates.push([i, jpg_path]);
+        }
     }
+    $("#thumbnail-array")[0].innerHTML = thumbnail_content;
+    insert_thumbnail_images(image_updates);
 }
 
 /**

From 2437d2c9ee38f22bf4bf594f79a8b7f92e7d2edd Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 14 Apr 2023 09:57:50 -0400
Subject: [PATCH 050/256] remove MAST call in thumbnails_query_ajax

---
 jwql/website/apps/jwql/data_containers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 43b1ff2ca..7b32f55d9 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -2159,7 +2159,8 @@ def thumbnails_query_ajax(rootnames):
         data_dict['file_data'][rootname]['inst'] = JWST_INSTRUMENT_NAMES_MIXEDCASE[filename_parser(rootname)['instrument']]
         data_dict['file_data'][rootname]['filename_dict'] = filename_dict
         data_dict['file_data'][rootname]['available_files'] = available_files
-        exp_start = get_expstart(data_dict['file_data'][rootname]['inst'], rootname)
+        root_file_info = RootFileInfo.objects.get(root_name=rootname)
+        exp_start = root_file_info.expstart
         data_dict['file_data'][rootname]['expstart'] = exp_start
         data_dict['file_data'][rootname]['expstart_iso'] = Time(exp_start, format='mjd').iso.split('.')[0]
         data_dict['file_data'][rootname]['suffixes'] = []

From ad219c88fca88ea5526916b31e3fac7087c538f2 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 14 Apr 2023 10:44:52 -0400
Subject: [PATCH 051/256] Fix MIRI subarrays

---
 jwql/utils/constants.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 432344cea..7dbc62e01 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -501,8 +501,8 @@ class QUERY_CONFIG_KEYS:
                             'niriss': ['FULL', 'SUBSTRIP96', 'SUBSTRIP256', 'SUB80', 'SUB64', 'SUB128', 'SUB256',
                                        'WFSS64R', 'WFSS128R', 'WFSS64C', 'WFSS128C', 'SUBAMPCAL', 'SUBTAAMI', 'SUBTASOSS'],
                             'nirspec': [],
-                            'miri': ['MIRIM_BRIGHTSKY', 'MIRIM_FULL', 'MIRIM_MASK1065', 'MIRIM_MASK1140',
-                                     'MIRIM_MASK1550', 'MIRIM_MASKLYOT', 'MIRIM_SLITLESSPRISM', 'MIRIM_SUB64', 'MIRIM_SUB128', 'MIRIM_SUB256'],
+                            'miri': ['BRIGHTSKY', 'FULL', 'MASK1065', 'MASK1140', 'MASK1550', 'MASKLYOT', 'SLITLESSPRISM',
+                                     'SUB64', 'SUB128', 'SUB256'],
                             'fgs': []}
 
 # Filename suffixes that need to include the association value in the suffix in

From 9a020a06d0c72409a2a0f8b8ab53aaf41a916c31 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 14 Apr 2023 12:33:21 -0400
Subject: [PATCH 052/256] Fix misspelled constant name

---
 jwql/jwql_monitors/monitor_filesystem.py | 6 +++---
 jwql/utils/constants.py                  | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 2bf9ef705..95eddceb0 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -56,7 +56,7 @@
 from jwql.database.database_interface import CentralStore
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
-from jwql.utils.constants import FILESYSYEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH
+from jwql.utils.constants import FILESYSTEM_MONITOR_SUBDIRS, FILE_SUFFIX_TYPES, FILTERS_PER_INSTRUMENT, INSTRUMENT_SERVICE_MATCH
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.utils import filename_parser
 from jwql.utils.utils import get_config
@@ -540,7 +540,7 @@ def plot_central_store_dirs():
     plot.circle(dates, availables, color='blue')
 
     # This part of the plot should cycle through areas and plot area used values vs. date
-    for area, color in zip(FILESYSYEM_MONITOR_SUBDIRS, colors):
+    for area, color in zip(FILESYSTEM_MONITOR_SUBDIRS, colors):
 
         # Query for used sizes
         results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area)
@@ -648,7 +648,7 @@ def update_central_store_database(central_storage_dict):
     central_storage_dict : dict
         A dictionary for the ``central_storage`` database table
     """
-    for area in FILESYSYEM_MONITOR_SUBDIRS:
+    for area in FILESYSTEM_MONITOR_SUBDIRS:
         new_record = {}
         new_record['date'] = central_storage_dict['date']
         new_record['area'] = area
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 63e40ffa6..8d8cbab73 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -235,7 +235,7 @@
                   'fgs': ['FGS_INTFLAT']}
 
 # output subdirectories to keep track of via the filesytem monitor
-FILESYSYEM_MONITOR_SUBDIRS = ['logs', 'outputs', 'preview_images', 'thumbnails', 'all']
+FILESYSTEM_MONITOR_SUBDIRS = ['logs', 'outputs', 'preview_images', 'thumbnails', 'all']
 
 FILTERS_PER_INSTRUMENT = {'fgs': [],
                           'miri': ['F1000W', 'F1130W', 'F1280W', 'OPAQUE', 'F2300C', 'F560W', 'P750L',

From bba5f08bcc8e1055dbcf5c0c19d640f2bebba095 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 14 Apr 2023 12:47:31 -0400
Subject: [PATCH 053/256] Remove unused bokeh templating file

---
 jwql/jwql_monitors/monitor_filesystem.py      |   1 +
 jwql/website/apps/jwql/bokeh_dashboard.py     |   1 +
 .../monitor_pages/monitor_filesystem_bokeh.py | 225 ------------------
 3 files changed, 2 insertions(+), 225 deletions(-)
 delete mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_filesystem_bokeh.py

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 95eddceb0..0d6158d51 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -11,6 +11,7 @@
     - Misty Cracraft
     - Sara Ogaz
     - Matthew Bourque
+    - Bryan Hilbert
 
 Use
 ---
diff --git a/jwql/website/apps/jwql/bokeh_dashboard.py b/jwql/website/apps/jwql/bokeh_dashboard.py
index 47a3933a3..2c282e274 100644
--- a/jwql/website/apps/jwql/bokeh_dashboard.py
+++ b/jwql/website/apps/jwql/bokeh_dashboard.py
@@ -11,6 +11,7 @@
 -------
 
     - Mees B. Fix
+    - Bryan Hilbert
 
 Use
 ---
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_filesystem_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_filesystem_bokeh.py
deleted file mode 100644
index fa208db79..000000000
--- a/jwql/website/apps/jwql/monitor_pages/monitor_filesystem_bokeh.py
+++ /dev/null
@@ -1,225 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Wed Jan 16 14:09:18 2019
-
-@author: gkanarek
-"""
-
-import json
-import os
-
-from astropy.table import Table, vstack
-from astropy.time import Time
-
-from jwql.bokeh_templating import BokehTemplate
-from jwql.utils.utils import get_config
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-
-FIG_FORMATS = """
-Figure:
-    tools: 'pan,box_zoom,reset,wheel_zoom,save'
-    x_axis_type: 'datetime'
-    x_axis_label: 'Date'
-    sizing_mode: 'stretch_both'
-Line:
-    line_width: 2
-"""
-
-
-class MonitorFilesystem(BokehTemplate):
-
-    def pre_init(self):
-        self._embed = True
-
-        # App design
-        self.format_string = FIG_FORMATS
-        self.interface_file = os.path.join(SCRIPT_DIR, "yaml", "monitor_filesystem_interface.yaml")
-
-        # Get path, directories and files in system and count files in all directories
-        self.settings = get_config()
-        self.filesystem = self.settings['filesystem']
-        self.outputs_dir = os.path.join(self.settings['outputs'],
-                                        'monitor_filesystem')
-
-        self.allowed_types = ['fits_files', 'uncal', 'cal', 'rate', 'rateints',
-                              'i2d', 'nrc', 'nrs', 'nis', 'mir', 'gui']
-
-        # Load any existing data
-        self.initial_load()
-
-        self.types_k = ['circle', 'diamond', 'square', 'triangle',
-                        'asterisk'] + ['x'] * 6
-        self.types_y = ['fits', 'uncal', 'cal', 'rate', 'rateint',
-                        'i2d', 'nrc', 'nrs', 'nis', 'mir', 'fgs']
-        self.types_c = ['black', 'red', 'blue', 'green', 'orange', 'purple',
-                        'midnightblue', 'springgreen', 'darkcyan',
-                        'dodgerblue', 'darkred']
-        self.types_l = ['Total FITS files', 'Uncalibrated FITS files',
-                        'Calibrated FITS files', 'Rate FITS files',
-                        'Rateints FITS files', 'I2D FITS files',
-                        'NIRCam FITS files', 'NIRSpec FITS files',
-                        'NIRISS FITS files', 'MIRI FITS files',
-                        'FGS FITS files']
-
-    def post_init(self):
-        self.update_plots(full=True)
-
-    def initial_load(self):
-        statsfile = os.path.join(self.outputs_dir, 'statsfile.json')
-        filebytype = os.path.join(self.outputs_dir, 'filesbytype.json')
-        sizebytype = os.path.join(self.outputs_dir, 'sizebytype.json')
-
-        self.statistics = Table(names=['timestamp', 'file_count', 'total',
-                                       'available', 'used', 'percent_used'],
-                                dtype=[Time, int, int, int, int, float])
-        self.statistics['percent_used'].format = "%.1f"
-        if os.path.exists(statsfile):
-            with open(statsfile) as f:
-                stats = json.load(f)
-            times, fc, tot, avail, used, perc = zip(*stats)
-            self.statistics['timestamp'] = Time(times)
-            self.statistics['file_count'] = map(int, fc)
-            self.statistics['total'] = map(int, tot)
-            self.statistics['available'] = map(int, avail)
-            self.statistics['used'] = map(int, used)
-            self.statistics['percent_used'] = map(float, perc)
-
-        self.ftypes = Table(names=['timestamp'] + self.allowed_types,
-                            dtype=[Time] + [int] * 11)
-        if os.path.exists(filebytype):
-            with open(filebytype) as f:
-                fbytype = json.load(f)
-            times, *ftypes = zip(*fbytype)
-            self.ftypes['timestamp'] = Time(times)
-            for c, colname in enumerate(self.allowed_types):
-                self.ftypes[colname] = map(int, ftypes[c])
-
-        self.stypes = Table(names=['timestamp'] + self.allowed_types,
-                            dtype=[Time] + [float] * 11)
-        if os.path.exists(sizebytype):
-            with open(sizebytype) as f:
-                sbytype = json.load(f)
-            times, *stypes = zip(*sbytype)
-            self.stypes['timestamp'] = Time(times)
-            for c, colname in enumerate(self.allowed_types):
-                self.stypes[colname] = map(int, stypes[c])
-
-    def update_plots(self, full=False):
-
-        if full:
-            # Initialize each ColumnDataSource so that we can use stream() later
-            self.refs['source_filecount'].data = {
-                'dates': self.statistics['timestamp'].datetime64,
-                'filecount': self.statistics['file_count'].data}
-
-            self.refs['source_stats'].data = {
-                'dates': self.statistics['timestamp'].datetime64,
-                'systemsize': self.statistics['total'].data.astype(float) / (1024.**3),
-                'freesize': self.statistics['available'].data.astype(float) / (1024.**3),
-                'usedsize': self.statistics['used'].data.astype(float) / (1024.**3)}
-
-            ftype_dict = {'dates': self.ftypes['timestamp'].datetime64}
-            ftype_dict.update({x: self.ftypes[y].data for x, y in zip(self.types_y,
-                               self.allowed_types)})
-            self.refs['source_files'].data = ftype_dict
-
-            stype_dict = {'dates': self.stypes['timestamp'].datetime64}
-            stype_dict.update({x: self.stypes[y].data for x, y in zip(self.types_y,
-                               self.allowed_types)})
-            self.refs['source_sizes'].data = stype_dict
-        else:
-            new_stats, new_files, new_sizes = self.read_new_data()
-            if new_stats:
-                self.refs['source_filecount'].stream({
-                    'dates': new_stats['timestamp'].datetime64,
-                    'filecount': new_stats['file_count'].data})
-                self.refs['source_stats'].stream({
-                    'dates': new_stats['timestamp'].datetime64,
-                    'systemsize': new_stats['total'].data,
-                    'freesize': new_stats['available'].data,
-                    'usedsize': new_stats['used'].data})
-            if new_files:
-                ftype_dict = {'dates': new_files['timestamp'].datetime64}
-                ftype_dict.update({x: new_files[y].data for x, y in zip(self.types_y,
-                                   self.allowed_types)})
-                self.refs['source_files'].stream(ftype_dict)
-
-            if new_sizes:
-                stype_dict = {'dates': new_sizes['timestamp'].datetime64}
-                stype_dict.update({x: new_sizes[y].data / (1024.**3)
-                                  for x, y in zip(self.types_y, self.allowed_types)})
-                self.refs['source_sizes'].data = stype_dict
-
-        if not self.statistics:
-            self.latest_timestamp = Time(0., format='unix')
-        else:
-            self.latest_timestamp = self.statistics['timestamp'].max()
-
-    def read_new_data(self):
-        """
-        Algorithm:
-            1. Read in the json files (this step will be replaced when we move
-               away from json) into tables.
-            2. Create new tables from all rows which have been added since the
-               last timestamp in the current tables.
-            3. Concatenate the new tables with a vertical join.
-            4. Return the new tables so they can be streamed to the plots.
-        """
-        statsfile = os.path.join(self.outputs_dir, 'statsfile.json')
-        filebytype = os.path.join(self.outputs_dir, 'filesbytype.json')
-        sizebytype = os.path.join(self.outputs_dir, 'sizebytype.json')
-
-        # Have any of the files been modified since the last timestamp?
-        stats_modtime = Time(os.stat(statsfile).st_mtime, format='unix')
-        files_modtime = Time(os.stat(filebytype).st_mtime, format='unix')
-        sizes_modtime = Time(os.stat(sizebytype).st_mtime, format='unix')
-
-        new_stats = Table(names=self.statistics.colnames,
-                          dtype=self.statistics.dtype)
-        new_files = Table(names=self.ftypes.colnames,
-                          dtype=self.ftypes.dtype)
-        new_sizes = Table(names=self.stypes.colnames,
-                          dtype=self.stypes.dtype)
-
-        if stats_modtime > self.latest_timestamp:
-            with open(statsfile) as f:
-                stats = json.load(f)
-            times, fc, tot, avail, used, perc = zip(*stats)
-            times = Time(times)
-            new_rows = times > self.latest_timestamp
-            new_stats['timestamp'] = times[new_rows]
-            new_stats['file_count'] = map(int, fc[new_rows])
-            new_stats['total'] = map(int, tot[new_rows])
-            new_stats['available'] = map(int, avail[new_rows])
-            new_stats['used'] = map(int, used[new_rows])
-            new_stats['percent_used'] = map(float, perc[new_rows])
-
-            self.statistics = vstack([self.statistics, new_stats])
-
-        if files_modtime > self.latest_timestamp:
-            with open(filebytype) as f:
-                fbytype = json.load(f)
-            times, *ftypes = zip(*fbytype)
-            times = Time(times)
-            new_rows = times > self.latest_timestamp
-            new_files['timestamp'] = times[new_rows]
-            for c, colname in enumerate(self.allowed_types):
-                new_files[colname] = map(int, ftypes[c][new_rows])
-
-            self.ftypes = vstack([self.ftypes, new_files])
-
-        if sizes_modtime > self.latest_timestamp:
-            with open(sizebytype) as f:
-                sbytype = json.load(f)
-            times, *stypes = zip(*sbytype)
-            times = Time(times)
-            new_rows = times > self.latest_timestamp
-            new_sizes['timestamp'] = times[new_rows]
-            for c, colname in enumerate(self.allowed_types):
-                new_sizes[colname] = map(int, stypes[c][new_rows])
-
-            self.stypes = vstack([self.stypes, new_sizes])
-
-        return new_stats, new_files, new_sizes

From 2eb6da7d1e205c43d9fb325340ff08f7f13a5064 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 14 Apr 2023 13:17:10 -0400
Subject: [PATCH 054/256] Remove old unused plotting functions

---
 jwql/jwql_monitors/monitor_filesystem.py | 230 -----------------------
 1 file changed, 230 deletions(-)

diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py
index 0d6158d51..2c1f4d379 100755
--- a/jwql/jwql_monitors/monitor_filesystem.py
+++ b/jwql/jwql_monitors/monitor_filesystem.py
@@ -410,236 +410,6 @@ def monitor_filesystem():
     update_characteristics_database(characteristics)
     update_central_store_database(central_storage_dict)
 
-    # Create the plots
-    #plot_filesystem_stats()
-
-
-def plot_by_filetype(plot_type, instrument):
-    """Plot ``count`` or ``size`` by filetype versus date for the given
-    instrument, or all instruments.
-
-    Parameters
-    ----------
-    plot_type : str
-        Which data to plot.  Either ``count`` or ``size``.
-    instrument : str
-        The instrument to plot for.  Can be a valid JWST instrument or
-        ``all`` to plot across all instruments.
-
-    Returns
-    -------
-    plot : bokeh.plotting.figure.Figure object
-        ``bokeh`` plot of total file counts versus date
-    """
-
-    # Determine plot title
-    if instrument == 'all':
-        title = 'Total File {} by Type'.format(plot_type.capitalize())
-    else:
-        instrument_title = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument]
-        title = '{} Total File {} by Type'.format(instrument_title, plot_type.capitalize())
-
-    if plot_type == 'count':
-        ytitle = 'Counts'
-    else:
-        ytitle = 'Size (TB)'
-
-    # Initialize plot
-    plot = figure(
-        tools='pan,box_zoom,wheel_zoom,reset,save',
-        x_axis_type='datetime',
-        title=title,
-        x_axis_label='Date',
-        y_axis_label=ytitle)
-    colors = itertools.cycle(palette)
-
-    for filetype, color in zip(FILE_SUFFIX_TYPES, colors):
-
-        # Query for counts
-        results = session.query(FilesystemInstrument.date, getattr(FilesystemInstrument, plot_type))\
-            .filter(FilesystemInstrument.filetype == filetype)
-
-        if instrument == 'all':
-            results = results.all()
-        else:
-            results = results.filter(FilesystemInstrument.instrument == instrument).all()
-
-        # Group by date
-        if results:
-            results_dict = defaultdict(int)
-            for date, value in results:
-                results_dict[date] += value
-
-            # Parse results so they can be easily plotted
-            dates = list(results_dict.keys())
-            values = list(results_dict.values())
-
-            # Plot the results
-            plot.line(dates, values, legend='{} files'.format(filetype), line_color=color)
-            plot.circle(dates, values, color=color)
-
-    session.close()
-
-    return plot
-
-
-def plot_filesystem_size():
-    """Plot filesystem sizes (size, used, available) versus date
-
-    Returns
-    -------
-    plot : bokeh.plotting.figure.Figure object
-        ``bokeh`` plot of total file counts versus date
-    """
-
-    # Plot system stats vs. date
-    results = session.query(FilesystemGeneral.date, FilesystemGeneral.total_file_size,
-                            FilesystemGeneral.used, FilesystemGeneral.available).all()
-    dates, total_sizes, useds, availables = zip(*results)
-    plot = figure(
-        tools='pan,box_zoom,wheel_zoom,reset,save',
-        x_axis_type='datetime',
-        title='System stats',
-        x_axis_label='Date',
-        y_axis_label='Size TB')
-    plot.line(dates, total_sizes, legend='Total size', line_color='red')
-    plot.circle(dates, total_sizes, color='red')
-    plot.line(dates, useds, legend='Used bytes', line_color='green')
-    plot.circle(dates, useds, color='green')
-    plot.line(dates, availables, legend='Free bytes', line_color='blue')
-    plot.circle(dates, availables, color='blue')
-
-    session.close()
-    return plot
-
-
-def plot_central_store_dirs():
-    """Plot central store sizes (size, used, available) versus date
-
-        Returns
-        -------
-        plot : bokeh.plotting.figure.Figure object
-            ``bokeh`` plot of total directory size versus date
-        """
-
-    # Plot system stats vs. date
-    results = session.query(CentralStore.date, CentralStore.size, CentralStore.available).all()
-
-    # Initialize plot
-    dates, total_sizes, availables = zip(*results)
-    plot = figure(
-        tools='pan,box_zoom,wheel_zoom,reset,save',
-        x_axis_type='datetime',
-        title='Central Store stats',
-        x_axis_label='Date',
-        y_axis_label='Size TB')
-    colors = itertools.cycle(palette)
-
-    plot.line(dates, total_sizes, legend='Total size', line_color='red')
-    plot.circle(dates, total_sizes, color='red')
-    plot.line(dates, availables, legend='Free', line_color='blue')
-    plot.circle(dates, availables, color='blue')
-
-    # This part of the plot should cycle through areas and plot area used values vs. date
-    for area, color in zip(FILESYSTEM_MONITOR_SUBDIRS, colors):
-
-        # Query for used sizes
-        results = session.query(CentralStore.date, CentralStore.used).filter(CentralStore.area == area)
-
-        # Group by date
-        if results:
-            results_dict = defaultdict(int)
-            for date, value in results:
-                results_dict[date] += value
-
-            # Parse results so they can be easily plotted
-            dates = list(results_dict.keys())
-            values = list(results_dict.values())
-
-            # Plot the results
-            plot.line(dates, values, legend='{} files'.format(area), line_color=color)
-            plot.circle(dates, values, color=color)
-
-    session.close()
-
-    return plot
-
-
-def plot_filesystem_stats():
-    """
-    Plot various filesystem statistics using ``bokeh`` and save them to
-    the output directory.
-    """
-    logging.info('Creating results plots')
-
-    p1 = plot_total_file_counts()
-    p2 = plot_filesystem_size()
-    p3 = plot_by_filetype('count', 'all')
-    p4 = plot_by_filetype('size', 'all')
-    p5 = plot_central_store_dirs()
-    plot_list = [p1, p2, p3, p4, p5]
-
-    for instrument in JWST_INSTRUMENT_NAMES:
-        plot_list.append(plot_by_filetype('count', instrument))
-        plot_list.append(plot_by_filetype('size', instrument))
-
-    # Create a layout with a grid pattern
-    grid_chunks = [plot_list[i:i + 2] for i in range(0, len(plot_list), 2)]
-    grid = gridplot(grid_chunks)
-
-    # Save all of the plots in one file
-    outputs_dir = os.path.join(OUTPUTS, 'monitor_filesystem')
-    outfile = os.path.join(outputs_dir, 'filesystem_monitor.html')
-    output_file(outfile)
-    save(grid)
-    set_permissions(outfile)
-    logging.info('\tSaved plot of all statistics to {}'.format(outfile))
-
-    # Save each plot's components
-    for plot in plot_list:
-        plot_name = plot.title.text.lower().replace(' ', '_')
-        plot.sizing_mode = 'stretch_both'
-        script, div = components(plot)
-
-        div_outfile = os.path.join(outputs_dir, "{}_component.html".format(plot_name))
-        with open(div_outfile, 'w') as f:
-            f.write(div)
-            f.close()
-        set_permissions(div_outfile)
-
-        script_outfile = os.path.join(outputs_dir, "{}_component.js".format(plot_name))
-        with open(script_outfile, 'w') as f:
-            f.write(script)
-            f.close()
-        set_permissions(script_outfile)
-
-        logging.info('\tSaved components files: {}_component.html and {}_component.js'.format(plot_name, plot_name))
-
-
-def plot_total_file_counts():
-    """Plot total file counts versus date
-
-    Returns
-    -------
-    plot : bokeh.plotting.figure.Figure object
-        ``bokeh`` plot of total file counts versus date
-    """
-
-    # Total file counts vs. date
-    results = session.query(FilesystemGeneral.date, FilesystemGeneral.total_file_count).all()
-    dates, file_counts = zip(*results)
-    plot = figure(
-        tools='pan,box_zoom,reset,wheel_zoom,save',
-        x_axis_type='datetime',
-        title="Total File Counts",
-        x_axis_label='Date',
-        y_axis_label='Count')
-    plot.line(dates, file_counts, line_width=2, line_color='blue')
-    plot.circle(dates, file_counts, color='blue')
-
-    session.close()
-
-    return plot
 
 def update_central_store_database(central_storage_dict):
     """Updates the ``CentralStore`` database table with info on disk space

From 8b3072cbda154910a4aacaf2e8dc028bbae7831d Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 17 Apr 2023 10:16:28 -0400
Subject: [PATCH 055/256] Remove now-incorrect import statement

---
 jwql/website/apps/jwql/monitor_pages/__init__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py
index 136bd10d6..251fe3f60 100644
--- a/jwql/website/apps/jwql/monitor_pages/__init__.py
+++ b/jwql/website/apps/jwql/monitor_pages/__init__.py
@@ -1,4 +1,3 @@
 from .monitor_cosmic_rays_bokeh import CosmicRayMonitor
-from .monitor_filesystem_bokeh import MonitorFilesystem
 from .monitor_mast_bokeh import MastMonitor
 from .monitor_readnoise_bokeh import ReadnoiseMonitor

From 99924c4faf9a5d089149e2bc863545be4bb26963 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 10:40:38 -0400
Subject: [PATCH 056/256] move build configuration from `setup.py` to
 `pyproject.toml`

---
 pyproject.toml | 69 ++++++++++++++++++++++++++++++++++++++++++++++----
 setup.py       | 58 ------------------------------------------
 2 files changed, 64 insertions(+), 63 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index cffa0df75..e9e48654e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,65 @@
-# pyproject.toml
+[project]
+name = "jwql"
+version = "1.1.1"
+description = "The James Webb Space Telescope Quicklook Project"
+authors = [
+  { name = "Matthew Bourque" },
+  { name = "Lauren Chambers" },
+  { name = "Misty Cracraft" },
+  { name = "Mike Engesser" },
+  { name = "Mees Fix" },
+  { name = "Joe Filippazzo" },
+  { name = "Bryan Hilbert" },
+]
+keywords = ["astronomy", "python"]
+classifiers = ["Programming Language :: Python"]
+dependencies = [
+  "asdf",
+  "astropy",
+  "astroquery",
+  "bandit",
+  "bokeh<3",
+  "crds",
+  "cryptography",
+  "django",
+  "flake8",
+  "inflection",
+  "ipython",
+  "jinja2",
+  "jsonschema",
+  "jwst",
+  "jwst_reffiles",
+  "matplotlib",
+  "nodejs",
+  "numpy",
+  "numpydoc",
+  "pandas",
+  "psycopg2",
+  "pysiaf",
+  "pytest",
+  "pytest-cov",
+  "pytest-mock",
+  "pyvo",
+  "scipy",
+  "sphinx",
+  "sphinx_rtd_theme",
+  "sqlalchemy",
+  "stdatamodels",
+  "stsci_rtd_theme",
+  "twine",
+  "wtforms",
+]
+
+[project.license]
+file = "LICENSE"
+content-type = "text/plain"
+
 [build-system]
-requires = ["setuptools<=65.5.0",
-			"numpy",
-			"wheel",
-			"setuptools_scm"]
\ No newline at end of file
+requires = ["setuptools>=61.2,<=65.5.0", "numpy", "wheel", "setuptools_scm"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+namespaces = false
diff --git a/setup.py b/setup.py
index 6d30c3958..2101b2a04 100644
--- a/setup.py
+++ b/setup.py
@@ -1,64 +1,6 @@
 import numpy as np
 from setuptools import setup
-from setuptools import find_packages
-
-VERSION = '1.1.1'
-
-AUTHORS = 'Matthew Bourque, Lauren Chambers, Misty Cracraft, Mike Engesser, Mees Fix, Joe Filippazzo, Bryan Hilbert, '
-AUTHORS += 'Graham Kanarek, Teagan King, Catherine Martlin, Shannon Osborne, Maria Pena-Guerrero, Johannes Sahlmann, '
-AUTHORS += 'Ben Sunnquist, Brian York'
-
-DESCRIPTION = 'The James Webb Space Telescope Quicklook Project'
-
-REQUIRES = [
-    'asdf',
-    'astropy',
-    'astroquery',
-    'bandit',
-    'bokeh<3',
-    'crds',
-    'cryptography',
-    'django',
-    'flake8',
-    'inflection',
-    'ipython',
-    'jinja2',
-    'jsonschema',
-    'jwst',
-    'jwst_reffiles',
-    'matplotlib',
-    'nodejs',
-    'numpy',
-    'numpydoc',
-    'pandas',
-    'psycopg2',
-    'pysiaf',
-    'pytest',
-    'pytest-cov',
-    'pytest-mock',
-    'pyvo',
-    'scipy',
-    'sphinx',
-    'sphinx_rtd_theme',
-    'sqlalchemy',
-    'stdatamodels',
-    'stsci_rtd_theme',
-    'twine',
-    'wtforms'
-]
 
 setup(
-    name='jwql',
-    version=VERSION,
-    description=DESCRIPTION,
-    url='https://github.com/spacetelescope/jwql.git',
-    author=AUTHORS,
-    author_email='jwql@stsci.edu',
-    license='BSD',
-    keywords=['astronomy', 'python'],
-    classifiers=['Programming Language :: Python'],
-    packages=find_packages(),
-    install_requires=REQUIRES,
-    include_package_data=True,
     include_dirs=[np.get_include()],
 )

From fcc01bcc00bb41ca9ad3218f45b55996f835048e Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 10:51:51 -0400
Subject: [PATCH 057/256] replace references to setup.py

---
 .github/workflows/ci.yml | 2 +-
 MANIFEST.in              | 2 +-
 README.md                | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 88f823818..c23a8a201 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -56,7 +56,7 @@ jobs:
           echo " "
           echo "Installing jwql package"
           echo " "
-          python setup.py develop
+          pip install -e .
 
           echo " "
           echo "Testing package installation"
diff --git a/MANIFEST.in b/MANIFEST.in
index 478c9f4d7..20b6bed98 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,7 +4,7 @@ include LICENSE
 include CHANGES.rst
 include environment_python_3_7.yml
 include environment_python_3_8.yml
-include setup.py
+include pyproject.toml
 include requirements.txt
 include rtd_requirements.txt
 include jwql/example_config.json
diff --git a/README.md b/README.md
index c3f041f11..addbefc19 100644
--- a/README.md
+++ b/README.md
@@ -49,7 +49,7 @@ It is highly suggested that contributors have a working installation of `anacond
 - [Miniconda](https://conda.io/miniconda.html)
 - [Anaconda](https://www.continuum.io/downloads)
 
-Requirements for contributing to the `jwql` package will be included in the `jwql` `conda` environment, which is included in our installation instructions below. Further package requirements will be provided for `jwql` by a `setup.py` script included in the repository.
+Requirements for contributing to the `jwql` package will be included in the `jwql` `conda` environment, which is included in our installation instructions below. Further package requirements will be provided for `jwql` by a `pyproject.toml` file included in the repository.
 
 ### Clone the `jwql` repo
 

From 5a2d5c03053d92dab38013bc530b44bc3c98a83f Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 10:52:02 -0400
Subject: [PATCH 058/256] read `pyproject.toml` file

---
 jwql/utils/logging_functions.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py
index 3cbadd35f..af2e98fe7 100644
--- a/jwql/utils/logging_functions.py
+++ b/jwql/utils/logging_functions.py
@@ -65,6 +65,11 @@ def my_main_function():
 import time
 import traceback
 
+if sys.version_info < (3, 11):
+      import tomli as tomllib
+ else:
+      import tomllib
+
 from functools import wraps
 
 from jwql.utils.permissions import set_permissions
@@ -218,15 +223,10 @@ def wrapped(*args, **kwargs):
         logging.info('Running as PID {}'.format(os.getpid()))
 
         # Read in setup.py file to build list of required modules
-        with open(get_config()['setup_file']) as f:
-            data = f.readlines()
-
-        for i, line in enumerate(data):
-            if 'REQUIRES = [' in line:
-                begin = i + 1
-            elif 'setup(' in line:
-                end = i - 2
-        required_modules = data[begin:end]
+        with open("pyproject.toml", "rb") as f:
+            data = tomllib.load(f)
+
+        required_modules = data['project']['dependencies']
 
         # Clean up the module list
         module_list = [item.strip().replace("'", "").replace(",", "").split("=")[0].split(">")[0].split("<")[0] for item in required_modules]

From 2e0a50322469c164ce2224071f5bb3c4cc4ba713 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 10:53:53 -0400
Subject: [PATCH 059/256] use `setuptools_scm` to retrieve version number from
 Git tag

---
 pyproject.toml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index e9e48654e..d21dadcf9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,5 @@
 [project]
 name = "jwql"
-version = "1.1.1"
 description = "The James Webb Space Telescope Quicklook Project"
 authors = [
   { name = "Matthew Bourque" },
@@ -49,6 +48,7 @@ dependencies = [
   "twine",
   "wtforms",
 ]
+dynamic = ["version"]
 
 [project.license]
 file = "LICENSE"
@@ -63,3 +63,5 @@ include-package-data = true
 
 [tool.setuptools.packages.find]
 namespaces = false
+
+[tool.setuptools_scm]

From 04b17c49a8b95c7125d7bbd51270b7b053e06ee1 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 11:00:24 -0400
Subject: [PATCH 060/256] add README to metadata

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index d21dadcf9..d10f5c10c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,7 @@
 [project]
 name = "jwql"
 description = "The James Webb Space Telescope Quicklook Project"
+readme = "README.md"
 authors = [
   { name = "Matthew Bourque" },
   { name = "Lauren Chambers" },

From 299d25548381ca8a88d7e3132d2d8da495a60c89 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 11:05:33 -0400
Subject: [PATCH 061/256] move optional dependencies into
 `optional-dependencies`

---
 .github/workflows/ci.yml |  2 +-
 pyproject.toml           | 22 ++++++++++++----------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c23a8a201..5accc9e3f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -56,7 +56,7 @@ jobs:
           echo " "
           echo "Installing jwql package"
           echo " "
-          pip install -e .
+          pip install -e .[test]
 
           echo " "
           echo "Testing package installation"
diff --git a/pyproject.toml b/pyproject.toml
index d10f5c10c..738db37a1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,14 +17,11 @@ dependencies = [
   "asdf",
   "astropy",
   "astroquery",
-  "bandit",
   "bokeh<3",
   "crds",
   "cryptography",
   "django",
-  "flake8",
   "inflection",
-  "ipython",
   "jinja2",
   "jsonschema",
   "jwst",
@@ -36,21 +33,26 @@ dependencies = [
   "pandas",
   "psycopg2",
   "pysiaf",
-  "pytest",
-  "pytest-cov",
-  "pytest-mock",
   "pyvo",
   "scipy",
-  "sphinx",
-  "sphinx_rtd_theme",
   "sqlalchemy",
   "stdatamodels",
-  "stsci_rtd_theme",
-  "twine",
   "wtforms",
 ]
 dynamic = ["version"]
 
+[project.optional-dependencies]
+test = [
+  "pytest",
+  "pytest-cov",
+  "pytest-mock",
+]  
+docs = [
+  "sphinx",
+  "sphinx_rtd_theme",
+  "stsci_rtd_theme",
+]
+
 [project.license]
 file = "LICENSE"
 content-type = "text/plain"

From 54ebc8dfbfe5f58c94f2de6f827fc52d644d5703 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 10:35:39 -0400
Subject: [PATCH 062/256] drop support for Python 3.8

---
 .github/workflows/ci.yml          |  2 +-
 README.md                         |  4 +-
 environment_python_3_8.yml        | 73 -------------------------------
 jwql/tests/test_pipeline_tools.py |  2 -
 4 files changed, 3 insertions(+), 78 deletions(-)
 delete mode 100644 environment_python_3_8.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 88f823818..8a21b84ca 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,7 +12,7 @@ jobs:
       max-parallel: 5
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.8, 3.9]
+        python-version: [3.9, "3.10"]
 
     steps:
       - uses: actions/checkout@v3
diff --git a/README.md b/README.md
index c3f041f11..32ab7708f 100644
--- a/README.md
+++ b/README.md
@@ -44,7 +44,7 @@ Getting `jwql` up and running on your own computer requires four steps, detailed
 
 ### Prerequisites
 
-It is highly suggested that contributors have a working installation of `anaconda` or `miniconda` for Python 3.8.  Downloads and installation instructions are  available here:
+It is highly suggested that contributors have a working installation of `anaconda` or `miniconda` for Python 3.9+.  Downloads and installation instructions are  available here:
 
 - [Miniconda](https://conda.io/miniconda.html)
 - [Anaconda](https://www.continuum.io/downloads)
@@ -83,7 +83,7 @@ source activate base/root
 
 **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step.
 
-Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_8.yml`, for python 3.8, and `environment_python_3.9.yml`, for python 3.9, are supported by `jwql`):
+Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
 
 ```
 conda env create -f environment_python_3_8.yml
diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml
deleted file mode 100644
index df3d3a173..000000000
--- a/environment_python_3_8.yml
+++ /dev/null
@@ -1,73 +0,0 @@
-# This file describes a conda environment that can be to install jwql
-#
-# Run the following command to set up this environment:
-# $ conda env create -f environment_python_3_8.yml
-#
-# The environment name can be overridden with the following command:
-# $ conda env create -n <custom name> -f environment_python_3_8.yml
-#
-# Run the following command to activate the environment:
-# $ source activate jwql-3.8
-#
-# To deactivate the environment run the following command:
-# $ source deactivate
-#
-# To remove the environment entirely, run the following command:
-# $ conda env remove -n jwql-3.8
-
-name: jwql-3.8
-
-channels:
-  - conda-forge
-  - defaults
-
-dependencies:
-  - astropy=5.2.1
-  - bokeh=2.4.3
-  - beautifulsoup4=4.11.2
-  - celery=5.2.7
-  - cryptography=39.0.1
-  - django=4.1.7
-  - flake8=6.0.0
-  - inflection=0.5.1
-  - ipython=8.10.0
-  - jinja2=3.1.2
-  - jsonschema=4.17.3
-  - matplotlib=3.7.0
-  - nodejs=18.12.1
-  - numpy=1.24.2
-  - numpydoc=1.5.0
-  - pandas=2.0.0
-  - pip=23.0
-  - postgresql=15.2
-  - psycopg2=2.9.3
-  - pytest=7.2.1
-  - pytest-cov=4.0.0
-  - pytest-mock=3.10.0
-  - python=3.8.16
-  - pyyaml=6.0
-  - redis
-  - scipy=1.9.3
-  - setuptools=67.3.1
-  - sphinx=6.1.3
-  - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.8
-  - twine=4.0.2
-  - wtforms=3.0.1
-
-  - pip:
-    - astroquery==0.4.6
-    - bandit==1.7.4
-    - jwst==1.10.0
-    - pysiaf==0.19.1
-    - pysqlite3==0.5.0
-    - pyvo==1.4
-    - redis==4.5.1
-    - selenium==4.8.0
-    - stdatamodels==1.3.1
-    - stsci_rtd_theme==1.0.0
-    - vine==5.0.0
-    - git+https://github.com/spacetelescope/jwst_reffiles
-
-    # Current package
-    - -e .
diff --git a/jwql/tests/test_pipeline_tools.py b/jwql/tests/test_pipeline_tools.py
index d28b19aea..841cdb6d4 100644
--- a/jwql/tests/test_pipeline_tools.py
+++ b/jwql/tests/test_pipeline_tools.py
@@ -67,7 +67,6 @@ def test_completed_pipeline_steps():
     assert completed_steps == true_completed
 
 
-@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Drizzle import issue with python 3.8')
 def test_get_pipeline_steps():
     """Test that the proper pipeline steps are returned for an
     instrument
@@ -139,7 +138,6 @@ def test_image_stack():
     assert exptimes == [[10.5], [10.5], [10.5]]
 
 
-@pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Drizzle import issue with python 3.8')
 def test_steps_to_run():
     """Test that the dictionaries for steps required and steps completed
     are correctly combined to create a dictionary of pipeline steps to

From f26a122419fa754466da2d8fffd3556a8e6878c3 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 07:45:37 -0400
Subject: [PATCH 063/256] run all jobs to completion

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8a21b84ca..060c23a73 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -10,6 +10,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       max-parallel: 5
+      fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
         python-version: [3.9, "3.10"]

From ca455d230ff53563256f213092d4f518ae457e5a Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 07:57:33 -0400
Subject: [PATCH 064/256] remove 3.10

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 060c23a73..065a7654b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.9, "3.10"]
+        python-version: [3.9]
 
     steps:
       - uses: actions/checkout@v3

From 7f39ed03c6c8099c0ef3ad5436299e372891004d Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 10:02:32 -0400
Subject: [PATCH 065/256] use micromamba to build environment and split job

---
 .github/workflows/ci.yml | 84 ++++++++++++++--------------------------
 1 file changed, 30 insertions(+), 54 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 88f823818..e87a0ae83 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,10 +2,24 @@ name: JWQL CI
 
 on: [push, pull_request]
 
-jobs:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
-  JWQL-CI:
+defaults:
+  run:
+    shell: bash -l {0}
 
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+      - run: pip install bandit
+      - run: bandit ./jwql/ -c .bandit
+
+  test:
     name: Python - ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
     strategy:
@@ -17,63 +31,25 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          activate-environment: jwql-${{ matrix.python-version }}
+      - run: |
+            echo "major=$(echo ${{ matrix.python-version }} | head -c 1)" >> $GITHUB_OUTPUT
+            echo "minor=$(echo ${{ matrix.python-version }} | tail -c 2)" >> $GITHUB_OUTPUT
+        id: python
 
-      - name: Bandit Check
-        uses: jpetrucciani/bandit-check@master
+      - uses: mamba-org/provision-with-micromamba@v15
         with:
-          path: "./jwql/"
-          bandit_flags: "-c .bandit"
-        continue-on-error: false
-        if: runner.os == 'Linux'
+          environment-file: ./environment_python_${{ steps.python.outputs.major }}_${{ steps.python.outputs.minor }}.yml
+          environment-name: jwql-${{ matrix.python-version }}
 
-      - name: Start Redis
-        uses: supercharge/redis-github-action@1.4.0
+      - run: pip install -e .
+
+      - run: conda env export
+      
+      - uses: supercharge/redis-github-action@1.4.0
         with:
           redis-version: 5.0
         if: runner.os == 'Linux'
 
-      - name: Build jwql conda environment and run tests
-        shell: bash -l {0}
-        run: |
-
-          echo "Setting useful environment variables:"
-          echo " "
-          export PYTHONVERSION=${{ matrix.python-version }}
-          export MAJOR="$(echo $PYTHONVERSION | head -c 1)"
-          export MINOR="$(echo $PYTHONVERSION | tail -c 2)"
-          echo "PYTHONVERSION: $PYTHONVERSION"
-
-          echo " "
-          echo "Installing jwql conda environment"
-          echo " "
-          cd $RUNNER_WORKSPACE/
-          cd jwql/
-          $CONDA/bin/conda env update -f environment_python_${MAJOR}_${MINOR}.yml -n jwql-${{ matrix.python-version }}
-
-          echo " "
-          echo "Installing jwql package"
-          echo " "
-          python setup.py develop
-
-          echo " "
-          echo "Testing package installation"
-          echo " "
-          python -c "import jwql; print('Version: ' + jwql.__version__); print('Path: ' + jwql.__path__[0])"
-
-          # echo " "
-          # echo "Reinstall numpy to fix numpy.core.multiarray error"
-          # echo " "
-          pip install -U numpy
-
-          echo " "
-          echo "The conda environment being used:"
-          echo " "
-          $CONDA/bin/conda env export
+      - run: python -c "import jwql; print('Version ' + jwql.__version__); print('Path ' + jwql.__path__[0])"
 
-          echo " "
-          echo "Running pytests"
-          echo " "
-          pytest jwql/tests/
+      - run: pytest jwql/tests/

From 08dbba8bf929599d9c0368e02c3a896516107d92 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 10:31:14 -0400
Subject: [PATCH 066/256] cache environment

---
 .github/workflows/ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e87a0ae83..55150f2ce 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,8 +25,8 @@ jobs:
     strategy:
       max-parallel: 5
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: [3.8, 3.9]
+        os: [ ubuntu-latest, macos-latest ]
+        python-version: [ 3.8, 3.9 ]
 
     steps:
       - uses: actions/checkout@v3
@@ -40,6 +40,8 @@ jobs:
         with:
           environment-file: ./environment_python_${{ steps.python.outputs.major }}_${{ steps.python.outputs.minor }}.yml
           environment-name: jwql-${{ matrix.python-version }}
+          cache-env: true
+          cache-downloads: true
 
       - run: pip install -e .
 

From a3ad305feb7c7fcd466f6196a09a69431501443c Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Mon, 17 Apr 2023 10:56:36 -0400
Subject: [PATCH 067/256] set job name

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 55150f2ce..660f4207f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
       - run: bandit ./jwql/ -c .bandit
 
   test:
-    name: Python - ${{ matrix.python-version }}
+    name: test (Python ${{ matrix.python-version }}, ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       max-parallel: 5

From 350a57548d19680183f1c09927aec5f2a7df1a23 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 07:59:06 -0400
Subject: [PATCH 068/256] do not fail fast on error

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 660f4207f..5f062ce01 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -24,6 +24,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       max-parallel: 5
+      fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest ]
         python-version: [ 3.8, 3.9 ]

From e320a3e76eec09db896f805e3d44b6b9a240a4e2 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 09:01:36 -0400
Subject: [PATCH 069/256] drop explicit environment name

---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6af15f405..bd648fe32 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -40,7 +40,6 @@ jobs:
       - uses: mamba-org/provision-with-micromamba@v15
         with:
           environment-file: ./environment_python_${{ steps.python.outputs.major }}_${{ steps.python.outputs.minor }}.yml
-          environment-name: jwql-${{ matrix.python-version }}
           cache-env: true
           cache-downloads: true
 

From 48f3af8c94d808ffe76b923261a7c5853907bdbb Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 09:43:23 -0400
Subject: [PATCH 070/256] also use pytest configuration

---
 pyproject.toml | 8 ++++++--
 pytest.ini     | 3 ---
 2 files changed, 6 insertions(+), 5 deletions(-)
 delete mode 100644 pytest.ini

diff --git a/pyproject.toml b/pyproject.toml
index 738db37a1..70b8ad6f9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,13 +25,13 @@ dependencies = [
   "jinja2",
   "jsonschema",
   "jwst",
-  "jwst_reffiles",
+  "jwst_reffiles @ git+https://github.com/spacetelescope/jwst_reffiles.git@master",
   "matplotlib",
   "nodejs",
   "numpy",
   "numpydoc",
   "pandas",
-  "psycopg2",
+  "psycopg2-binary",
   "pysiaf",
   "pyvo",
   "scipy",
@@ -68,3 +68,7 @@ include-package-data = true
 namespaces = false
 
 [tool.setuptools_scm]
+
+[tool.pytest]
+junit_family = "xunit2"
+addopts = "--ignore=jwql/website/apps/jwql/static"
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index 16d36fb6b..000000000
--- a/pytest.ini
+++ /dev/null
@@ -1,3 +0,0 @@
-[pytest]
-junit_family = xunit2
-addopts = --ignore=jwql/website/apps/jwql/static

From dee2b329ab7c9ce0bfac8e09bfe0e11c20a4354e Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 09:50:13 -0400
Subject: [PATCH 071/256] fix import indentation

---
 jwql/utils/logging_functions.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py
index af2e98fe7..ab4d93095 100644
--- a/jwql/utils/logging_functions.py
+++ b/jwql/utils/logging_functions.py
@@ -66,9 +66,9 @@ def my_main_function():
 import traceback
 
 if sys.version_info < (3, 11):
-      import tomli as tomllib
- else:
-      import tomllib
+    import tomli as tomllib
+else:
+    import tomllib
 
 from functools import wraps
 

From 81e782ed3441c0200b3df871aa9ff859ac2dba52 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 09:57:49 -0400
Subject: [PATCH 072/256] restore dependency for egg build

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 70b8ad6f9..d941dbc56 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ dependencies = [
   "jinja2",
   "jsonschema",
   "jwst",
-  "jwst_reffiles @ git+https://github.com/spacetelescope/jwst_reffiles.git@master",
+  "jwst_reffiles",
   "matplotlib",
   "nodejs",
   "numpy",

From 7df644e728883e62f1a406a33d3ef123c2474329 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 10:24:41 -0400
Subject: [PATCH 073/256] use latest Python instead of system

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bd648fe32..be16d2e8e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,6 +16,8 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
       - run: pip install bandit
       - run: bandit ./jwql/ -c .bandit
 

From 258b9a87d95426ab3a9ae246c3a5523dfbec1308 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 18 Apr 2023 10:42:24 -0400
Subject: [PATCH 074/256] use test extra

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index be16d2e8e..9c8792715 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -45,7 +45,7 @@ jobs:
           cache-env: true
           cache-downloads: true
 
-      - run: pip install -e .
+      - run: pip install -e .[test]
 
       - run: conda env export
       

From 2236cd6ab0d00d950eb452bc34bae50ff26fd27a Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 18 Apr 2023 15:34:05 -0400
Subject: [PATCH 075/256] fix database typo

---
 jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py
index e601127fd..d57a83c4b 100755
--- a/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_dark_bokeh.py
@@ -447,7 +447,7 @@ def __init__(self, instrument):
         self.db = DarkMonitorData(self.instrument)
 
         # Now we need to loop over the available apertures and create plots for each
-        self.available_apertures = get_unique_values_per_column(self.db, 'aperture')
+        self.available_apertures = get_unique_values_per_column(self.db.stats_table, 'aperture')
 
         # Require entries for all full frame apertures. If there are no data for a
         # particular full frame entry, then produce an empty plot, in order to
@@ -651,8 +651,10 @@ def create_plot(self):
             # for each amp, we can get away with using use_amp=1 at the moment.
             if '5' in self.mean_dark:
                 use_amp = '5'
+                legend_label = 'Full aperture'
             else:
                 use_amp = '1'
+                legend_label = 'Amp 1'
 
             # If there are trending data for multiple amps, then we can plot each
             if len(self.mean_dark) > 1:
@@ -682,7 +684,7 @@ def create_plot(self):
                                background_fill_color="#fafafa")
 
             # Plot the "main" amp data along with error bars
-            self.plot.scatter(x='time', y='mean_dark', fill_color="navy", alpha=0.75, source=source)
+            self.plot.scatter(x='time', y='mean_dark', fill_color="navy", alpha=0.75, source=source, legend_label=legend_label)
             self.plot.add_layout(Whisker(source=source, base="time", upper="error_upper", lower="error_lower", line_color='navy'))
             hover_tool = HoverTool(tooltips=[('Dark rate:', '@mean_dark'),
                                              ('Date:', '@time{%d %b %Y}')

From 17037b034d211244b40eed2fbc0690529c6012d2 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 19 Apr 2023 12:39:03 -0400
Subject: [PATCH 076/256] Remove bokeh templating files for mast monitor

---
 .../apps/jwql/monitor_pages/__init__.py       |  1 -
 .../jwql/monitor_pages/monitor_mast_bokeh.py  | 92 -------------------
 .../yaml/monitor_mast_interface.yaml          | 64 -------------
 3 files changed, 157 deletions(-)
 delete mode 100644 jwql/website/apps/jwql/monitor_pages/monitor_mast_bokeh.py
 delete mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/monitor_mast_interface.yaml

diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py
index 251fe3f60..1405523c0 100644
--- a/jwql/website/apps/jwql/monitor_pages/__init__.py
+++ b/jwql/website/apps/jwql/monitor_pages/__init__.py
@@ -1,3 +1,2 @@
 from .monitor_cosmic_rays_bokeh import CosmicRayMonitor
-from .monitor_mast_bokeh import MastMonitor
 from .monitor_readnoise_bokeh import ReadnoiseMonitor
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_mast_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_mast_bokeh.py
deleted file mode 100644
index 4276276dd..000000000
--- a/jwql/website/apps/jwql/monitor_pages/monitor_mast_bokeh.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Tue Feb  5 15:19:20 2019
-
-@author: gkanarek
-"""
-
-import os
-
-from astropy.time import Time
-import pandas as pd
-
-from jwql.bokeh_templating import BokehTemplate
-from jwql.utils.utils import get_config
-
-SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
-class MastMonitor(BokehTemplate):
-
-    def pre_init(self):
-        self._embed = True
-
-        # App design
-        self.format_string = None
-        self.interface_file = os.path.join(SCRIPT_DIR, 'yaml', "monitor_mast_interface.yaml")
-
-        self.settings = get_config()
-        self.output_dir = self.settings['outputs']
-
-        self.read_new_data()
-
-        self.cache_time = Time(0., format='unix')
-
-        self.jwst_bar_colors = self.caom_bar_colors = 3
-        self.jwst_datacols = []
-        self.caom_datacols = []
-
-    def post_init(self):
-        self.update_plots()
-
-    def read_new_data(self):
-        """
-        Placeholder to read what are currently Pandas dataframe dumps. Replace
-        this when we have a new database infrastructure.
-        """
-
-        jwst_filepath = os.path.join(self.outputs_dir, 'database_monitor_jwst.json')
-        caom_filepath = os.path.join(self.outputs_dir, 'database_monitor_caom.json')
-
-        jwst_modtime = Time(os.stat(jwst_filepath).st_mtime, format='unix')
-        caom_modtime = Time(os.stat(caom_filepath).st_mtime, format='unix')
-
-        if jwst_modtime >= self.cache_time:
-            self.jwst_df = pd.read_json(jwst_filepath, orient='records')
-        if caom_modtime >= self.cache_time:
-            self.caom_df = pd.read_json(caom_filepath, orient='records')
-
-        self.cache_time = Time.now()
-
-    def update_plots(self):
-        """
-        Update the various sources and variables for the MAST monitor bar charts.
-        """
-
-        self.read_new_data()
-
-        jwst_groups = list(self.jwst_df['instrument'])
-        caom_groups = list(self.caom_df['instrument'])
-
-        self.jwst_datacols = [col for col in list(self.jwst_df.columns) if col != 'instrument']
-        self.caom_datacols = [col for col in list(self.caom_df.columns) if col != 'instrument']
-
-        jwst_data = {'groups': jwst_groups}
-        caom_data = {'groups': caom_groups}
-
-        for col in self.jwst_datacols:
-            jwst_data.update({col: list(self.jwst_df[col])})
-        for col in self.caom_datacols:
-            caom_data.update({col: list(self.caom_df[col])})
-
-        self.jwst_bar_colors = max(3, len(self.jwst_datacols))
-        self.caom_bar_colors = max(3, len(self.caom_datacols))
-
-        jwst_x = [(group, datacol) for group in jwst_groups for datacol in self.jwst_datacols]
-        jwst_counts = sum(zip(*[jwst_data[col] for col in self.jwst_datacols]), ())
-        caom_x = [(group, datacol) for group in caom_groups for datacol in self.caom_datacols]
-        caom_counts = sum(zip(*[caom_data[col] for col in self.caom_datacols]), ())
-
-        self.refs['jwst_source'].data = {'x': jwst_x, 'counts': jwst_counts}
-        self.refs['caom_source'].data = {'x': caom_x, 'counts': caom_counts}
diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_mast_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_mast_interface.yaml
deleted file mode 100644
index 810984eaa..000000000
--- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_mast_interface.yaml
+++ /dev/null
@@ -1,64 +0,0 @@
-# JWST figure
-- !ColumnDataSource: &jwst_source
-    ref: "jwst_source"
-    data:
-        x: []
-        counts: []
-- !HoverTool: &jwst_hover
-    ref: "jwst_hover"
-    tooltips: [('count', '@counts')]
-- !FactorRange: &jwst_xrange
-    ref: "jwst_xrange"
-    factors: !self.jwst_groups
-- !factor_cmap: &jwst_cmap
-    ref: "jwst_cmap"
-    arg: 'x'
-    palette: !Category20c[!self.jwst_bar_colors ]
-    factors: !self.jwst_datacols
-    start: 1
-    end: 2
-- !Figure: &jwst_figure
-    ref: "jwst_figure"
-    x_range: *jwst_xrange
-    plot_height: 250
-    tools: [*jwst_hover ]
-    elements:
-        - kind: 'vbar'
-          x: 'x'
-          top: 'counts'
-          width: 0.9
-          source: *jwst_source
-          line_color: 'white'
-          fill_color: *jwst_cmap
-# CAOM figure
-- !ColumnDataSource: &caom_source
-    ref: "caom_source"
-    data:
-        x: []
-        counts: []
-- !HoverTool: &caom_hover
-    ref: "caom_hover"
-    tooltips: [('count', '@counts')]
-- !FactorRange: &caom_xrange
-    ref: "caom_xrange"
-    factors: !self.caom_groups
-- !factor_cmap: &caom_cmap
-    ref: "caom_cmap"
-    arg: 'x'
-    palette: !Category20c[!self.caom_bar_colors ]
-    factors: !self.caom_datacols
-    start: 1
-    end: 2
-- !Figure: &caom_figure
-    ref: "caom_figure"
-    x_range: *caom_xrange
-    plot_height: 250
-    tools: [*caom_hover ]
-    elements:
-        - kind: 'vbar'
-          x: 'x'
-          top: 'counts'
-          width: 0.9
-          source: *caom_source
-          line_color: 'white'
-          fill_color: *caom_cmap
\ No newline at end of file

From 253cf114c627254978e6f880e6d46358d5cf99ec Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 19 Apr 2023 12:40:09 -0400
Subject: [PATCH 077/256] Remove bokeh templating yaml for filesystem monitor

---
 .../yaml/monitor_filesystem_interface.yaml    | 155 ------------------
 1 file changed, 155 deletions(-)
 delete mode 100644 jwql/website/apps/jwql/monitor_pages/yaml/monitor_filesystem_interface.yaml

diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_filesystem_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/monitor_filesystem_interface.yaml
deleted file mode 100644
index a1da194e9..000000000
--- a/jwql/website/apps/jwql/monitor_pages/yaml/monitor_filesystem_interface.yaml
+++ /dev/null
@@ -1,155 +0,0 @@
-# Unified x range for all files
-- !Range1d: &xr_all
-    ref: "xr_all"
-    start: 0
-    end: 100
-    bounds: !!python/tuple [0, 100]
-#File count figure
-- !ColumnDataSource: &source_filecount
-    ref: "source_filecount"
-    data:
-        dates: []
-        filecount: []
-- !Range1d: &yr_filecount
-    ref: "yr_filecount"
-    start: 0
-    end: 100
-    bounds: !!python/tuple [0, 100]
-- !Figure: &fig_filecount
-    ref: "fig_filecount"
-    title: "Total File Counts"
-    y_axis_label: 'Count'
-    x_range: *xr_all
-    y_range: *yr_filecount
-    elements:
-        - {'kind': 'line', 'x': 'dates', 'y': 'filecount', 'line_color': 'blue', 'source': *source_filecount}
-#System stats figure
-- !ColumnDataSource: &source_stats
-    ref: "source_stats"
-    data:
-        dates: []
-        systemsize: []
-        freesize: []
-        usedsize: []
-- !Range1d: &yr_stats
-    ref: "yr_stats"
-    start: 0
-    end: 100
-    bounds: !!python/tuple [0, 100]
-- !Figure: &fig_stats
-    ref: "fig_system_stats"
-    title: "System Stats"
-    y_axis_label: 'Count'
-    x_range: *xr_all
-    y_range: *yr_stats
-    elements:
-        - {'kind': 'line', 'x': 'dates', 'y': 'systemsize', 'line_color': 'red', 'source': *source_stats, 'legend': 'Total size'}
-        - {'kind': 'line', 'x': 'dates', 'y': 'freesize', 'line_color': 'blue', 'source': *source_stats, 'legend': 'Free bytes'}
-        - {'kind': 'line', 'x': 'dates', 'y': 'usedsize', 'line_color': 'green', 'source': *source_stats, 'legend': 'Used bytes'}
-        - {'kind': 'circle', 'x': 'dates', 'y': 'systemsize', 'color': 'red', 'source': *source_stats}
-        - {'kind': 'circle', 'x': 'dates', 'y': 'freesize', 'color': 'blue', 'source': *source_stats}
-        - {'kind': 'circle', 'x': 'dates', 'y': 'usedsize', 'color': 'green', 'source': *source_stats}
-#File types figure
-- !ColumnDataSource: &source_files
-    ref: "source_files"
-    data:
-        dates: []
-        fits: []
-        uncal: []
-        cal: []
-        rate: []
-        rateint: []
-        i2d: []
-        nrc: []
-        nrs: []
-        nis: []
-        mir: []
-        fgs: []
-- !Range1d: &yr_files
-    ref: "yr_files"
-    start: 0
-    end: 100
-    bounds: !!python/tuple [0, 100]
-- !Figure: &fig_files
-    ref: "fig_filecount_type"
-    title: "Total File Counts by Type"
-    y_axis_label: 'Count'
-    x_range: *xr_all
-    y_range: *yr_files
-    elements:
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[0] , 'line_color': !self.types_c[0] , 'source': *source_files, 'legend': !self.types_l[0] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[1] , 'line_color': !self.types_c[1] , 'source': *source_files, 'legend': !self.types_l[1] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[2] , 'line_color': !self.types_c[2] , 'source': *source_files, 'legend': !self.types_l[2] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[3] , 'line_color': !self.types_c[3] , 'source': *source_files, 'legend': !self.types_l[3] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[4] , 'line_color': !self.types_c[4] , 'source': *source_files, 'legend': !self.types_l[4] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[5] , 'line_color': !self.types_c[5] , 'source': *source_files, 'legend': !self.types_l[5] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[6] , 'line_color': !self.types_c[6] , 'source': *source_files, 'legend': !self.types_l[6] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[7] , 'line_color': !self.types_c[7] , 'source': *source_files, 'legend': !self.types_l[7] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[8] , 'line_color': !self.types_c[8] , 'source': *source_files, 'legend': !self.types_l[8] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[9] , 'line_color': !self.types_c[9] , 'source': *source_files, 'legend': !self.types_l[9] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[10] , 'line_color': !self.types_c[10] , 'source': *source_files, 'legend': !self.types_l[10] }
-        - {'kind': !self.types_k[0] , 'x': 'dates', 'y': !self.types_y[0] , 'color': !self.types_c[0] , 'source': *source_files}
-        - {'kind': !self.types_k[1] , 'x': 'dates', 'y': !self.types_y[1] , 'color': !self.types_c[1] , 'source': *source_files}
-        - {'kind': !self.types_k[2] , 'x': 'dates', 'y': !self.types_y[2] , 'color': !self.types_c[2] , 'source': *source_files}
-        - {'kind': !self.types_k[3] , 'x': 'dates', 'y': !self.types_y[3] , 'color': !self.types_c[3] , 'source': *source_files}
-        - {'kind': !self.types_k[4] , 'x': 'dates', 'y': !self.types_y[4] , 'color': !self.types_c[4] , 'source': *source_files}
-        - {'kind': !self.types_k[5] , 'x': 'dates', 'y': !self.types_y[5] , 'color': !self.types_c[5] , 'source': *source_files}
-        - {'kind': !self.types_k[6] , 'x': 'dates', 'y': !self.types_y[6] , 'color': !self.types_c[6] , 'source': *source_files}
-        - {'kind': !self.types_k[7] , 'x': 'dates', 'y': !self.types_y[7] , 'color': !self.types_c[7] , 'source': *source_files}
-        - {'kind': !self.types_k[8] , 'x': 'dates', 'y': !self.types_y[8] , 'color': !self.types_c[8] , 'source': *source_files}
-        - {'kind': !self.types_k[9] , 'x': 'dates', 'y': !self.types_y[9] , 'color': !self.types_c[9] , 'source': *source_files}
-        - {'kind': !self.types_k[10] , 'x': 'dates', 'y': !self.types_y[10] , 'color': !self.types_c[10] , 'source': *source_files}
-#File sizes figure
-- !ColumnDataSource: &source_sizes
-    ref: "source_sizes"
-    data:
-        dates: []
-        fits: []
-        uncal: []
-        cal: []
-        rate: []
-        rateint: []
-        i2d: []
-        nrc: []
-        nrs: []
-        nis: []
-        mir: []
-        fgs: []
-- !Range1d: &yr_sizes
-    ref: "yr_sizes"
-    start: 0
-    end: 100
-    bounds: !!python/tuple [0, 100]
-- !Figure: &fig_sizes
-    ref: "fig_size_type"
-    title: "Total File Sizes by Type"
-    y_axis_label: 'GB'
-    x_range: *xr_all
-    y_range: *yr_sizes
-    elements:
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[0] , 'line_color': !self.types_c[0] , 'source': *source_sizes, 'legend': !self.types_l[0] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[1] , 'line_color': !self.types_c[1] , 'source': *source_sizes, 'legend': !self.types_l[1] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[2] , 'line_color': !self.types_c[2] , 'source': *source_sizes, 'legend': !self.types_l[2] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[3] , 'line_color': !self.types_c[3] , 'source': *source_sizes, 'legend': !self.types_l[3] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[4] , 'line_color': !self.types_c[4] , 'source': *source_sizes, 'legend': !self.types_l[4] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[5] , 'line_color': !self.types_c[5] , 'source': *source_sizes, 'legend': !self.types_l[5] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[6] , 'line_color': !self.types_c[6] , 'source': *source_sizes, 'legend': !self.types_l[6] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[7] , 'line_color': !self.types_c[7] , 'source': *source_sizes, 'legend': !self.types_l[7] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[8] , 'line_color': !self.types_c[8] , 'source': *source_sizes, 'legend': !self.types_l[8] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[9] , 'line_color': !self.types_c[9] , 'source': *source_sizes, 'legend': !self.types_l[9] }
-        - {'kind': 'line', 'x': 'dates', 'y': !self.types_y[10] , 'line_color': !self.types_c[10] , 'source': *source_sizes, 'legend': !self.types_l[10] }
-        - {'kind': !self.types_k[0] , 'x': 'dates', 'y': !self.types_y[0] , 'color': !self.types_c[0] , 'source': *source_sizes}
-        - {'kind': !self.types_k[1] , 'x': 'dates', 'y': !self.types_y[1] , 'color': !self.types_c[1] , 'source': *source_sizes}
-        - {'kind': !self.types_k[2] , 'x': 'dates', 'y': !self.types_y[2] , 'color': !self.types_c[2] , 'source': *source_sizes}
-        - {'kind': !self.types_k[3] , 'x': 'dates', 'y': !self.types_y[3] , 'color': !self.types_c[3] , 'source': *source_sizes}
-        - {'kind': !self.types_k[4] , 'x': 'dates', 'y': !self.types_y[4] , 'color': !self.types_c[4] , 'source': *source_sizes}
-        - {'kind': !self.types_k[5] , 'x': 'dates', 'y': !self.types_y[5] , 'color': !self.types_c[5] , 'source': *source_sizes}
-        - {'kind': !self.types_k[6] , 'x': 'dates', 'y': !self.types_y[6] , 'color': !self.types_c[6] , 'source': *source_sizes}
-        - {'kind': !self.types_k[7] , 'x': 'dates', 'y': !self.types_y[7] , 'color': !self.types_c[7] , 'source': *source_sizes}
-        - {'kind': !self.types_k[8] , 'x': 'dates', 'y': !self.types_y[8] , 'color': !self.types_c[8] , 'source': *source_sizes}
-        - {'kind': !self.types_k[9] , 'x': 'dates', 'y': !self.types_y[9] , 'color': !self.types_c[9] , 'source': *source_sizes}
-        - {'kind': !self.types_k[10] , 'x': 'dates', 'y': !self.types_y[10] , 'color': !self.types_c[10] , 'source': *source_sizes}
-#Widget layout
-- !gridplot:
-    ref: "filesystem_layout"
-    arg: [[[*fig_filecount, *fig_stats], [*fig_files, *fig_sizes]]]
\ No newline at end of file

From 120f8126cf10483c96fd5c58c6179c81ba88b21d Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Thu, 20 Apr 2023 13:34:10 -0500
Subject: [PATCH 078/256] Renaming env files and updating ci

---
 .github/workflows/ci.yml                                 | 9 ++-------
 ...onment_python_3_10.yml => environment_python_3.10.yml | 0
 environment_python_3_9.yml => environment_python_3.9.yml | 0
 3 files changed, 2 insertions(+), 7 deletions(-)
 rename environment_python_3_10.yml => environment_python_3.10.yml (100%)
 rename environment_python_3_9.yml => environment_python_3.9.yml (100%)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9c8792715..3b047377a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -29,19 +29,14 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.9]
+        python-version: [3.9, "3.10"]
 
     steps:
       - uses: actions/checkout@v3
 
-      - run: |
-            echo "major=$(echo ${{ matrix.python-version }} | head -c 1)" >> $GITHUB_OUTPUT
-            echo "minor=$(echo ${{ matrix.python-version }} | tail -c 2)" >> $GITHUB_OUTPUT
-        id: python
-
       - uses: mamba-org/provision-with-micromamba@v15
         with:
-          environment-file: ./environment_python_${{ steps.python.outputs.major }}_${{ steps.python.outputs.minor }}.yml
+          environment-file: ./environment_python_${{ matrix.python-version }}.yml
           cache-env: true
           cache-downloads: true
 
diff --git a/environment_python_3_10.yml b/environment_python_3.10.yml
similarity index 100%
rename from environment_python_3_10.yml
rename to environment_python_3.10.yml
diff --git a/environment_python_3_9.yml b/environment_python_3.9.yml
similarity index 100%
rename from environment_python_3_9.yml
rename to environment_python_3.9.yml

From ea9846e37f6d48761427b1648c82ac7c81e0ad96 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Apr 2023 15:03:51 -0400
Subject: [PATCH 079/256] Fix path to pyproject.toml

---
 jwql/utils/logging_functions.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py
index ab4d93095..abd82a45e 100644
--- a/jwql/utils/logging_functions.py
+++ b/jwql/utils/logging_functions.py
@@ -223,7 +223,8 @@ def wrapped(*args, **kwargs):
         logging.info('Running as PID {}'.format(os.getpid()))
 
         # Read in setup.py file to build list of required modules
-        with open("pyproject.toml", "rb") as f:
+        toml_file = os.path.join(os.path.dirname(get_config()['setup_file']), 'pyproject.toml')
+        with open(toml_file, "rb") as f:
             data = tomllib.load(f)
 
         required_modules = data['project']['dependencies']

From f3c8836581a0f50645e6e8490feae20e091434f2 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 8 May 2023 12:39:53 -0400
Subject: [PATCH 080/256] correct logging statement

---
 jwql/shared_tasks/shared_tasks.py | 34 +++++++++++++++----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 516738ed0..c2f7626a1 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -231,7 +231,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
     with process.stderr:
         log_subprocess_output(process.stderr)
-    result = process.wait()    
+    result = process.wait()
     logging.info("Subprocess result was {}".format(result))
 
     if not os.path.isfile(res_file):
@@ -241,7 +241,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
             for line in status:
                 logging.error(line.strip())
             return status
-        
+
     with open(res_file, 'r') as inf:
         status = inf.readlines()
     return status
@@ -251,17 +251,17 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
 def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, step_args={}):
     """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each
     step as a separate output file, then return the name-and-path of the file as reduced
-    in the reduction directory. Once all requested extensions have been produced, the 
+    in the reduction directory. Once all requested extensions have been produced, the
     pipeline will return.
 
     Parameters
     ----------
     input_file_name : str
         File on which to run the pipeline steps
-    
+
     short_name : str
         Name of the file to be calibrated after any extensions have been stripped off.
-    
+
     ext_or_exts : list
         List of extensions to be retrieved.
 
@@ -291,7 +291,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
     output_dir = os.path.join(config['transfer_dir'], "outgoing")
     msg = "Input from {}, calibrate in {}, output to {}"
     logging.info(msg.format(input_dir, cal_dir, output_dir))
-    
+
     input_file = os.path.join(input_dir, input_file_name)
     current_dir = os.path.dirname(__file__)
     cmd_name = os.path.join(current_dir, "run_pipeline.py")
@@ -308,9 +308,9 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
         logging.info("Requesting {}".format(calibrated_files))
 
     cores = 'all'
-    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, 
+    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file,
                             short_name, result_file, cores)
-    
+
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
     else:
@@ -323,7 +323,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
             logging.error("\t{}".format(line.strip()))
         if core_fail:
             cores = "half"
-            status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, 
+            status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
                                     input_file, short_name, result_file, cores)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
@@ -337,7 +337,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                     logging.error("\t{}".format(line.strip()))
                 if core_fail:
                     cores = "none"
-                    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, 
+                    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
                                             input_file, short_name, result_file, cores)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
@@ -346,7 +346,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                         logging.error("Pipeline subprocess failed.")
         if not managed:
             raise ValueError("Pipeline Failed")
-    
+
     for file in calibrated_files:
         logging.info("Checking for output {}".format(file))
         if not os.path.isfile(os.path.join(cal_dir, file)):
@@ -420,14 +420,14 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
     short_name = input_file_name.replace("_uncal", "").replace("_0thgroup", "").replace(".fits", "")
     ensure_dir_exists(cal_dir)
     output_dir = os.path.join(config["transfer_dir"], "outgoing")
-    
+
     cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py")
     result_file = os.path.join(cal_dir, short_name+"_status.txt")
 
     cores = 'all'
-    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, 
+    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file,
                             short_name, result_file, cores)
-    
+
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
     else:
@@ -440,7 +440,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
             logging.error("\t{}".format(line.strip()))
         if core_fail:
             cores = "half"
-            status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, 
+            status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
                                     input_file, short_name, result_file, cores)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
@@ -454,7 +454,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
                     logging.error("\t{}".format(line.strip()))
                 if core_fail:
                     cores = "none"
-                    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, 
+                    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
                                             input_file, short_name, result_file, cores)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
@@ -549,7 +549,7 @@ def prep_file(input_file, in_ext):
         logging.critical(msg.format(short_name))
         raise ValueError("Redis lock for {} is in an unknown state".format(short_name))
     logging.info("\t\tAcquired Lock.")
-    logging.info("\t\tCopying {} to {}".format(input_file, send_path))
+    logging.info("\t\tCopying {} to {}".format(uncal_file, send_path))
     copy_files([uncal_file], send_path)
     return short_name, cal_lock, os.path.join(send_path, uncal_name)
 

From 90d79e38e7882ae40f3eb3e529d849790ad5ea16 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 8 May 2023 15:20:53 -0400
Subject: [PATCH 081/256] update run_save_jump

---
 jwql/shared_tasks/run_pipeline.py | 57 +++++++++++++++----------------
 jwql/shared_tasks/shared_tasks.py | 38 ++++++++++-----------
 2 files changed, 47 insertions(+), 48 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 9a196ecdd..124f51a15 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -43,17 +43,17 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
     status_file_name = short_name + "_status.txt"
     status_file = os.path.join(work_directory, status_file_name)
     uncal_file = os.path.join(work_directory, input_file_basename)
-    
+
     with open(status_file, 'a+') as status_f:
         status_f.write("Running run_pipe\n")
         status_f.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename)))
         status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir)))
         status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file)))
-    
+
     try:
         copy_files([input_file], work_directory)
         set_permissions(uncal_file)
-    
+
         steps = get_pipeline_steps(instrument)
         first_step_to_be_run = True
         for step_name in steps:
@@ -94,7 +94,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
                             # If the dither_points entry is not populated, then ignore this change
                             pass
                         model[0].save(output_file)
-                
+
                     done = True
                     for output in outputs:
                         output_name = "{}_{}.fits".format(short_name, output)
@@ -110,7 +110,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
             status_f.write("{}\n".format(e))
             status_f.write("FAILED")
         sys.exit(1)
-    
+
     with open(status_file, "a+") as status_f:
         status_f.write("SUCCEEDED")
     # Done.
@@ -130,7 +130,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     sys.stderr.write("Starting pipeline\n")
     with open(status_file, 'a+') as status_f:
         status_f.write("Starting pipeline\n")
-    
+
     try:
         copy_files([input_file], work_directory)
         set_permissions(uncal_file)
@@ -157,51 +157,50 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         # and use the run() method so that we can set parameters
         # progammatically.
         model = Detector1Pipeline()
+        params = {}
 
         # Always true
         if instrument == 'nircam':
-            model.refpix.odd_even_rows = False
+            params['refpix'] = dict(odd_even_rows=False)
 
         # Default CR rejection threshold is too low
-        model.jump.rejection_threshold = 15
+        #params['jump'] = dict(rejection_threshold=15)
 
         # Turn off IPC step until it is put in the right place
-        model.ipc.skip = True
+        params['ipc'] = dict(skip=True)
 
-        model.jump.save_results = True
-        model.jump.output_dir = work_directory
-        model.jump.maximum_cores = max_cores
-        jump_output = uncal_file.replace('uncal', 'jump')
+        # Set up to save jump step output
+        params['jump']['save_results'] = True
+        params['jump']['output_dir'] = work_directory
+        params['jump']['maximum_cores'] = max_cores
+        jump_output = short_name + '_jump.fits'
 
         # Check to see if the jump version of the requested file is already
         # present
         run_jump = not os.path.isfile(jump_output)
 
         if ramp_fit:
-            model.ramp_fit.save_results = True
-            model.ramp_fit.maximum_cores = max_cores
-            # model.save_results = True
-            model.output_dir = work_directory
-            # pipe_output = os.path.join(output_dir, input_file_only.replace('uncal', 'rate'))
-            pipe_output = uncal_file.replace('uncal', '0_ramp_fit')
+            params['ramp_fit'] = dict(save_results=True, maximum_cores=max_cores)
+
+            pipe_output = os.path.join(work_directory, short_name + '_0_ramp_fit.fits')
             run_slope = not os.path.isfile(pipe_output)
             if save_fitopt:
-                model.ramp_fit.save_opt = True
-                fitopt_output = uncal_file.replace('uncal', 'fitopt')
+                params['ramp_fit']['save_opt'] = True
+                fitopt_output = os.path.join(work_directory, short_name + '_fitopt.fits')
                 run_fitopt = not os.path.isfile(fitopt_output)
             else:
-                model.ramp_fit.save_opt = False
+                params['ramp_fit']['save_opt'] = False
                 fitopt_output = None
                 run_fitopt = False
         else:
-            model.ramp_fit.skip = True
+            params['ramp_fit']['skip'] = True
             pipe_output = None
             fitopt_output = None
             run_slope = False
             run_fitopt = False
 
         if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
-            model.run(datamodel)
+            model.call(datamodel, output_dir=work_directory)
         else:
             print(("Files with all requested calibration states for {} already present in "
                    "output directory. Skipping pipeline call.".format(uncal_file)))
@@ -211,7 +210,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
             status_f.write("{}\n".format(e))
             status_f.write("FAILED")
         sys.exit(1)
-    
+
     with open(status_file, "a+") as status_f:
         status_f.write("{}\n".format(jump_output))
         status_f.write("{}\n".format(pipe_output))
@@ -258,14 +257,14 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
 
     with open(general_status_file, "a+") as status_file:
         status_file.write("Finished parsing args at {}\n".format(time.ctime()))
-    
+
     input_file = args.input_file
     instrument = args.instrument
     short_name = args.short_name
     working_path = args.working_path
     pipe_type = args.pipe
     outputs = args.outputs
-    
+
     status_file = os.path.join(working_path, short_name+"_status.txt")
     with open(status_file, 'w') as out_file:
         out_file.write("Starting Process\n")
@@ -275,10 +274,10 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         out_file.write("\tinstrument is {} ({})\n".format(instrument, type(instrument)))
         out_file.write("\tinput_file is {} ({})\n".format(input_file, type(input_file)))
         out_file.write("\tshort_name is {} ({})\n".format(short_name, type(short_name)))
-    
+
     if not os.path.isfile(args.input_file):
         raise FileNotFoundError("No input file {}".format(args.input_file))
-    
+
     if pipe_type not in ['jump', 'cal']:
         raise ValueError("Unknown calibration type {}".format(pipe_type))
 
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 516738ed0..c4dceee32 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -231,7 +231,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
     with process.stderr:
         log_subprocess_output(process.stderr)
-    result = process.wait()    
+    result = process.wait()
     logging.info("Subprocess result was {}".format(result))
 
     if not os.path.isfile(res_file):
@@ -241,7 +241,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
             for line in status:
                 logging.error(line.strip())
             return status
-        
+
     with open(res_file, 'r') as inf:
         status = inf.readlines()
     return status
@@ -251,17 +251,17 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
 def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, step_args={}):
     """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each
     step as a separate output file, then return the name-and-path of the file as reduced
-    in the reduction directory. Once all requested extensions have been produced, the 
+    in the reduction directory. Once all requested extensions have been produced, the
     pipeline will return.
 
     Parameters
     ----------
     input_file_name : str
         File on which to run the pipeline steps
-    
+
     short_name : str
         Name of the file to be calibrated after any extensions have been stripped off.
-    
+
     ext_or_exts : list
         List of extensions to be retrieved.
 
@@ -291,7 +291,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
     output_dir = os.path.join(config['transfer_dir'], "outgoing")
     msg = "Input from {}, calibrate in {}, output to {}"
     logging.info(msg.format(input_dir, cal_dir, output_dir))
-    
+
     input_file = os.path.join(input_dir, input_file_name)
     current_dir = os.path.dirname(__file__)
     cmd_name = os.path.join(current_dir, "run_pipeline.py")
@@ -308,9 +308,9 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
         logging.info("Requesting {}".format(calibrated_files))
 
     cores = 'all'
-    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file, 
+    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file,
                             short_name, result_file, cores)
-    
+
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
     else:
@@ -323,7 +323,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
             logging.error("\t{}".format(line.strip()))
         if core_fail:
             cores = "half"
-            status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, 
+            status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
                                     input_file, short_name, result_file, cores)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
@@ -337,7 +337,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                     logging.error("\t{}".format(line.strip()))
                 if core_fail:
                     cores = "none"
-                    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, 
+                    status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
                                             input_file, short_name, result_file, cores)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
@@ -346,7 +346,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                         logging.error("Pipeline subprocess failed.")
         if not managed:
             raise ValueError("Pipeline Failed")
-    
+
     for file in calibrated_files:
         logging.info("Checking for output {}".format(file))
         if not os.path.isfile(os.path.join(cal_dir, file)):
@@ -417,17 +417,17 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
         logging.error("File {} not found!".format(input_file))
         raise FileNotFoundError("{} not found".format(input_file))
 
-    short_name = input_file_name.replace("_uncal", "").replace("_0thgroup", "").replace(".fits", "")
+    short_name = input_file_name[0: input_file_name.rfind('_')]
     ensure_dir_exists(cal_dir)
     output_dir = os.path.join(config["transfer_dir"], "outgoing")
-    
+
     cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py")
     result_file = os.path.join(cal_dir, short_name+"_status.txt")
 
     cores = 'all'
-    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, 
+    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file,
                             short_name, result_file, cores)
-    
+
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
     else:
@@ -440,7 +440,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
             logging.error("\t{}".format(line.strip()))
         if core_fail:
             cores = "half"
-            status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, 
+            status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
                                     input_file, short_name, result_file, cores)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
@@ -454,7 +454,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
                     logging.error("\t{}".format(line.strip()))
                 if core_fail:
                     cores = "none"
-                    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, 
+                    status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
                                             input_file, short_name, result_file, cores)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
@@ -528,7 +528,7 @@ def prep_file(input_file, in_ext):
     input_path, input_name = os.path.split(input_file)
     logging.info("\tPath is {}, file is {}".format(input_path, input_name))
 
-    if "uncal" not in in_ext:
+    if "uncal" not in in_ext and "dark" not in in_ext:
         logging.info("\tSwitching from {} to uncal".format(in_ext))
         uncal_name = os.path.basename(input_file).replace(in_ext, "uncal")
         uncal_file = filesystem_path(uncal_name, check_existence=True)
@@ -762,7 +762,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
     file_or_files : str or list-of-str
         Name (or names) of the result file(s), including path(s)
     """
-    logging.info("Pipeline call requestion calibrated extensions {}".format(ext_or_exts))
+    logging.info("Pipeline call requested calibrated extensions {}".format(ext_or_exts))
     for input_file in input_files:
         logging.info("\tCalibrating {}".format(input_file))
 

From 00e9058c6a8189363e43a00b57bafd975b151523 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 8 May 2023 15:45:02 -0400
Subject: [PATCH 082/256] Turn off steps when input is dark.fits

---
 jwql/shared_tasks/run_pipeline.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 124f51a15..361501ac5 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -166,9 +166,6 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         # Default CR rejection threshold is too low
         #params['jump'] = dict(rejection_threshold=15)
 
-        # Turn off IPC step until it is put in the right place
-        params['ipc'] = dict(skip=True)
-
         # Set up to save jump step output
         params['jump']['save_results'] = True
         params['jump']['output_dir'] = work_directory
@@ -199,8 +196,23 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
             run_slope = False
             run_fitopt = False
 
+        # If the input file is dark.fits rather than uncal.fits, then skip
+        # all of the pipeline steps that are run prior to dark subtraction
+        if 'dark.fits' in input_file:
+            if instrument.lower() == 'miri':
+                steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe',
+                                 'lastframe', 'reset', 'linearity', 'rscd']
+            else:
+                steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias',
+                                 'refpix', 'linearity']
+            for step in steps_to_skip:
+                params[step] = dict(skip: True)
+        else:
+            # Turn off IPC step until it is put in the right place
+            params['ipc'] = dict(skip=True)
+
         if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
-            model.call(datamodel, output_dir=work_directory)
+            model.call(datamodel, output_dir=work_directory, steps=params)
         else:
             print(("Files with all requested calibration states for {} already present in "
                    "output directory. Skipping pipeline call.".format(uncal_file)))

From 858b06eeed934d97995c336b68900d632d5c42b5 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 8 May 2023 17:30:03 -0400
Subject: [PATCH 083/256] Update run_pipe to optionally skip steps

---
 .../common_monitors/dark_monitor.py           |  8 +++----
 jwql/shared_tasks/run_pipeline.py             | 24 +++++++++++++++----
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 26d433ed1..8750914d5 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -402,11 +402,11 @@ def exclude_existing_badpix(self, badpix, pixel_type):
         new_pixels_y : list
             List of y coordinates of new bad pixels
         """
-        
+
         if len(badpix[0]) == 0:
             logging.warning("\tNo new {} pixels to check.".format(pixel_type))
             return ([], [])
-        
+
         logging.info("\tChecking {} potential new {} pixels".format(len(badpix[0]), pixel_type))
 
         if pixel_type not in ['hot', 'dead', 'noisy']:
@@ -440,7 +440,7 @@ def exclude_existing_badpix(self, badpix, pixel_type):
             if len(np.intersect1d(ind_x[0], ind_y[0])) == 0:
                 new_pixels_x.append(x)
                 new_pixels_y.append(y)
-        
+
         logging.info("\t\tKeeping {} {} pixels".format(len(new_pixels_x), pixel_type))
 #             pixel = (x, y)
 #             if pixel not in already_found:
@@ -810,7 +810,7 @@ def process(self, file_list):
                 # Add new noisy pixels to the database
                 logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0])))
                 self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time)
-            
+
             logging.info("Creating Mean Slope Image {}".format(slope_image))
             # Create png file of mean slope image. Add bad pixels only for full frame apertures
             self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix,
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 361501ac5..c59126ebe 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -27,7 +27,7 @@
 from jwst.saturation import SaturationStep
 from jwst.superbias import SuperBiasStep
 
-from jwql.instrument_monitors.pipeline_tools import PIPELINE_STEP_MAPPING, get_pipeline_steps
+from jwql.instrument_monitors.pipeline_tools import PIPELINE_STEP_MAPPING, completed_pipeline_steps, get_pipeline_steps
 from jwql.utils.logging_functions import configure_logging
 from jwql.utils.permissions import set_permissions
 from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
@@ -55,15 +55,26 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         set_permissions(uncal_file)
 
         steps = get_pipeline_steps(instrument)
+
+        # If the input file is a file other than uncal.fits, then we may only need to run a
+        # subset of steps. Check the completed steps in the input file and set steps such
+        # that anything that is already complete is not re-run.
+        if 'uncal.fits' not in input_file:
+            completed_steps = completed_pipeline_steps(filename)
+            for step in steps:
+                if step in completed_steps:
+                    if completed_steps[step]:
+                        steps[step] = False
+
         first_step_to_be_run = True
         for step_name in steps:
-            sys.stderr.write("Running step {}\n".format(step_name))
-            with open(status_file, 'a+') as status_f:
-                status_f.write("Running step {}\n".format(step_name))
             kwargs = {}
             if step_name in ['jump', 'rate']:
                 kwargs = {'maximum_cores': max_cores}
             if steps[step_name]:
+                sys.stderr.write("Running step {}\n".format(step_name))
+                with open(status_file, 'a+') as status_f:
+                    status_f.write("Running step {}\n".format(step_name))
                 output_file_name = short_name + "_{}.fits".format(step_name)
                 output_file = os.path.join(work_directory, output_file_name)
                 # skip already-done steps
@@ -104,6 +115,11 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
                     if done:
                         sys.stderr.write("Done pipeline.\n")
                         break
+            else:
+                sys.stderr.write("Skipping step {}\n".format(step_name))
+                with open(status_file, 'a+') as status_f:
+                    status_f.write("Skipping step {}\n".format(step_name))
+
     except Exception as e:
         with open(status_file, "a+") as status_f:
             status_f.write("EXCEPTION\n")

From e7527f7b62df8605407741356c67bd5f68e63e35 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 9 May 2023 10:33:36 -0400
Subject: [PATCH 084/256] Fix logic for skipping completed steps

---
 jwql/shared_tasks/run_pipeline.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index c59126ebe..82e85fd1a 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -57,15 +57,19 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         steps = get_pipeline_steps(instrument)
 
         # If the input file is a file other than uncal.fits, then we may only need to run a
-        # subset of steps. Check the completed steps in the input file and set steps such
-        # that anything that is already complete is not re-run.
+        # subset of steps. Check the completed steps in the input file. Find the latest step
+        # that has been completed, and skip that plus all prior steps
         if 'uncal.fits' not in input_file:
             completed_steps = completed_pipeline_steps(filename)
             for step in steps:
-                if step in completed_steps:
-                    if completed_steps[step]:
-                        steps[step] = False
+                steps[step] = not completed_steps[step]
 
+        # Special case: if the input file is a dark.fits file, then we want to skip the
+        # dark_current subtraction step
+        if 'dark.fits' in input_file:
+            step['dark_current'] = False
+
+        # Run each specified step
         first_step_to_be_run = True
         for step_name in steps:
             kwargs = {}

From 712894b86bdb1d982e38df78c9f9ae605aacaf59 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 9 May 2023 11:02:46 -0400
Subject: [PATCH 085/256] Add missing steps to step list

---
 jwql/instrument_monitors/pipeline_tools.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py
index e9754bd17..9390802e8 100644
--- a/jwql/instrument_monitors/pipeline_tools.py
+++ b/jwql/instrument_monitors/pipeline_tools.py
@@ -165,12 +165,8 @@ def get_pipeline_steps(instrument):
 
     # Order is important in 'steps' lists below!!
     if instrument == 'MIRI':
-        steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', 'lastframe',
-                 'linearity', 'rscd', 'dark_current', 'refpix', 'persistence', 'jump', 'rate']
-        # No persistence correction for MIRI
-        steps.remove('persistence')
-        # MIRI is limited to one frame per group
-        steps.remove('group_scale')
+        steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', 'lastframe', 'reset',
+                 'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate', 'gain_scale']
     else:
         steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias', 'refpix', 'linearity',
                  'persistence', 'dark_current', 'jump', 'rate']

From 3cd5cd618526705db3c0ea6eb84221dfbb80498c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 9 May 2023 12:31:09 -0400
Subject: [PATCH 086/256] rateints support

---
 jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +-
 jwql/shared_tasks/run_pipeline.py                        | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 8750914d5..36b56a5cb 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -712,7 +712,7 @@ def process(self, file_list):
                 logging.info("\t\tAdding {} to calibration set".format(filename))
                 pipeline_files.append(filename)
 
-        outputs = run_parallel_pipeline(pipeline_files, "dark", "rate", self.instrument)
+        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument)
         for filename in file_list:
             processed_file = filename.replace("_dark", "_rate")
             if processed_file not in slope_files and os.path.isfile(processed_file):
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 82e85fd1a..aef81fd56 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -60,7 +60,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # subset of steps. Check the completed steps in the input file. Find the latest step
         # that has been completed, and skip that plus all prior steps
         if 'uncal.fits' not in input_file:
-            completed_steps = completed_pipeline_steps(filename)
+            completed_steps = completed_pipeline_steps(input_file)
             for step in steps:
                 steps[step] = not completed_steps[step]
 
@@ -109,6 +109,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
                             # If the dither_points entry is not populated, then ignore this change
                             pass
                         model[0].save(output_file)
+                        if 'rateints' in outputs:
+                            model[1].save(output_file.replace('rate', 'rateints'))
 
                     done = True
                     for output in outputs:

From 43facbefd92e8ba1f6a004eab56e3f508397aa5a Mon Sep 17 00:00:00 2001
From: Maria Pena-Guerrero <pena@stsci.edu>
Date: Thu, 11 May 2023 13:47:57 -0400
Subject: [PATCH 087/256] added quick instructions for NRS TA monitors

---
 jwql/website/apps/jwql/templates/msata_monitor.html | 13 +++++++++++++
 jwql/website/apps/jwql/templates/wata_monitor.html  |  9 +++++++++
 2 files changed, 22 insertions(+)

diff --git a/jwql/website/apps/jwql/templates/msata_monitor.html b/jwql/website/apps/jwql/templates/msata_monitor.html
index 3203f069b..0779b7252 100644
--- a/jwql/website/apps/jwql/templates/msata_monitor.html
+++ b/jwql/website/apps/jwql/templates/msata_monitor.html
@@ -26,6 +26,19 @@ <h2>Updates to this monitor only occur with new data</h2>
           This monitor uses all MSATA data available and the plots include all data.
           The plots will only be updated if new MSATA data is in MAST, otherwise, the
           monitor will show all data previously obtained.
+          <h2>Instructions</h2>
+          For general MSATA monitoring you only need to look at the first two plots
+          below: 1. MSATA Status and 2. MSATA Least Squares Residual V2-V3 Offsets.
+          Plot 1 shows the successful Visit IDs (blue points, which always correspond
+          to the second position - 1/2 shutter moved from the first), the unsuccessful
+          ones (red points), and the in-progress images (gray points, which always
+          correspond to the first position of the TA). MSATA can do up-to two tries
+          so you may see 4 entries for the same program ID in the gray points.
+          Plot 2 shows the V2-V3 least square residual offsets, the closer to the
+          purple cross (the half facet coordinates) the better. The default limits of
+          the plot are the acceptable values, if you see any point at the edges or
+          outside the limits of (-0.5, 0.5) for both axis, please let any of the
+          NIRSpec TA experts: Tracy Beck, Charles Proffit, and/or Tony Keyes.
           <br>
           <br>
 
diff --git a/jwql/website/apps/jwql/templates/wata_monitor.html b/jwql/website/apps/jwql/templates/wata_monitor.html
index d891e00bd..6403f97c3 100644
--- a/jwql/website/apps/jwql/templates/wata_monitor.html
+++ b/jwql/website/apps/jwql/templates/wata_monitor.html
@@ -26,6 +26,15 @@ <h2>Updates to this monitor only occur with new data</h2>
           This monitor uses all WATA data available and the plots include all data.
           The plots will only be updated if new WATA data is in MAST, otherwise, the
           monitor will show all data previously obtained.
+          <h2>Instructions</h2>
+          For general WATA monitoring you only need to look at the first two plots
+          below: 1. WATA Status and 2. WATA Residual V2-V3 Offsets.
+          Plot 1 shows the successful Visit IDs (blue points) and the unsuccessful
+          ones (red points).
+          Plot 2 shows the V2-V3 residual offsets, the closer to (0,0) the better. The
+          default limits of the plot are the acceptable values, if you see any point
+          at the edges or outside the limits of (-0.5, 0.5) for both axis, please let
+          any of the NIRSpec TA experts: Tracy Beck, Charles Proffit, and/or Tony Keyes.
           <br>
           <br>
 

From fb2bb90ca20c206a1b23c09d640cdc589c67703d Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 10:11:10 -0400
Subject: [PATCH 088/256] trivial commit

---
 jwql/shared_tasks/shared_tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index c4dceee32..d191311cd 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -702,7 +702,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False):
         uncal_name = os.path.basename(uncal_file)
         result = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe)
         logging.info("\t\tStarting with ID {}".format(result.id))
-        processed_path = result.get()
+        #processed_path = result.get()
         logging.info("\t\tPipeline Complete")
         output = retrieve_files(short_name, ext_or_exts, retrieve_dir)
     except Exception as e:
@@ -789,7 +789,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
         for short_name in results:
             try:
                 logging.info("\tWaiting for {} ({})".format(short_name, results[short_name].id))
-                processed_path = results[short_name].get()
+                #processed_path = results[short_name].get()
                 logging.info("\t{} retrieved".format(short_name))
                 outputs[input_file_paths[short_name]] = retrieve_files(short_name, ext_or_exts, output_dirs[short_name])
                 logging.info("\tFiles copied for {}".format(short_name))

From dc97986425be985989ad2385e39c0c59eb7edf62 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 10:51:33 -0400
Subject: [PATCH 089/256] Move monitor_mast functions into mast_utils. Update
 tests.

---
 jwql/jwql_monitors/monitor_mast.py            |   1 -
 ...est_monitor_mast.py => test_mast_utils.py} |   9 +-
 jwql/utils/mast_utils.py                      | 251 +++++++++++++++++-
 3 files changed, 248 insertions(+), 13 deletions(-)
 rename jwql/tests/{test_monitor_mast.py => test_mast_utils.py} (89%)

diff --git a/jwql/jwql_monitors/monitor_mast.py b/jwql/jwql_monitors/monitor_mast.py
index a9fef8362..e9335bf6b 100755
--- a/jwql/jwql_monitors/monitor_mast.py
+++ b/jwql/jwql_monitors/monitor_mast.py
@@ -35,7 +35,6 @@
 from jwql.utils.protect_module import lock_module
 
 
-# Temporary until JWST operations: switch to test string for MAST request URL
 ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
 if not ON_GITHUB_ACTIONS:
     Mast._portal_api_connection.MAST_REQUEST_URL = get_config()['mast_request_url']
diff --git a/jwql/tests/test_monitor_mast.py b/jwql/tests/test_mast_utils.py
similarity index 89%
rename from jwql/tests/test_monitor_mast.py
rename to jwql/tests/test_mast_utils.py
index 23e700128..c4105c96f 100755
--- a/jwql/tests/test_monitor_mast.py
+++ b/jwql/tests/test_mast_utils.py
@@ -23,6 +23,7 @@
 
 from jwql.jwql_monitors import monitor_mast as mm
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES
+from jwql.utils import mast_utils as mu
 from jwql.utils.utils import get_config
 
 # Temporary until JWST operations: switch to test string for MAST request URL
@@ -47,7 +48,7 @@ def test_caom_instrument_keywords():
     instruments"""
     kw = []
     for ins in JWST_INSTRUMENT_NAMES:
-        kw.append(mm.instrument_keywords(ins, caom=True)['keyword'].tolist())
+        kw.append(mu.instrument_keywords(ins, caom=True)['keyword'].tolist())
 
     assert kw[0] == kw[1] == kw[2] == kw[3] == kw[4]
 
@@ -57,7 +58,7 @@ def test_filtered_instrument_keywords():
     different for all instruments"""
     kw = []
     for ins in JWST_INSTRUMENT_NAMES:
-        kw.append(mm.instrument_keywords(ins, caom=False)['keyword'].tolist())
+        kw.append(mu.instrument_keywords(ins, caom=False)['keyword'].tolist())
 
     assert kw[0] != kw[1] != kw[2] != kw[3] != kw[4]
 
@@ -65,7 +66,7 @@ def test_filtered_instrument_keywords():
 def test_instrument_inventory_filtering():
     """Test to see that the instrument inventory can be filtered"""
     filt = 'GR150R'
-    data = mm.instrument_inventory('niriss',
+    data = mu.instrument_inventory('niriss',
                                    add_filters={'filter': filt},
                                    return_data=True)
 
@@ -78,7 +79,7 @@ def test_instrument_dataproduct_filtering():
     """Test to see that the instrument inventory can be filtered
     by data product"""
     dp = 'spectrum'
-    data = mm.instrument_inventory('nirspec', dataproduct=dp, caom=True,
+    data = mu.instrument_inventory('nirspec', dataproduct=dp, caom=True,
                                    return_data=True)
 
     dps = [row['dataproduct_type'] for row in data['data']]
diff --git a/jwql/utils/mast_utils.py b/jwql/utils/mast_utils.py
index d9e2c58bb..6524a8c6a 100644
--- a/jwql/utils/mast_utils.py
+++ b/jwql/utils/mast_utils.py
@@ -15,8 +15,243 @@
 
  """
 
-from jwql.jwql_monitors import monitor_mast
-from jwql.utils.constants import JWST_DATAPRODUCTS, JWST_INSTRUMENT_NAMES_MIXEDCASE
+import logging
+import os
+
+from astroquery.mast import Mast
+from bokeh.embed import components
+from bokeh.io import save, output_file
+import pandas as pd
+
+from jwql.utils.constants import JWST_DATAPRODUCTS, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MAST_QUERY_LIMIT
+from jwql.utils.permissions import set_permissions
+from jwql.utils.utils import ensure_dir_exists, get_config
+from jwql.utils.plotting import bar_chart
+
+
+ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
+if not ON_GITHUB_ACTIONS:
+    Mast._portal_api_connection.MAST_REQUEST_URL = get_config()['mast_request_url']
+
+# Increase the limit on the number of entries that can be returned by
+# a MAST query.
+Mast._portal_api_connection.PAGESIZE = MAST_QUERY_LIMIT
+
+
+def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
+                         add_filters=None, add_requests=None,
+                         caom=False, return_data=False):
+    """Get the counts for a given instrument and data product
+
+    Parameters
+    ----------
+    instrument: str
+        The instrument name, i.e. one of ['niriss','nircam','nirspec',
+        'miri','fgs']
+    dataproduct: sequence, str
+        The type of data product to search
+    add_filters: dict
+        The ('paramName':'values') pairs to include in the 'filters'
+        argument of the request e.g. add_filters = {'filter':'GR150R'}
+    add_requests: dict
+        The ('request':'value') pairs to include in the request
+        e.g. add_requests = {'pagesize':1, 'page':1}
+    caom: bool
+        Query CAOM service
+    return_data: bool
+        Return the actual data instead of counts only
+
+    Returns
+    -------
+    int, dict
+        The number of database records that satisfy the search criteria
+        or a dictionary of the data if `return_data=True`
+    """
+    filters = []
+
+    # Make sure the dataproduct is a list
+    if isinstance(dataproduct, str):
+        dataproduct = [dataproduct]
+
+    # Make sure the instrument is supported
+    if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENT_NAMES]:
+        raise TypeError('Supported instruments include:', JWST_INSTRUMENT_NAMES)
+
+    # CAOM service
+    if caom:
+
+        # Declare the service
+        service = 'Mast.Caom.Filtered'
+
+        # Set the filters
+        filters += [{'paramName': 'obs_collection', 'values': ['JWST']},
+                    {'paramName': 'instrument_name', 'values': [instrument]},
+                    {'paramName': 'dataproduct_type', 'values': dataproduct}]
+
+    # Instruent filtered service
+    else:
+
+        # Declare the service
+        service = 'Mast.Jwst.Filtered.{}'.format(instrument.title())
+
+    # Include additonal filters
+    if isinstance(add_filters, dict):
+        filters += [{"paramName": name, "values": [val]}
+                    for name, val in add_filters.items()]
+
+    # Assemble the request
+    params = {'columns': 'COUNT_BIG(*)',
+              'filters': filters,
+              'removenullcolumns': True}
+
+    # Just get the counts
+    if return_data:
+        params['columns'] = '*'
+
+    # Add requests
+    if isinstance(add_requests, dict):
+        params.update(add_requests)
+
+    response = Mast.service_request_async(service, params)
+    result = response[0].json()
+
+    # Return all the data
+    if return_data:
+        return result
+
+    # Or just the counts
+    else:
+        return result['data'][0]['Column1']
+
+
+def instrument_keywords(instrument, caom=False):
+    """Get the keywords for a given instrument service
+
+    Parameters
+    ----------
+    instrument: str
+        The instrument name, i.e. one of ['niriss','nircam','nirspec',
+        'miri','fgs']
+    caom: bool
+        Query CAOM service
+
+    Returns
+    -------
+    pd.DataFrame
+        A DataFrame of the keywords
+    """
+    # Retrieve one dataset to get header keywords
+    if not caom:
+        filter_to_add = {'program': '01440'}
+    else:
+        filter_to_add = {'proposal_id': '01440'}
+    sample = instrument_inventory(instrument, return_data=True, caom=caom,
+                                  add_requests={'pagesize': 1, 'page': 1},
+                                  add_filters=filter_to_add)
+    data = [[i['name'], i['type']] for i in sample['fields']]
+    keywords = pd.DataFrame(data, columns=('keyword', 'dtype'))
+
+    return keywords
+
+
+def jwst_inventory(instruments=JWST_INSTRUMENT_NAMES,
+                   dataproducts=['image', 'spectrum', 'cube'],
+                   caom=False, plot=False, output_dir=None):
+    """Gather a full inventory of all JWST data in each instrument
+    service by instrument/dtype
+
+    Parameters
+    ----------
+    instruments: sequence
+        The list of instruments to count
+    dataproducts: sequence
+        The types of dataproducts to count
+    caom: bool
+        Query CAOM service
+    plot: bool
+        Return a pie chart of the data
+    output_dir: str
+        Directory into which plots are saved
+
+    Returns
+    -------
+    astropy.table.table.Table
+        The table of record counts for each instrument and mode
+    """
+    if output_dir is None:
+        output_dir = os.path.join(get_config()['outputs'], 'mast_utils')
+    ensure_dir_exists(output_dir)
+
+    logging.info('Searching database...')
+    # Iterate through instruments
+    inventory, keywords = [], {}
+    for instrument in instruments:
+        ins = [instrument]
+        for dp in dataproducts:
+            count = instrument_inventory(instrument, dataproduct=dp, caom=caom)
+            ins.append(count)
+
+        # Get the total
+        ins.append(sum(ins[-3:]))
+
+        # Add it to the list
+        inventory.append(ins)
+
+        # Add the keywords to the dict
+        keywords[instrument] = instrument_keywords(instrument, caom=caom)
+
+    logging.info('Completed database search for {} instruments and {} data products.'.
+                 format(instruments, dataproducts))
+
+    # Make the table
+    all_cols = ['instrument'] + dataproducts + ['total']
+    table = pd.DataFrame(inventory, columns=all_cols)
+
+    # Plot it
+    if plot:
+        if caom:
+            output_filename = 'database_monitor_caom'
+        else:
+            output_filename = 'database_monitor_jwst'
+
+        # Make the plot
+        plt = bar_chart(table, 'instrument', dataproducts,
+                        title="JWST Inventory")
+
+        # Save the plot as full html
+        html_filename = output_filename + '.html'
+        outfile = os.path.join(output_dir, html_filename)
+        output_file(outfile)
+        save(plt)
+        set_permissions(outfile)
+
+        logging.info('Saved Bokeh plots as HTML file: {}'.format(html_filename))
+
+        # Save the plot as components
+        plt.sizing_mode = 'stretch_both'
+        script, div = components(plt)
+
+        div_outfile = os.path.join(output_dir, output_filename + "_component.html")
+        with open(div_outfile, 'w') as f:
+            f.write(div)
+            f.close()
+        set_permissions(div_outfile)
+
+        script_outfile = os.path.join(output_dir, output_filename + "_component.js")
+        with open(script_outfile, 'w') as f:
+            f.write(script)
+            f.close()
+        set_permissions(script_outfile)
+
+        logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format(
+            output_filename, output_filename))
+
+    # Melt the table
+    table = pd.melt(table, id_vars=['instrument'],
+                    value_vars=dataproducts,
+                    value_name='files', var_name='dataproduct')
+
+    return table, keywords
 
 
 def mast_query(instrument, templates, start_date, end_date, aperture=None, detector=None, filter_name=None,
@@ -73,7 +308,7 @@ def mast_query(instrument, templates, start_date, end_date, aperture=None, detec
     # Make sure instrument is correct case
     instrument = JWST_INSTRUMENT_NAMES_MIXEDCASE[instrument.lower()]
 
-    # monitor_mast.instrument_inventory does not allow list inputs to
+    # instrument_inventory does not allow list inputs to
     # the added_filters input (or at least if you do provide a list, then
     # it becomes a nested list when it sends the query to MAST. The
     # nested list is subsequently ignored by MAST.)
@@ -101,8 +336,8 @@ def mast_query(instrument, templates, start_date, end_date, aperture=None, detec
         if lamp is not None:
             parameters["lamp"] = lamp
 
-        query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
-                                                  add_filters=parameters, return_data=True, caom=False)
+        query = instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
+                                     add_filters=parameters, return_data=True, caom=False)
         if len(query['data']) > 0:
             query_results.extend(query['data'])
 
@@ -147,7 +382,7 @@ def mast_query_miri(detector, aperture, templates, start_date, end_date):
 
     instrument = 'MIRI'
 
-    # monitor_mast.instrument_inventory does not allow list inputs to
+    # instrument_inventory does not allow list inputs to
     # the added_filters input (or at least if you do provide a list, then
     # it becomes a nested list when it sends the query to MAST. The
     # nested list is subsequently ignored by MAST.)
@@ -164,8 +399,8 @@ def mast_query_miri(detector, aperture, templates, start_date, end_date):
             parameters = {"date_obs_mjd": {"min": start_date, "max": end_date},
                           "detector": detector, "exp_type": template_name}
 
-        query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
-                                                  add_filters=parameters, return_data=True, caom=False)
+        query = instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
+                                     add_filters=parameters, return_data=True, caom=False)
         if len(query['data']) > 0:
             query_results.extend(query['data'])
 

From 9836cc792814b989d10ff9cf2297ccd6ac80f09e Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 11:02:13 -0400
Subject: [PATCH 090/256] Replace mentions of monitor_mast with mast_utils

---
 jwql/tests/test_mast_utils.py |  4 ++--
 jwql/utils/monitor_utils.py   | 14 +++++++-------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/jwql/tests/test_mast_utils.py b/jwql/tests/test_mast_utils.py
index c4105c96f..0fc6c877e 100755
--- a/jwql/tests/test_mast_utils.py
+++ b/jwql/tests/test_mast_utils.py
@@ -1,6 +1,6 @@
 #! /usr/bin/env python
 
-"""Tests for the ``monitor_mast`` module.
+"""Tests for the ``mast_utils`` module.
 
 Authors
 -------
@@ -14,7 +14,7 @@
     suppress verbose output to stdout):
     ::
 
-        pytest -s test_monitor_mast.py
+        pytest -s test_mast_utils.py
 """
 
 import os
diff --git a/jwql/utils/monitor_utils.py b/jwql/utils/monitor_utils.py
index f9258e15f..e4bc13852 100644
--- a/jwql/utils/monitor_utils.py
+++ b/jwql/utils/monitor_utils.py
@@ -22,9 +22,9 @@
 
 
 from jwql.database.database_interface import Monitor, engine
-from jwql.jwql_monitors import monitor_mast
 from jwql.utils.constants import ASIC_TEMPLATES, JWST_DATAPRODUCTS, MAST_QUERY_LIMIT
 from jwql.utils.logging_functions import configure_logging, get_log_status
+from jwql.utils import mast_utils
 from jwql.utils.utils import filename_parser
 
 
@@ -121,7 +121,7 @@ def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None):
         instrument = 'MIRI'
         dark_template = ['MIR_DARKALL', 'MIR_DARKIMG', 'MIR_DARKMRS']
 
-    # monitor_mast.instrument_inventory does not allow list inputs to
+    # instrument_inventory does not allow list inputs to
     # the added_filters input (or at least if you do provide a list, then
     # it becomes a nested list when it sends the query to MAST. The
     # nested list is subsequently ignored by MAST.)
@@ -137,8 +137,8 @@ def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None):
         if readpatt is not None:
             parameters["readpatt"] = readpatt
 
-        query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
-                                                  add_filters=parameters, return_data=True, caom=False)
+        query = mast_utils.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
+                                                add_filters=parameters, return_data=True, caom=False)
         if 'data' in query.keys():
             if len(query['data']) > 0:
                 query_results.extend(query['data'])
@@ -181,7 +181,7 @@ def mast_query_ta(instrument, aperture, start_date, end_date, readpatt=None):
         else:
             exp_types = ['NRS_TACQ', 'NRS_MSATA']
 
-    # monitor_mast.instrument_inventory does not allow list inputs to
+    # instrument_inventory does not allow list inputs to
     # the added_filters input (or at least if you do provide a list, then
     # it becomes a nested list when it sends the query to MAST. The
     # nested list is subsequently ignored by MAST.)
@@ -197,8 +197,8 @@ def mast_query_ta(instrument, aperture, start_date, end_date, readpatt=None):
         if readpatt is not None:
             parameters["readpatt"] = readpatt
 
-        query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
-                                                  add_filters=parameters, return_data=True, caom=False)
+        query = mast_utils.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
+                                                add_filters=parameters, return_data=True, caom=False)
         if 'data' in query.keys():
             if len(query['data']) > 0:
                 query_results.extend(query['data'])

From 4d575dd7e5fc22ab1058494305de2b3afb136c2c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 11:06:48 -0400
Subject: [PATCH 091/256] Remove UNUSED monitor_mast imports

---
 .../common_monitors/dark_monitor.py                   | 11 +++++------
 .../nirspec_monitors/ta_monitors/msata_monitor.py     |  1 -
 .../nirspec_monitors/ta_monitors/wata_monitor.py      |  1 -
 3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 26d433ed1..e5c6b65c8 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -100,9 +100,8 @@
 from jwql.database.database_interface import NIRSpecDarkQueryHistory, NIRSpecDarkPixelStats, NIRSpecDarkDarkCurrent
 from jwql.database.database_interface import FGSDarkQueryHistory, FGSDarkPixelStats, FGSDarkDarkCurrent
 from jwql.instrument_monitors import pipeline_tools
-from jwql.jwql_monitors import monitor_mast
 from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline
-from jwql.utils import calculations, instrument_properties, monitor_utils
+from jwql.utils import calculations, instrument_properties, mast_utils, monitor_utils
 from jwql.utils.constants import ASIC_TEMPLATES, DARK_MONITOR_MAX_BADPOINTS_TO_PLOT, JWST_INSTRUMENT_NAMES, FULL_FRAME_APERTURES
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS
 from jwql.utils.logging_functions import log_info, log_fail
@@ -402,11 +401,11 @@ def exclude_existing_badpix(self, badpix, pixel_type):
         new_pixels_y : list
             List of y coordinates of new bad pixels
         """
-        
+
         if len(badpix[0]) == 0:
             logging.warning("\tNo new {} pixels to check.".format(pixel_type))
             return ([], [])
-        
+
         logging.info("\tChecking {} potential new {} pixels".format(len(badpix[0]), pixel_type))
 
         if pixel_type not in ['hot', 'dead', 'noisy']:
@@ -440,7 +439,7 @@ def exclude_existing_badpix(self, badpix, pixel_type):
             if len(np.intersect1d(ind_x[0], ind_y[0])) == 0:
                 new_pixels_x.append(x)
                 new_pixels_y.append(y)
-        
+
         logging.info("\t\tKeeping {} {} pixels".format(len(new_pixels_x), pixel_type))
 #             pixel = (x, y)
 #             if pixel not in already_found:
@@ -810,7 +809,7 @@ def process(self, file_list):
                 # Add new noisy pixels to the database
                 logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0])))
                 self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time)
-            
+
             logging.info("Creating Mean Slope Image {}".format(slope_image))
             # Create png file of mean slope image. Add bad pixels only for full frame apertures
             self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix,
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index d8a34a2a6..d3593ffcb 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -58,7 +58,6 @@
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.database.database_interface import session, engine
 from jwql.database.database_interface import NIRSpecTAQueryHistory, NIRSpecTAStats
-from jwql.jwql_monitors import monitor_mast
 from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser
 
 
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index 267f3efc1..337001b4e 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -54,7 +54,6 @@
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.database.database_interface import session, engine
 from jwql.database.database_interface import NIRSpecTAQueryHistory, NIRSpecTAStats
-from jwql.jwql_monitors import monitor_mast
 from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser
 
 

From f9a5a912738f6c0f783c9206007580054bb24d70 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 11:26:55 -0400
Subject: [PATCH 092/256] Update documentation

---
 docs/source/jwql_monitors.rst | 6 ------
 docs/source/tests.rst         | 4 ++--
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/docs/source/jwql_monitors.rst b/docs/source/jwql_monitors.rst
index 55446ea97..42221e004 100644
--- a/docs/source/jwql_monitors.rst
+++ b/docs/source/jwql_monitors.rst
@@ -25,9 +25,3 @@ monitor_filesystem.py
 .. automodule:: jwql.jwql_monitors.monitor_filesystem
     :members:
     :undoc-members:
-
-monitor_mast.py
----------------
-.. automodule:: jwql.jwql_monitors.monitor_mast
-    :members:
-    :undoc-members:
\ No newline at end of file
diff --git a/docs/source/tests.rst b/docs/source/tests.rst
index cc7d38d67..576a126e9 100644
--- a/docs/source/tests.rst
+++ b/docs/source/tests.rst
@@ -74,9 +74,9 @@ test_logging_functions.py
     :members:
     :undoc-members:
 
-test_monitor_mast.py
+test_mast_utils.py
 --------------------
-.. automodule:: jwql.tests.test_monitor_mast
+.. automodule:: jwql.tests.test_mast_utils
     :members:
     :undoc-members:
 

From 1b33f0b43794cc4519dd7c3695cc92a69b0a8c98 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 11:34:39 -0400
Subject: [PATCH 093/256] Remove db.py from doc list

---
 docs/source/website.rst | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/docs/source/website.rst b/docs/source/website.rst
index 530f367ae..6f0da9b31 100644
--- a/docs/source/website.rst
+++ b/docs/source/website.rst
@@ -26,12 +26,6 @@ data_containers.py
     :members:
     :undoc-members:
 
-db.py
------
-.. automodule:: jwql.website.apps.jwql.db
-    :members:
-    :undoc-members:
-
 forms.py
 --------
 .. automodule:: jwql.website.apps.jwql.forms

From c03d1eed3d4d44ebd1b65ef994cd706ef1f3b314 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 11:40:51 -0400
Subject: [PATCH 094/256] Add phinx_rtd_theme to rtd_requirements

---
 rtd_requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index 5ba4b8091..d3f595ab7 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -10,6 +10,7 @@ pygments==2.14.0
 pytest==7.2.1
 redis==4.5.1
 sphinx>=2
+sphinx_rtd_theme==1.2.0
 stsci_rtd_theme==1.0.0
 tomli==2.0.1
 git+https://github.com/spacetelescope/jwst_reffiles

From 653f10d860ea8ae29b3a00f2d9bdc96b63880f70 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 11:48:57 -0400
Subject: [PATCH 095/256] Remove mast_monitor.py

---
 jwql/jwql_monitors/monitor_mast.py | 293 -----------------------------
 jwql/tests/test_mast_utils.py      |   1 -
 2 files changed, 294 deletions(-)
 delete mode 100755 jwql/jwql_monitors/monitor_mast.py

diff --git a/jwql/jwql_monitors/monitor_mast.py b/jwql/jwql_monitors/monitor_mast.py
deleted file mode 100755
index e9335bf6b..000000000
--- a/jwql/jwql_monitors/monitor_mast.py
+++ /dev/null
@@ -1,293 +0,0 @@
-#! /usr/bin/env python
-
-"""This module is home to a suite of MAST queries that gather bulk
-properties of available JWST data for JWQL.
-
-Authors
--------
-
-    Joe Filippazzo
-
-Use
----
-
-    To get an inventory of all JWST files do:
-    ::
-
-        from jwql.jwql_monitors import monitor_mast
-        inventory, keywords = monitor_mast.jwst_inventory()
-"""
-
-import logging
-import os
-
-from astroquery.mast import Mast
-from bokeh.embed import components
-from bokeh.io import save, output_file
-import pandas as pd
-
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_DATAPRODUCTS, MAST_QUERY_LIMIT
-from jwql.utils.logging_functions import log_info, log_fail
-from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import get_config
-from jwql.utils import monitor_utils
-from jwql.utils.plotting import bar_chart
-from jwql.utils.protect_module import lock_module
-
-
-ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
-if not ON_GITHUB_ACTIONS:
-    Mast._portal_api_connection.MAST_REQUEST_URL = get_config()['mast_request_url']
-
-# Increase the limit on the number of entries that can be returned by
-# a MAST query.
-Mast._portal_api_connection.PAGESIZE = MAST_QUERY_LIMIT
-
-
-def instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
-                         add_filters=None, add_requests=None,
-                         caom=False, return_data=False):
-    """Get the counts for a given instrument and data product
-
-    Parameters
-    ----------
-    instrument: str
-        The instrument name, i.e. one of ['niriss','nircam','nirspec',
-        'miri','fgs']
-    dataproduct: sequence, str
-        The type of data product to search
-    add_filters: dict
-        The ('paramName':'values') pairs to include in the 'filters'
-        argument of the request e.g. add_filters = {'filter':'GR150R'}
-    add_requests: dict
-        The ('request':'value') pairs to include in the request
-        e.g. add_requests = {'pagesize':1, 'page':1}
-    caom: bool
-        Query CAOM service
-    return_data: bool
-        Return the actual data instead of counts only
-
-    Returns
-    -------
-    int, dict
-        The number of database records that satisfy the search criteria
-        or a dictionary of the data if `return_data=True`
-    """
-    filters = []
-
-    # Make sure the dataproduct is a list
-    if isinstance(dataproduct, str):
-        dataproduct = [dataproduct]
-
-    # Make sure the instrument is supported
-    if instrument.lower() not in [ins.lower() for ins in JWST_INSTRUMENT_NAMES]:
-        raise TypeError('Supported instruments include:', JWST_INSTRUMENT_NAMES)
-
-    # CAOM service
-    if caom:
-
-        # Declare the service
-        service = 'Mast.Caom.Filtered'
-
-        # Set the filters
-        filters += [{'paramName': 'obs_collection', 'values': ['JWST']},
-                    {'paramName': 'instrument_name', 'values': [instrument]},
-                    {'paramName': 'dataproduct_type', 'values': dataproduct}]
-
-    # Instruent filtered service
-    else:
-
-        # Declare the service
-        service = 'Mast.Jwst.Filtered.{}'.format(instrument.title())
-
-    # Include additonal filters
-    if isinstance(add_filters, dict):
-        filters += [{"paramName": name, "values": [val]}
-                    for name, val in add_filters.items()]
-
-    # Assemble the request
-    params = {'columns': 'COUNT_BIG(*)',
-              'filters': filters,
-              'removenullcolumns': True}
-
-    # Just get the counts
-    if return_data:
-        params['columns'] = '*'
-
-    # Add requests
-    if isinstance(add_requests, dict):
-        params.update(add_requests)
-
-    response = Mast.service_request_async(service, params)
-    result = response[0].json()
-
-    # Return all the data
-    if return_data:
-        return result
-
-    # Or just the counts
-    else:
-        return result['data'][0]['Column1']
-
-
-def instrument_keywords(instrument, caom=False):
-    """Get the keywords for a given instrument service
-
-    Parameters
-    ----------
-    instrument: str
-        The instrument name, i.e. one of ['niriss','nircam','nirspec',
-        'miri','fgs']
-    caom: bool
-        Query CAOM service
-
-    Returns
-    -------
-    pd.DataFrame
-        A DataFrame of the keywords
-    """
-    # Retrieve one dataset to get header keywords
-    if not caom:
-        filter_to_add = {'program': '01440'}
-    else:
-        filter_to_add = {'proposal_id': '01440'}
-    sample = instrument_inventory(instrument, return_data=True, caom=caom,
-                                  add_requests={'pagesize': 1, 'page': 1},
-                                  add_filters=filter_to_add)
-    data = [[i['name'], i['type']] for i in sample['fields']]
-    keywords = pd.DataFrame(data, columns=('keyword', 'dtype'))
-
-    return keywords
-
-
-def jwst_inventory(instruments=JWST_INSTRUMENT_NAMES,
-                   dataproducts=['image', 'spectrum', 'cube'],
-                   caom=False, plot=False):
-    """Gather a full inventory of all JWST data in each instrument
-    service by instrument/dtype
-
-    Parameters
-    ----------
-    instruments: sequence
-        The list of instruments to count
-    dataproducts: sequence
-        The types of dataproducts to count
-    caom: bool
-        Query CAOM service
-    plot: bool
-        Return a pie chart of the data
-
-    Returns
-    -------
-    astropy.table.table.Table
-        The table of record counts for each instrument and mode
-    """
-    logging.info('Searching database...')
-    # Iterate through instruments
-    inventory, keywords = [], {}
-    for instrument in instruments:
-        ins = [instrument]
-        for dp in dataproducts:
-            count = instrument_inventory(instrument, dataproduct=dp, caom=caom)
-            ins.append(count)
-
-        # Get the total
-        ins.append(sum(ins[-3:]))
-
-        # Add it to the list
-        inventory.append(ins)
-
-        # Add the keywords to the dict
-        keywords[instrument] = instrument_keywords(instrument, caom=caom)
-
-    logging.info('Completed database search for {} instruments and {} data products.'.
-                 format(instruments, dataproducts))
-
-    # Make the table
-    all_cols = ['instrument'] + dataproducts + ['total']
-    table = pd.DataFrame(inventory, columns=all_cols)
-
-    # Plot it
-    if plot:
-        # Determine plot location and names
-        output_dir = get_config()['outputs']
-
-        if caom:
-            output_filename = 'database_monitor_caom'
-        else:
-            output_filename = 'database_monitor_jwst'
-
-        # Make the plot
-        plt = bar_chart(table, 'instrument', dataproducts,
-                        title="JWST Inventory")
-
-        # Save the plot as full html
-        html_filename = output_filename + '.html'
-        outfile = os.path.join(output_dir, 'monitor_mast', html_filename)
-        output_file(outfile)
-        save(plt)
-        set_permissions(outfile)
-
-        logging.info('Saved Bokeh plots as HTML file: {}'.format(html_filename))
-
-        # Save the plot as components
-        plt.sizing_mode = 'stretch_both'
-        script, div = components(plt)
-
-        div_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.html")
-        with open(div_outfile, 'w') as f:
-            f.write(div)
-            f.close()
-        set_permissions(div_outfile)
-
-        script_outfile = os.path.join(output_dir, 'monitor_mast', output_filename + "_component.js")
-        with open(script_outfile, 'w') as f:
-            f.write(script)
-            f.close()
-        set_permissions(script_outfile)
-
-        logging.info('Saved Bokeh components files: {}_component.html and {}_component.js'.format(
-            output_filename, output_filename))
-
-    # Melt the table
-    table = pd.melt(table, id_vars=['instrument'],
-                    value_vars=dataproducts,
-                    value_name='files', var_name='dataproduct')
-
-    return table, keywords
-
-
-@log_fail
-@log_info
-def monitor_mast():
-    """Tabulates the inventory of all JWST data products in the MAST
-    archive and generates plots.
-    """
-    logging.info('Beginning database monitoring.')
-
-    # Perform inventory of the JWST service
-    jwst_inventory(instruments=JWST_INSTRUMENT_NAMES,
-                   dataproducts=['image', 'spectrum', 'cube'],
-                   caom=False, plot=True)
-
-    # Perform inventory of the CAOM service
-    jwst_inventory(instruments=JWST_INSTRUMENT_NAMES,
-                   dataproducts=['image', 'spectrum', 'cube'],
-                   caom=True, plot=True)
-
-
-@lock_module
-def protected_code():
-    """Protected code ensures only 1 instance of module will run at any given time"""
-
-    # Configure logging
-    module = os.path.basename(__file__).strip('.py')
-    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
-
-    # Run the monitors
-    monitor_mast()
-    monitor_utils.update_monitor_table(module, start_time, log_file)
-
-
-if __name__ == '__main__':
-    protected_code()
diff --git a/jwql/tests/test_mast_utils.py b/jwql/tests/test_mast_utils.py
index 0fc6c877e..6267be9bc 100755
--- a/jwql/tests/test_mast_utils.py
+++ b/jwql/tests/test_mast_utils.py
@@ -21,7 +21,6 @@
 
 from astroquery.mast import Mast
 
-from jwql.jwql_monitors import monitor_mast as mm
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES
 from jwql.utils import mast_utils as mu
 from jwql.utils.utils import get_config

From c3fec3aecdde6c8171e03ec4909930a729a051a5 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 15 May 2023 11:59:56 -0400
Subject: [PATCH 096/256] Fix import in CR monitor

---
 .../common_monitors/cosmic_ray_monitor.py            | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py b/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py
index 3db7afee1..19c905806 100755
--- a/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/cosmic_ray_monitor.py
@@ -61,8 +61,8 @@
 from jwql.database.database_interface import FGSCosmicRayQueryHistory
 from jwql.database.database_interface import FGSCosmicRayStats
 from jwql.database.database_interface import session, engine
-from jwql.jwql_monitors import monitor_mast
 from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline
+from jwql.utils import mast_utils
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS
 from jwql.utils.logging_functions import configure_logging
 from jwql.utils.logging_functions import log_info
@@ -557,7 +557,7 @@ def process(self, file_list):
             files
         """
         for file_chunk in grouper(file_list, 100):
-        
+
             input_files = []
             in_ext = "uncal"
             out_exts = defaultdict(lambda: ['jump', '0_ramp_fit'])
@@ -622,7 +622,7 @@ def process(self, file_list):
 
                 dir_name = '_'.join(os.path.basename(file_name).split('_')[:2])  # file_name[51:76]
                 self.obs_dir = os.path.join(self.data_dir, dir_name)
-                
+
                 if file_name not in output_files:
                     skip = False
                     head = fits.getheader(file_name)
@@ -844,9 +844,9 @@ def query_mast(self):
         data_product = JWST_DATAPRODUCTS
         parameters = {"date_obs_mjd": {"min": self.query_start, "max": self.query_end}, "apername": self.aperture}
 
-        result = monitor_mast.instrument_inventory(self.instrument, data_product,
-                                                   add_filters=parameters,
-                                                   return_data=True)
+        result = mast_utils.instrument_inventory(self.instrument, data_product,
+                                                 add_filters=parameters,
+                                                 return_data=True)
 
         return result
 

From 3e3af74672ece149aab129aeaa3cd3c37a7fd363 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 22 May 2023 16:21:47 -0400
Subject: [PATCH 097/256] update step_args in shared_tasks scripts, to be able
 to skip pipe steps

---
 .../common_monitors/dark_monitor.py           |  6 +-
 jwql/instrument_monitors/pipeline_tools.py    | 23 ++++---
 jwql/shared_tasks/run_pipeline.py             | 46 ++++++++++---
 jwql/shared_tasks/shared_tasks.py             | 65 +++++++++++++++----
 4 files changed, 110 insertions(+), 30 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 36b56a5cb..a04080ed5 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -712,7 +712,11 @@ def process(self, file_list):
                 logging.info("\t\tAdding {} to calibration set".format(filename))
                 pipeline_files.append(filename)
 
-        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument)
+        # Specify that we want to skip the dark current correction step
+        step_args = {'dark_current': {'skip: True'}}
+
+        # Call the pipeline
+        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args)
         for filename in file_list:
             processed_file = filename.replace("_dark", "_rate")
             if processed_file not in slope_files and os.path.isfile(processed_file):
diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py
index 9390802e8..a5fd4a464 100644
--- a/jwql/instrument_monitors/pipeline_tools.py
+++ b/jwql/instrument_monitors/pipeline_tools.py
@@ -25,6 +25,7 @@
 from jwst.dq_init import DQInitStep
 from jwst.dark_current import DarkCurrentStep
 from jwst.firstframe import FirstFrameStep
+from jwst.gain_scale import GainScaleStep
 from jwst.group_scale import GroupScaleStep
 from jwst.ipc import IPCStep
 from jwst.jump import JumpStep
@@ -34,6 +35,7 @@
 from jwst.pipeline.calwebb_detector1 import Detector1Pipeline
 from jwst.ramp_fitting import RampFitStep
 from jwst.refpix import RefPixStep
+from jwst.reset import ResetStep
 from jwst.rscd import RscdStep
 from jwst.saturation import SaturationStep
 from jwst.superbias import SuperBiasStep
@@ -43,16 +45,17 @@
 
 # Define the fits header keyword that accompanies each step
 PIPE_KEYWORDS = {'S_GRPSCL': 'group_scale', 'S_DQINIT': 'dq_init', 'S_SATURA': 'saturation',
-                 'S_REFPIX': 'refpix', 'S_SUPERB': 'superbias',
+                 'S_REFPIX': 'refpix', 'S_SUPERB': 'superbias', 'S_RESET': 'reset',
                  'S_PERSIS': 'persistence', 'S_DARK': 'dark_current', 'S_LINEAR': 'linearity',
                  'S_FRSTFR': 'firstframe', 'S_LASTFR': 'lastframe', 'S_RSCD': 'rscd',
-                 'S_JUMP': 'jump', 'S_RAMP': 'rate'}
+                 'S_JUMP': 'jump', 'S_RAMP': 'rate', 'S_GANSCL': 'gain_scale', 'S_IPC': 'ipc'}
 
 PIPELINE_STEP_MAPPING = {'dq_init': DQInitStep, 'dark_current': DarkCurrentStep,
-                         'firstframe': FirstFrameStep, 'group_scale': GroupScaleStep,
-                         'ipc': IPCStep, 'jump': JumpStep, 'lastframe': LastFrameStep,
-                         'linearity': LinearityStep, 'persistence': PersistenceStep,
-                         'rate': RampFitStep, 'refpix': RefPixStep, 'rscd': RscdStep,
+                         'firstframe': FirstFrameStep, 'gain_scale': GainScaleStep,
+                         'group_scale': GroupScaleStep, 'ipc': IPCStep, 'jump': JumpStep,
+                         'lastframe': LastFrameStep, 'linearity': LinearityStep,
+                         'persistence': PersistenceStep, 'rate': RampFitStep,
+                         'refpix': RefPixStep, 'reset': ResetStep, 'rscd': RscdStep,
                          'saturation': SaturationStep, 'superbias': SuperBiasStep}
 
 # Readout patterns that have nframes != a power of 2. These readout patterns
@@ -182,13 +185,13 @@ def get_pipeline_steps(instrument):
     # IPC correction currently not done for any instrument
     steps.remove('ipc')
 
-    # Initialize using PIPE_KEYWORDS so the steps will be in the right order
+    # Initialize using OrderedDict so the steps will be in the right order
     required_steps = OrderedDict({})
     for key in steps:
         required_steps[key] = True
-    for key in PIPE_KEYWORDS.values():
-        if key not in required_steps.keys():
-            required_steps[key] = False
+    #for key in PIPE_KEYWORDS.values():
+    #    if key not in required_steps.keys():
+    #        required_steps[key] = False
 
     return required_steps
 
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index aef81fd56..79fa8d260 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -5,6 +5,7 @@
 from collections import OrderedDict
 from copy import deepcopy
 from glob import glob
+import json
 import os
 import shutil
 import sys
@@ -33,7 +34,7 @@
 from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
 
 
-def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_cores='all'):
+def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_cores='all', step_args={}):
     """Run the steps of ``calwebb_detector1`` on the input file, saving the result of each
     step as a separate output file, then return the name-and-path of the file as reduced
     in the reduction directory.
@@ -61,20 +62,47 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # that has been completed, and skip that plus all prior steps
         if 'uncal.fits' not in input_file:
             completed_steps = completed_pipeline_steps(input_file)
+
+            # Reverse the boolean value, so that now steps answers the question: "Do we need
+            # to run this step?""
             for step in steps:
                 steps[step] = not completed_steps[step]
 
         # Special case: if the input file is a dark.fits file, then we want to skip the
         # dark_current subtraction step
-        if 'dark.fits' in input_file:
-            step['dark_current'] = False
+        #if 'dark.fits' in input_file:
+        #    steps['dark_current'] = False
+
+        # Make sure we don't run steps out of order. Find the latest step that has been
+        # run, and only run subsequent steps. This protects against cases where some early
+        # step was not run. In that case, we don't want to go back and run it because running
+        # pipeline steps out of order doesn't work.
+        last_run = 'group_scale'  # initialize to the first step
+        for step in steps:
+            if not steps[step]:
+                last_run = deepcopy(step)
+
+        for step in steps:
+            if step == last_run:
+                break
+            if step != last_run:
+                steps[step] = False
+
+        # Set any steps the user specifically asks to skip
+        for step, step_dict in step_args.items():
+            if 'skip' in step_dict:
+                print(f'SKIP the {step}!!')
+                if step_dict['skip']:
+                    steps[step] = False
 
         # Run each specified step
         first_step_to_be_run = True
         for step_name in steps:
             kwargs = {}
+            if step_name in step_args:
+                kwargs = step_args[step_name]
             if step_name in ['jump', 'rate']:
-                kwargs = {'maximum_cores': max_cores}
+                kwargs['maximum_cores'] = max_cores
             if steps[step_name]:
                 sys.stderr.write("Running step {}\n".format(step_name))
                 with open(status_file, 'a+') as status_f:
@@ -138,7 +166,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
     # Done.
 
 
-def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True, max_cores='all'):
+def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=True, save_fitopt=True, max_cores='all', step_args={}):
     """Call ``calwebb_detector1`` on the provided file, running all
     steps up to the ``ramp_fit`` step, and save the result. Optionally
     run the ``ramp_fit`` step and save the resulting slope file as well.
@@ -228,7 +256,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
                 steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias',
                                  'refpix', 'linearity']
             for step in steps_to_skip:
-                params[step] = dict(skip: True)
+                params[step] = dict(skip=True)
         else:
             # Turn off IPC step until it is put in the right place
             params['ipc'] = dict(skip=True)
@@ -269,6 +297,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     out_help = 'Comma-separated list of output extensions (for cal only, otherwise just "all")'
     name_help = 'Input file name with no path or extensions'
     cores_help = 'Maximum cores to use (default "all")'
+    step_args_help = 'Step-specific parameter value nested dictionary'
     parser = argparse.ArgumentParser(description='Run local calibration')
     parser.add_argument('pipe', metavar='PIPE', type=str, help=pipe_help)
     parser.add_argument('outputs', metavar='OUTPUTS', type=str, help=out_help)
@@ -277,6 +306,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     parser.add_argument('input_file', metavar='FILE', type=str, help=file_help)
     parser.add_argument('short_name', metavar='NAME', type=str, help=name_help)
     parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help)
+    parser.add_argument('step_args', metavar='STEP_ARGS', type=json.loads, help=step_args_help)
 
     with open(general_status_file, "a+") as status_file:
         status_file.write("Created argument parser at {}\n".format(time.ctime()))
@@ -319,12 +349,12 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         if pipe_type == 'jump':
             with open(status_file, 'a+') as out_file:
                 out_file.write("Running jump pipeline.\n")
-            run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True, max_cores=args.max_cores)
+            run_save_jump(input_file, short_name, working_path, instrument, ramp_fit=True, save_fitopt=True, max_cores=args.max_cores, step_args=args.step_args)
         elif pipe_type == 'cal':
             with open(status_file, 'a+') as out_file:
                 out_file.write("Running cal pipeline.\n")
             outputs = outputs.split(",")
-            run_pipe(input_file, short_name, working_path, instrument, outputs, max_cores=args.max_cores)
+            run_pipe(input_file, short_name, working_path, instrument, outputs, max_cores=args.max_cores, step_args=args.step_args)
     except Exception as e:
         with open(status_file, 'a+') as out_file:
             out_file.write("Exception when starting pipeline.\n")
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index d191311cd..eef421ecd 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -224,9 +224,42 @@ def collect_after_task(**kwargs):
     gc.collect()
 
 
-def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores):
-    command = "{} {} {} '{}' {} {} {} {}"
-    command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores)
+def convert_step_args_to_string(args_dict):
+    """Convert the nested dictionary containing pipeline step parameter keyword/value pairs
+    to a string so that it can be passed via command line
+
+    Parameters
+    ----------
+    args_dict : dict
+        Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing
+        keyword value pairs for that step.
+
+    Returns
+    -------
+    args_str : str
+        String representation of ``args_dict``
+    """
+    args_str='{'
+
+    for i, step in enumerate(args_dict):
+        args_str += f'"{step}":'
+        args_str += '{'
+        for j, (param, val) in enumerate(args_dict[step].items()):
+            args_str += f'"{param}":"{val}"'
+            if j < len(args_dict[step])-1:
+                args_str += ', '
+        args_str += '}'
+        if i < len(args_dict)-1:
+            args_str += ','
+    return args_str
+
+
+def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args):
+    # Convert step_args dictionary to a string so that it can be passed via command line
+    step_args_str = convert_step_args_to_string(step_args)
+
+    command = "{} {} {} '{}' {} {} {} {} {}"
+    command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
     logging.info("Running {}".format(command))
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
     with process.stderr:
@@ -309,7 +342,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
 
     cores = 'all'
     status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument, input_file,
-                            short_name, result_file, cores)
+                            short_name, result_file, cores, step_args)
 
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
@@ -324,7 +357,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
         if core_fail:
             cores = "half"
             status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
-                                    input_file, short_name, result_file, cores)
+                                    input_file, short_name, result_file, cores, step_args)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
                 managed = True
@@ -338,7 +371,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
                 if core_fail:
                     cores = "none"
                     status = run_subprocess(cmd_name, "cal", outputs, cal_dir, instrument,
-                                            input_file, short_name, result_file, cores)
+                                            input_file, short_name, result_file, cores, step_args)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
                         managed = True
@@ -554,7 +587,7 @@ def prep_file(input_file, in_ext):
     return short_name, cal_lock, os.path.join(send_path, uncal_name)
 
 
-def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False):
+def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False, step_args={}):
     """Starts the standard or save_jump pipeline for the provided file.
 
     .. warning::
@@ -591,6 +624,11 @@ def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=Fa
     jump_pipe : bool
         Whether the detector1 jump pipeline is being used (e.g. the bad pixel monitor)
 
+    step_args : dict
+        Pipeline step arguments to be passed to the pipeline call. Nested dictionary with keys that
+        are the step names (as seen in pipeline_tools.PIPELINE_STEP_MAPPING). Each value is a
+        dictionary of keyword value pairs that are relevant for that step.
+
     Returns
     -------
     result : celery.result.AsyncResult
@@ -606,9 +644,9 @@ def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=Fa
                 ramp_fit = True
             elif "fitopt" in ext:
                 save_fitopt = True
-        result = calwebb_detector1_save_jump.delay(input_file, instrument, ramp_fit=ramp_fit, save_fitopt=save_fitopt)
+        result = calwebb_detector1_save_jump.delay(input_file, instrument, ramp_fit=ramp_fit, save_fitopt=save_fitopt, step_args=step_args)
     else:
-        result = run_calwebb_detector1.delay(input_file, short_name, ext_or_exts, instrument)
+        result = run_calwebb_detector1.delay(input_file, short_name, ext_or_exts, instrument, step_args=step_args)
     return result
 
 
@@ -716,7 +754,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False):
     return output
 
 
-def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pipe=False):
+def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pipe=False, step_args={}):
     """Convenience function for using the ``run_calwebb_detector1`` function on a list of
     data files, breaking them into parallel celery calls, collecting the results together,
     and returning the results as another list. In particular, this function will do the
@@ -757,6 +795,11 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
     jump_pipe : bool
         Whether the detector1 jump pipeline is being used (e.g. the bad pixel monitor)
 
+    step_args : dict
+        Pipeline step arguments to be passed to the pipeline call. Nested dictionary with keys that
+        are the step names (as seen in pipeline_tools.PIPELINE_STEP_MAPPING). Each value is a
+        dictionary of keyword value pairs that are relevant for that step.
+
     Returns
     -------
     file_or_files : str or list-of-str
@@ -782,7 +825,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
             output_dirs[short_name] = retrieve_dir
             input_file_paths[short_name] = input_file
             locks[short_name] = cal_lock
-            results[short_name] = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe)
+            results[short_name] = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe, step_args=step_args)
             logging.info("\tStarting {} with ID {}".format(short_name, results[short_name].id))
         logging.info("Celery tasks submitted.")
         logging.info("Waiting for task results")

From 458c789fd130249a2accfcac76779bddff540394 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 22 May 2023 16:50:51 -0400
Subject: [PATCH 098/256] Propogate user-input pipeline keywords.

---
 jwql/shared_tasks/run_pipeline.py | 13 ++++++++++++-
 jwql/shared_tasks/shared_tasks.py |  9 +++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 79fa8d260..ba321eafa 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -256,11 +256,22 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
                 steps_to_skip = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias',
                                  'refpix', 'linearity']
             for step in steps_to_skip:
-                params[step] = dict(skip=True)
+                step_dict = dict(skip=True)
+                if step in params:
+                    params[step] = params[step].update(step_dict)
+                else:
+                    params[step] = dict(skip=True)
         else:
             # Turn off IPC step until it is put in the right place
             params['ipc'] = dict(skip=True)
 
+        # Include any user-specified parameters
+        for step_name in step_args:
+            if step_name in params:
+                params[step_name] = params[step_name].update(step_args[step_name])
+            else:
+                params[step_name] = step_args[step_name]
+
         if run_jump or (ramp_fit and run_slope) or (save_fitopt and run_fitopt):
             model.call(datamodel, output_dir=work_directory, steps=params)
         else:
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index eef421ecd..450405058 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -332,8 +332,13 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
     result_file = os.path.join(cal_dir, short_name+"_status.txt")
     if "all" in ext_or_exts:
         logging.info("All outputs requested")
-        out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity",
-                    "persistence", "dark_current", "jump", "rate"]
+        if instrument.lower() != 'miri':
+            out_exts = ["dq_init", "saturation", "superbias", "refpix", "linearity",
+                        "persistence", "dark_current", "jump", "rate"]
+        else:
+            out_exts = ["group_scale", "dq_init", "saturation", "firstframe", "lastframe", "reset",
+                        "linearity", "rscd", "dark_current", "refpix", "jump", "rate", "gain_scale"]
+
         calibrated_files = ["{}_{}.fits".format(short_name, ext) for ext in out_exts]
         logging.info("Requesting {}".format(calibrated_files))
     else:

From 7a1e1858df9ad08aaca299ec6abd71ed883f74c5 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 22 May 2023 17:10:40 -0400
Subject: [PATCH 099/256] Updating conda requirements

---
 environment_python_3.10.yml | 28 ++++++++++++++--------------
 environment_python_3.9.yml  | 28 ++++++++++++++--------------
 2 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 68a1fa077..6ef4590b0 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -22,36 +22,36 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.2.1
+  - astropy=5.2.2
   - bokeh=2.4.3
-  - beautifulsoup4=4.11.2
+  - beautifulsoup4=4.12.2
   - celery=5.2.7
-  - cryptography=39.0.1
-  - django=4.1.7
+  - cryptography=40.0.2
+  - django=4.2.1
   - flake8=6.0.0
   - inflection=0.5.1
-  - ipython=8.10.0
+  - ipython=8.13.2
   - jinja2=3.1.2
   - jsonschema=4.17.3
   - matplotlib=3.7.0
-  - nodejs=18.12.1
-  - numpy=1.24.2
+  - nodejs=18.15.0
+  - numpy=1.24.3
   - numpydoc=1.5.0
-  - pandas=2.0.0
-  - pip=23.0
+  - pandas=2.0.1
+  - pip=23.1.2
   - postgresql=15.2
   - psycopg2=2.9.3
-  - pytest=7.2.1
+  - pytest=7.3.1
   - pytest-cov=4.0.0
   - pytest-mock=3.10.0
   - python=3.10.9
   - pyyaml=6.0
   - redis
-  - scipy=1.9.3
-  - setuptools=67.3.1
-  - sphinx=6.1.3
+  - scipy=1.10.1
+  - setuptools=67.7.2
+  - sphinx=6.2.1
   - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.8
+  - sqlalchemy=2.0.15
   - twine=4.0.2
   - wtforms=3.0.1
 
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index 2aadf5067..c021f3858 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -22,36 +22,36 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.2.1
+  - astropy=5.2.2
   - bokeh=2.4.3
-  - beautifulsoup4=4.11.2
+  - beautifulsoup4=4.12.2
   - celery=5.2.7
-  - cryptography=39.0.1
-  - django=4.1.7
+  - cryptography=40.0.2
+  - django=4.2.1
   - flake8=6.0.0
   - inflection=0.5.1
-  - ipython=8.10.0
+  - ipython=8.13.2
   - jinja2=3.1.2
   - jsonschema=4.17.3
   - matplotlib=3.7.0
-  - nodejs=18.12.1
-  - numpy=1.24.2
+  - nodejs=18.15.0
+  - numpy=1.24.3
   - numpydoc=1.5.0
-  - pandas=2.0.0
-  - pip=23.0
+  - pandas=2.0.1
+  - pip=23.1.2
   - postgresql=15.2
   - psycopg2=2.9.3
-  - pytest=7.2.1
+  - pytest=7.3.1
   - pytest-cov=4.0.0
   - pytest-mock=3.10.0
   - python=3.9.16
   - pyyaml=6.0
   - redis
-  - scipy=1.9.3
-  - setuptools=67.3.1
-  - sphinx=6.1.3
+  - scipy=1.10.1
+  - setuptools=67.7.2
+  - sphinx=6.2.1
   - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.8
+  - sqlalchemy=2.0.15
   - twine=4.0.2
   - wtforms=3.0.1
 

From 38cea5b4e9d30ff311ea435d62348054faed11d3 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Tue, 23 May 2023 10:05:34 -0400
Subject: [PATCH 100/256] Updating pip requirements

---
 environment_python_3.10.yml | 10 +++++-----
 environment_python_3.9.yml  | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 6ef4590b0..ea1eff17e 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -57,13 +57,13 @@ dependencies:
 
   - pip:
     - astroquery==0.4.6
-    - bandit==1.7.4
-    - jwst==1.10.0
+    - bandit==1.7.5
+    - jwst==1.10.2
     - pysiaf==0.19.1
     - pysqlite3==0.5.0
-    - pyvo==1.4
-    - redis==4.5.1
-    - selenium==4.8.0
+    - pyvo==1.4.1
+    - redis==4.5.5
+    - selenium==4.9.1
     - stdatamodels==1.3.1
     - stsci_rtd_theme==1.0.0
     - vine==5.0.0
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index c021f3858..564ecb0e6 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -57,13 +57,13 @@ dependencies:
 
   - pip:
     - astroquery==0.4.6
-    - bandit==1.7.4
-    - jwst==1.10.0
+    - bandit==1.7.5
+    - jwst==1.10.2
     - pysiaf==0.19.1
     - pysqlite3==0.5.0
-    - pyvo==1.4
-    - redis==4.5.1
-    - selenium==4.8.0
+    - pyvo==1.4.1
+    - redis==4.5.5
+    - selenium==4.9.1
     - stdatamodels==1.3.1
     - stsci_rtd_theme==1.0.0
     - vine==5.0.0

From 1a289b0e3bb803505b987c7514570f8ddc9309b7 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Tue, 23 May 2023 10:51:11 -0400
Subject: [PATCH 101/256] Adding ruff and removing flake8

---
 environment_python_3.10.yml | 2 +-
 environment_python_3.9.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index ea1eff17e..8e9de9f20 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -28,7 +28,6 @@ dependencies:
   - celery=5.2.7
   - cryptography=40.0.2
   - django=4.2.1
-  - flake8=6.0.0
   - inflection=0.5.1
   - ipython=8.13.2
   - jinja2=3.1.2
@@ -47,6 +46,7 @@ dependencies:
   - python=3.10.9
   - pyyaml=6.0
   - redis
+  - ruff=0.0.269
   - scipy=1.10.1
   - setuptools=67.7.2
   - sphinx=6.2.1
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index 564ecb0e6..27d1b79f9 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -28,7 +28,6 @@ dependencies:
   - celery=5.2.7
   - cryptography=40.0.2
   - django=4.2.1
-  - flake8=6.0.0
   - inflection=0.5.1
   - ipython=8.13.2
   - jinja2=3.1.2
@@ -47,6 +46,7 @@ dependencies:
   - python=3.9.16
   - pyyaml=6.0
   - redis
+  - ruff=0.0.269
   - scipy=1.10.1
   - setuptools=67.7.2
   - sphinx=6.2.1

From 2acadedace5bcf859a860e3caae9c3c614f868f8 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 23 May 2023 11:35:32 -0400
Subject: [PATCH 102/256] Make step_args optional in run_pipeline.py

---
 jwql/shared_tasks/run_pipeline.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index ba321eafa..6f4250349 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -317,7 +317,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     parser.add_argument('input_file', metavar='FILE', type=str, help=file_help)
     parser.add_argument('short_name', metavar='NAME', type=str, help=name_help)
     parser.add_argument('max_cores', metavar='CORES', type=str, help=cores_help)
-    parser.add_argument('step_args', metavar='STEP_ARGS', type=json.loads, help=step_args_help)
+    parser.add_argument('--step_args', metavar='STEP_ARGS', type=json.loads, default='{}', help=step_args_help)
 
     with open(general_status_file, "a+") as status_file:
         status_file.write("Created argument parser at {}\n".format(time.ctime()))

From 8b3ac9ef08e2c12aa17e48c9fc21fb2ecb696c3d Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 23 May 2023 12:00:10 -0400
Subject: [PATCH 103/256] Fix pipeline step test

---
 jwql/tests/test_pipeline_tools.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/jwql/tests/test_pipeline_tools.py b/jwql/tests/test_pipeline_tools.py
index 841cdb6d4..a79d687c8 100644
--- a/jwql/tests/test_pipeline_tools.py
+++ b/jwql/tests/test_pipeline_tools.py
@@ -107,9 +107,10 @@ def test_get_pipeline_steps():
 
     # MIRI
     miri_req_steps = pipeline_tools.get_pipeline_steps('miri')
-    miri_steps = ['dq_init', 'saturation', 'firstframe', 'lastframe',
-                  'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate']
-    not_required = ['group_scale', 'ipc', 'superbias', 'persistence']
+    miri_steps = ['group_scale', 'dq_init', 'saturation', 'firstframe', 'lastframe',
+                  'reset', 'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate',
+                  'gain_scale']
+    not_required = ['ipc', 'superbias', 'persistence']
     miri_dict = OrderedDict({})
     for step in miri_steps:
         miri_dict[step] = True

From ac4dde8866e5dbe42e8ffaf8ccd792976dcf8227 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 23 May 2023 12:15:03 -0400
Subject: [PATCH 104/256] Clean up

---
 jwql/instrument_monitors/pipeline_tools.py | 3 ---
 jwql/shared_tasks/run_pipeline.py          | 5 -----
 jwql/shared_tasks/shared_tasks.py          | 2 +-
 3 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/jwql/instrument_monitors/pipeline_tools.py b/jwql/instrument_monitors/pipeline_tools.py
index a5fd4a464..c39e3bda4 100644
--- a/jwql/instrument_monitors/pipeline_tools.py
+++ b/jwql/instrument_monitors/pipeline_tools.py
@@ -189,9 +189,6 @@ def get_pipeline_steps(instrument):
     required_steps = OrderedDict({})
     for key in steps:
         required_steps[key] = True
-    #for key in PIPE_KEYWORDS.values():
-    #    if key not in required_steps.keys():
-    #        required_steps[key] = False
 
     return required_steps
 
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 6f4250349..2e34f58e3 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -68,11 +68,6 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
             for step in steps:
                 steps[step] = not completed_steps[step]
 
-        # Special case: if the input file is a dark.fits file, then we want to skip the
-        # dark_current subtraction step
-        #if 'dark.fits' in input_file:
-        #    steps['dark_current'] = False
-
         # Make sure we don't run steps out of order. Find the latest step that has been
         # run, and only run subsequent steps. This protects against cases where some early
         # step was not run. In that case, we don't want to go back and run it because running
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 450405058..047084d62 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -258,7 +258,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     # Convert step_args dictionary to a string so that it can be passed via command line
     step_args_str = convert_step_args_to_string(step_args)
 
-    command = "{} {} {} '{}' {} {} {} {} {}"
+    command = "{} {} {} '{}' {} {} {} {} --step_args {}"
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
     logging.info("Running {}".format(command))
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)

From 756592f474f38b0e608ebd6a17816b2664c76521 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Tue, 23 May 2023 15:11:54 -0400
Subject: [PATCH 105/256] updating pyproject.toml

---
 pyproject.toml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index d941dbc56..f17cefdf9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -71,4 +71,6 @@ namespaces = false
 
 [tool.pytest]
 junit_family = "xunit2"
-addopts = "--ignore=jwql/website/apps/jwql/static"
+
+[tool.pytest.ini_options]
+norecursedirs = ["jwql/website/apps/jwql/static"]
\ No newline at end of file

From b35f6408af574b1bb222e80ee283f9ba132645b5 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 23 May 2023 17:17:36 -0400
Subject: [PATCH 106/256] Fix code duplication

---
 .../ta_monitors/wata_monitor.py               | 21 -------------------
 1 file changed, 21 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index 337001b4e..5850701ac 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -702,27 +702,6 @@ def get_uncal_names(self, file_list):
                 good_files.append(filename)
         return good_files
 
-    def get_uncal_names(self, file_list):
-        """Replace the last suffix for _uncal and return list.
-        Parameters
-        ----------
-        file_list : list
-            List of fits files
-        Returns
-        -------
-        good_files : list
-            Filtered list of uncal file names
-        """
-        good_files = []
-        for filename in file_list:
-            # Names look like: jw01133003001_02101_00001_nrs2_cal.fits
-            if '_uncal' not in filename:
-                suffix2replace = filename.split('_')[-1]
-                filename = filename.replace(suffix2replace, 'uncal.fits')
-            if filename not in good_files:
-                good_files.append(filename)
-        return good_files
-
     def update_ta_success_txtfile(self):
         """Create a text file with all the failed and successful WATA.
         Parameters

From 798691c0b066d2e4a8c0ce48399563798c35417f Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Thu, 25 May 2023 12:56:41 -0400
Subject: [PATCH 107/256] Fix for test with new versioning system

---
 jwql/tests/test_setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/tests/test_setup.py b/jwql/tests/test_setup.py
index 4cced8671..5eed720c8 100644
--- a/jwql/tests/test_setup.py
+++ b/jwql/tests/test_setup.py
@@ -29,4 +29,4 @@ def test_version_number():
 
     assert isinstance(jwql.__version__, str)
     version_parts = jwql.__version__.split('.')
-    assert len(version_parts) == 3
+    assert len(version_parts) >= 3

From 3caac1adccb09c5e3d9e2636b0bc846831026786 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Thu, 25 May 2023 13:33:50 -0400
Subject: [PATCH 108/256] Decouple TA monitor run from page load

---
 .../ta_monitors/msata_monitor.py              | 28 +++++++++++++++++-
 .../ta_monitors/wata_monitor.py               | 29 ++++++++++++++++++-
 jwql/website/apps/jwql/monitor_views.py       | 24 +++++++--------
 jwql/website/apps/jwql/static/js/jwql.js      |  6 ++--
 .../apps/jwql/templates/msata_monitor.html    | 17 ++---------
 .../apps/jwql/templates/wata_monitor.html     | 17 ++---------
 6 files changed, 73 insertions(+), 48 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index d3593ffcb..b8fd7ba70 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -1062,6 +1062,32 @@ def update_ta_success_txtfile(self):
                 line = "{:<50} {:<50} {:<50}".format(suc, ta_inprogress[idx], ta_failure[idx])
                 txt.write(line + "\n")
 
+    def read_existing_html(self):
+        """
+        This function gets the data from the Bokeh html file created with
+        the NIRSpec TA monitor script.
+        """
+        self.output_dir = os.path.join(get_config()['outputs'], 'msata_monitor')
+        ensure_dir_exists(self.output_dir)
+
+        self.output_file_name = os.path.join(self.output_dir, "msata_layout.html")
+        if not os.path.isfile(self.output_file_name):
+            return
+
+        # open the html file and get the contents
+        with open(self.output_file_name, "r") as html_file:
+            contents = html_file.read()
+
+        soup = BeautifulSoup(contents, 'html.parser').body
+
+        # find the script elements
+        script1 = str(soup.find('script', type='text/javascript'))
+        script2 = str(soup.find('script', type='application/json'))
+
+        # find the div element
+        div = str(soup.find('div', class_='bk-root'))
+        return div, script1, script2
+
     @log_fail
     @log_info
     def run(self):
@@ -1164,7 +1190,7 @@ def run(self):
             logging.info('\t{} MSATA files were used to make plots.'.format(msata_files_used4plots))
             # update the list of successful and failed TAs
             self.update_ta_success_txtfile()
-            logging.info('\t{} MSATA status file was updated')
+            logging.info('\tMSATA status file was updated')
         else:
             logging.info('\tMSATA monitor skipped.')
 
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index 5850701ac..b330e2dfc 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -749,6 +749,33 @@ def update_ta_success_txtfile(self):
                 line = "{:<50} {:<50}".format(suc, ta_failure[idx])
                 txt.write(line + "\n")
 
+    def read_existing_html(self):
+        """
+        This function gets the data from the Bokeh html file created with
+        the NIRSpec TA monitor script.
+        """
+        self.output_dir = os.path.join(get_config()['outputs'], 'wata_monitor')
+        ensure_dir_exists(self.output_dir)
+
+        self.output_file_name = os.path.join(self.output_dir, "wata_layout.html")
+        if not os.path.isfile(self.output_file_name):
+            return
+
+        # open the html file and get the contents
+        with open(self.output_file_name, "r") as html_file:
+            contents = html_file.read()
+
+        soup = BeautifulSoup(contents, 'html.parser').body
+
+        # find the script elements
+        script1 = str(soup.find('script', type='text/javascript'))
+        script2 = str(soup.find('script', type='application/json'))
+
+        # find the div element
+        div = str(soup.find('div', class_='bk-root'))
+        return div, script1, script2
+
+
     @log_fail
     @log_info
     def run(self):
@@ -849,7 +876,7 @@ def run(self):
             logging.info('\t{} WATA files were used to make plots.'.format(wata_files_used4plots))
             # update the list of successful and failed TAs
             self.update_ta_success_txtfile()
-            logging.info('\t{} WATA status file was updated')
+            logging.info('\tWATA status file was updated')
         else:
             logging.info('\tWATA monitor skipped.')
 
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index 23b5dafb0..b62f339de 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -270,15 +270,13 @@ def msata_monitoring_ajax(request):
     JsonResponse object
         Outgoing response sent to the webpage
     """
-    # run the monitor
-    module = 'msata_monitor'
-    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
+    # retrieve existing monitor html content
     monitor = msata_monitor.MSATA()
-    monitor.run()
-    monitor_utils.update_monitor_table(module, start_time, log_file)
+    div, script1, script2 = monitor.read_existing_html()
 
-    context = {'script': monitor.script,
-               'div': monitor.div}
+    context = {'script1': script1,
+               'script2': script2,
+               'div': div}
 
     return JsonResponse(context, json_dumps_params={'indent': 2})
 
@@ -321,14 +319,12 @@ def wata_monitoring_ajax(request):
     JsonResponse object
         Outgoing response sent to the webpage
     """
-    # run the monitor
-    module = 'wata_monitor'
-    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
+    # retrieve existing monitor html content
     monitor = wata_monitor.WATA()
-    monitor.run()
-    monitor_utils.update_monitor_table(module, start_time, log_file)
+    div, script1, script2 = monitor.read_existing_html()
 
-    context = {'script': monitor.script,
-               'div': monitor.div}
+    context = {'script1': script1,
+               'script2': script2,
+               'div': div}
 
     return JsonResponse(context, json_dumps_params={'indent': 2})
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index f4166d475..e86dfa9c6 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1001,7 +1001,8 @@ function update_msata_page(base_url) {
 
             // Build div content
             var content = data["div"];
-            content += data["script"];
+            content += data["script1"];
+            content += data["script2"];
 
             /* Add the content to the div
             *    Note: <script> elements inserted via innerHTML are intentionally disabled/ignored by the browser.  Directly inserting script via jquery.
@@ -1033,7 +1034,8 @@ function update_wata_page(base_url) {
 
             // Build div content
             var content = data["div"];
-            content += data["script"];
+            content += data["script1"];
+            content += data["script2"];
 
             /* Add the content to the div
             *    Note: <script> elements inserted via innerHTML are intentionally disabled/ignored by the browser.  Directly inserting script via jquery.
diff --git a/jwql/website/apps/jwql/templates/msata_monitor.html b/jwql/website/apps/jwql/templates/msata_monitor.html
index 3203f069b..0c3c597e1 100644
--- a/jwql/website/apps/jwql/templates/msata_monitor.html
+++ b/jwql/website/apps/jwql/templates/msata_monitor.html
@@ -13,22 +13,9 @@
 		<h1>{{ inst }} MSATA Monitor</h1>
         <!--Horizontal line-->
         <hr>
-        <!--Include an image located at jwql/website/apps/jwql/static/img/-->
-        <div style="text-align: center">
-          <img src={{ static('img/jwql_logo_full_transparent.png') }}
-               width=20%
-               alt="JWQL Logo"
-               title="JWQL Logo">
-        </div>
-
         <p id= "description">
-          <h2>Updates to this monitor only occur with new data</h2>
           This monitor uses all MSATA data available and the plots include all data.
-          The plots will only be updated if new MSATA data is in MAST, otherwise, the
-          monitor will show all data previously obtained.
-          <br>
-          <br>
-
+          The plots are updated regularly from new data in MAST.
         </p>
 
         <!-- Loading animation -->
@@ -44,7 +31,7 @@ <h2>Updates to this monitor only occur with new data</h2>
                         </div>
                     </div>
                 </div>
-                Searching MAST ...
+                Loading ...
             </div>
         </div>
 
diff --git a/jwql/website/apps/jwql/templates/wata_monitor.html b/jwql/website/apps/jwql/templates/wata_monitor.html
index d891e00bd..e61588af3 100644
--- a/jwql/website/apps/jwql/templates/wata_monitor.html
+++ b/jwql/website/apps/jwql/templates/wata_monitor.html
@@ -13,22 +13,9 @@
 		<h1>{{ inst }} WATA Monitor</h1>
         <!--Horizontal line-->
         <hr>
-        <!--Include an image located at jwql/website/apps/jwql/static/img/-->
-        <div style="text-align: center">
-          <img src={{ static('img/jwql_logo_full_transparent.png') }}
-               width=20%
-               alt="JWQL Logo"
-               title="JWQL Logo">
-        </div>
-
         <p id= "description">
-          <h2>Updates to this monitor only occur with new data</h2>
           This monitor uses all WATA data available and the plots include all data.
-          The plots will only be updated if new WATA data is in MAST, otherwise, the
-          monitor will show all data previously obtained.
-          <br>
-          <br>
-
+          The plots are updated regularly from new data in MAST.
         </p>
 
         <!-- Loading animation -->
@@ -44,7 +31,7 @@ <h2>Updates to this monitor only occur with new data</h2>
                         </div>
                     </div>
                 </div>
-                Searching MAST ...
+                Loading ...
             </div>
         </div>
 

From 8da7365ef8e3b44428921282a6b589c34b95e870 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Thu, 25 May 2023 16:52:06 -0400
Subject: [PATCH 109/256] Replace MAST query with Django model query. Fix bug
 with latest data being appended to plot multiple times.

---
 .../ta_monitors/msata_monitor.py              | 36 +++++++----
 .../ta_monitors/wata_monitor.py               | 39 ++++++++----
 jwql/utils/monitor_utils.py                   | 60 +++++++++++++++++++
 3 files changed, 113 insertions(+), 22 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index b8fd7ba70..4af161bae 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -34,11 +34,13 @@
 
 
 # general imports
+import json
 import os
 import logging
+import shutil
+
 import numpy as np
 import pandas as pd
-import json
 from bs4 import BeautifulSoup
 from datetime import datetime
 from astropy.time import Time
@@ -154,8 +156,6 @@ def get_tainfo_from_fits(self, fits_file):
                     msata = True
                     break
             if not msata:
-                # print('\n WARNING! This file is not MSATA: ', fits_file)
-                # print('  Skiping msata_monitor for this file  \n')
                 return None
             main_hdr = ff[0].header
             try:
@@ -988,6 +988,8 @@ def pull_filenames(self, file_info):
         for list_element in file_info:
             if 'filename' in list_element:
                 files.append(list_element['filename'])
+            elif 'root_name' in list_element:
+                files.append(list_element['root_name'])
         return files
 
     def get_uncal_names(self, file_list):
@@ -1003,10 +1005,13 @@ def get_uncal_names(self, file_list):
         """
         good_files = []
         for filename in file_list:
-            # Names look like: jw01133003001_02101_00001_nrs2_cal.fits
-            if '_uncal' not in filename:
+            if filename.endswith('.fits'):
+                # MAST names look like: jw01133003001_02101_00001_nrs2_cal.fits
                 suffix2replace = filename.split('_')[-1]
                 filename = filename.replace(suffix2replace, 'uncal.fits')
+            else:
+                # rootnames look like: jw01133003001_02101_00001_nrs2
+                filename += '_uncal.fits'
             if filename not in good_files:
                 good_files.append(filename)
         return good_files
@@ -1072,7 +1077,7 @@ def read_existing_html(self):
 
         self.output_file_name = os.path.join(self.output_dir, "msata_layout.html")
         if not os.path.isfile(self.output_file_name):
-            return
+            return 'No MSATA data available', '', ''
 
         # open the html file and get the contents
         with open(self.output_file_name, "r") as html_file:
@@ -1123,7 +1128,7 @@ def run(self):
             self.prev_data, self.query_start = self.prev_data2expected_format(prev_data_dict)
             logging.info('\tPrevious data read from html file: {}'.format(self.output_file_name))
             # move this plot to a previous version
-            os.rename(self.output_file_name, os.path.join(self.output_dir, "prev_msata_layout.html"))
+            shutil.copyfile(self.output_file_name, os.path.join(self.output_dir, "prev_msata_layout.html"))
         # fail save - start from the beginning if there is no html file
         else:
             self.query_start = self.query_very_beginning
@@ -1133,11 +1138,18 @@ def run(self):
         self.query_end = Time.now().mjd
         logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
 
-        # Query MAST using the aperture and the time of the
+        # Query for data using the aperture and the time of the
         # most recent previous search as the starting time
-        new_entries = monitor_utils.mast_query_ta(self.instrument, self.aperture, self.query_start, self.query_end)
+
+        # via MAST:
+        # new_entries = monitor_utils.mast_query_ta(
+        #     self.instrument, self.aperture, self.query_start, self.query_end)
+
+        # via django model:
+        new_entries = monitor_utils.model_query_ta(
+            self.instrument, self.aperture, self.query_start, self.query_end)
         msata_entries = len(new_entries)
-        logging.info('\tMAST query has returned {} MSATA files for {}, {}.'.format(msata_entries, self.instrument, self.aperture))
+        logging.info('\tQuery has returned {} MSATA files for {}, {}.'.format(msata_entries, self.instrument, self.aperture))
 
         # Filter new entries to only keep uncal files
         new_entries = self.pull_filenames(new_entries)
@@ -1148,6 +1160,10 @@ def run(self):
         # Get full paths to the files
         new_filenames = []
         for filename_of_interest in new_entries:
+            if (self.prev_data is not None
+                    and filename_of_interest in self.prev_data['filename'].values):
+                logging.warning('\t\tFile {} already in previous data. Skipping.'.format(filename_of_interest))
+                continue
             try:
                 new_filenames.append(filesystem_path(filename_of_interest))
                 logging.warning('\tFile {} included for processing.'.format(filename_of_interest))
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index b330e2dfc..e4bf27612 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -30,11 +30,13 @@
 
 
 # general imports
+import json
 import os
 import logging
+import shutil
+
 import numpy as np
 import pandas as pd
-import json
 from bs4 import BeautifulSoup
 from datetime import datetime
 from astropy.time import Time
@@ -148,8 +150,6 @@ def get_tainfo_from_fits(self, fits_file):
                     wata = True
                     break
             if not wata:
-                # print('\n WARNING! This file is not WATA: ', fits_file)
-                # print('  Skiping wata_monitor for this file  \n')
                 return None
             main_hdr = ff[0].header
             try:
@@ -679,6 +679,8 @@ def pull_filenames(self, file_info):
         for list_element in file_info:
             if 'filename' in list_element:
                 files.append(list_element['filename'])
+            elif 'root_name' in list_element:
+                files.append(list_element['root_name'])
         return files
 
     def get_uncal_names(self, file_list):
@@ -694,10 +696,13 @@ def get_uncal_names(self, file_list):
         """
         good_files = []
         for filename in file_list:
-            # Names look like: jw01133003001_02101_00001_nrs2_cal.fits
-            if '_uncal' not in filename:
+            if filename.endswith('.fits'):
+                # MAST names look like: jw01133003001_02101_00001_nrs2_cal.fits
                 suffix2replace = filename.split('_')[-1]
                 filename = filename.replace(suffix2replace, 'uncal.fits')
+            else:
+                # rootnames look like: jw01133003001_02101_00001_nrs2
+                filename += '_uncal.fits'
             if filename not in good_files:
                 good_files.append(filename)
         return good_files
@@ -759,7 +764,7 @@ def read_existing_html(self):
 
         self.output_file_name = os.path.join(self.output_dir, "wata_layout.html")
         if not os.path.isfile(self.output_file_name):
-            return
+            return 'No WATA data available', '', ''
 
         # open the html file and get the contents
         with open(self.output_file_name, "r") as html_file:
@@ -775,7 +780,6 @@ def read_existing_html(self):
         div = str(soup.find('div', class_='bk-root'))
         return div, script1, script2
 
-
     @log_fail
     @log_info
     def run(self):
@@ -809,8 +813,8 @@ def run(self):
         if os.path.isfile(self.output_file_name):
             self.prev_data, self.query_start = self.get_data_from_html(self.output_file_name)
             logging.info('\tPrevious data read from html file: {}'.format(self.output_file_name))
-            # move this plot to a previous version
-            os.rename(self.output_file_name, os.path.join(self.output_dir, "prev_wata_layout.html"))
+            # copy this plot to a previous version
+            shutil.copyfile(self.output_file_name, os.path.join(self.output_dir, "prev_wata_layout.html"))
         # fail save - start from the beginning if there is no html file
         else:
             self.query_start = self.query_very_beginning
@@ -820,11 +824,18 @@ def run(self):
         self.query_end = Time.now().mjd
         logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
 
-        # Query MAST using the aperture and the time of the
+        # Query for data using the aperture and the time of the
         # most recent previous search as the starting time
-        new_entries = monitor_utils.mast_query_ta(self.instrument, self.aperture, self.query_start, self.query_end)
+
+        # via MAST:
+        # new_entries = monitor_utils.mast_query_ta(
+        #     self.instrument, self.aperture, self.query_start, self.query_end)
+
+        # via django model:
+        new_entries = monitor_utils.model_query_ta(
+            self.instrument, self.aperture, self.query_start, self.query_end)
         wata_entries = len(new_entries)
-        logging.info('\tMAST query has returned {} WATA files for {}, {}.'.format(wata_entries, self.instrument, self.aperture))
+        logging.info('\tQuery has returned {} WATA files for {}, {}.'.format(wata_entries, self.instrument, self.aperture))
 
         # Filter new entries to only keep uncal files
         new_entries = self.pull_filenames(new_entries)
@@ -835,6 +846,10 @@ def run(self):
         # Get full paths to the files
         new_filenames = []
         for filename_of_interest in new_entries:
+            if (self.prev_data is not None
+                    and filename_of_interest in self.prev_data['filename'].values):
+                logging.warning('\t\tFile {} already in previous data. Skipping.'.format(filename_of_interest))
+                continue
             try:
                 new_filenames.append(filesystem_path(filename_of_interest))
                 logging.warning('\tFile {} included for processing.'.format(filename_of_interest))
diff --git a/jwql/utils/monitor_utils.py b/jwql/utils/monitor_utils.py
index e4bc13852..3b5297cc4 100644
--- a/jwql/utils/monitor_utils.py
+++ b/jwql/utils/monitor_utils.py
@@ -19,6 +19,7 @@
 import datetime
 import os
 from astroquery.mast import Mast, Observations
+from django import setup
 
 
 from jwql.database.database_interface import Monitor, engine
@@ -32,6 +33,21 @@
 # a MAST query.
 Mast._portal_api_connection.PAGESIZE = MAST_QUERY_LIMIT
 
+# Determine if the code is being run as part of a github action or Readthedocs build
+ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
+ON_READTHEDOCS = False
+if 'READTHEDOCS' in os.environ:  # pragma: no cover
+    ON_READTHEDOCS = os.environ['READTHEDOCS']
+
+if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    # These lines are needed in order to use the Django models in a standalone
+    # script (as opposed to code run as a result of a webpage request). If these
+    # lines are not run, the script will crash when attempting to import the
+    # Django models in the line below.
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    setup()
+    from jwql.website.apps.jwql.models import RootFileInfo
+
 
 def exclude_asic_tuning(mast_results):
     """Given a list of file information from a MAST query, filter out
@@ -206,6 +222,50 @@ def mast_query_ta(instrument, aperture, start_date, end_date, readpatt=None):
     return query_results
 
 
+def model_query_ta(instrument, aperture, start_date, end_date, readpatt=None):
+    """Use local Django model to search for TA data.
+
+    Parameters
+    ----------
+    instrument : str
+        Instrument name (e.g. ``nirspec``)
+    aperture : str
+        Detector aperture to search for (e.g. ``NRS_S1600A1_SLIT``)
+    start_date : float
+        Starting date for the search in MJD
+    end_date : float
+        Ending date for the search in MJD
+    readpatt : str
+        Readout pattern to search for (e.g. ``RAPID``). If None,
+        readout pattern will not be added to the query parameters.
+
+    Returns
+    -------
+    query_results : list
+        List of dictionaries containing the query results
+    """
+    if aperture == 'NRS_S1600A1_SLIT':
+        exp_types = ['NRS_TASLIT', 'NRS_BOTA', 'NRS_WATA']
+    else:
+        exp_types = ['NRS_TACQ', 'NRS_MSATA']
+
+    filter_kwargs = {
+        'instrument__iexact': instrument,
+        'aperture__iexact': aperture,
+        'exp_type__in': exp_types,
+        'expstart__gte': start_date,
+        'expstart__lte': end_date
+    }
+
+    if readpatt is not None:
+        filter_kwargs['readpatt'] = readpatt
+
+    # get file info by instrument from local model
+    root_file_info = RootFileInfo.objects.filter(**filter_kwargs)
+
+    return root_file_info.values()
+
+
 def update_monitor_table(module, start_time, log_file):
     """Update the ``monitor`` database table with information about
     the instrument monitor run

From 6e612788ba1ea668c1862d8d678b933bc99bcd3d Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 26 May 2023 12:10:58 -0400
Subject: [PATCH 110/256] Use DATE-BEG instead of DATE-OBS for time information

---
 .../nirspec_monitors/ta_monitors/msata_monitor.py         | 2 +-
 .../nirspec_monitors/ta_monitors/wata_monitor.py          | 2 +-
 jwql/tests/test_msata_monitor.py                          | 8 ++++++--
 jwql/tests/test_wata_monitor.py                           | 2 +-
 jwql/website/apps/jwql/templates/msata_monitor.html       | 2 +-
 jwql/website/apps/jwql/templates/wata_monitor.html        | 2 +-
 6 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index 4af161bae..206c9a7a5 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -98,7 +98,7 @@ def __init__(self):
 
         # dictionary to define required keywords to extract MSATA data and where it lives
         self.keywds2extract = {'FILENAME': {'loc': 'main_hdr', 'alt_key': None, 'name': 'filename', 'type': str},
-                               'DATE-OBS': {'loc': 'main_hdr', 'alt_key': None, 'name': 'date_obs', 'type': str},
+                               'DATE-BEG': {'loc': 'main_hdr', 'alt_key': None, 'name': 'date_obs', 'type': str},
                                'OBS_ID': {'loc': 'main_hdr', 'alt_key': None, 'name': 'visit_id', 'type': str},
                                'FILTER': {'loc': 'main_hdr', 'alt_key': 'FWA_POS', 'name': 'tafilter', 'type': str},
                                'DETECTOR': {'loc': 'main_hdr', 'alt_key': None, 'name': 'detector', 'type': str},
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index e4bf27612..6f53cf7fe 100644
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -93,7 +93,7 @@ def __init__(self):
 
         # structure to define required keywords to extract and where they live
         self.keywds2extract = {'FILENAME': {'loc': 'main_hdr', 'alt_key': None, 'name': 'filename', 'type': str},
-                               'DATE-OBS': {'loc': 'main_hdr', 'alt_key': None, 'name': 'date_obs'},
+                               'DATE-BEG': {'loc': 'main_hdr', 'alt_key': None, 'name': 'date_obs'},
                                'OBS_ID': {'loc': 'main_hdr', 'alt_key': 'OBSID', 'name': 'visit_id'},
                                'FILTER': {'loc': 'main_hdr', 'alt_key': 'FWA_POS', 'name': 'tafilter'},
                                'READOUT': {'loc': 'main_hdr', 'alt_key': 'READPATT', 'name': 'readout'},
diff --git a/jwql/tests/test_msata_monitor.py b/jwql/tests/test_msata_monitor.py
index 8650d5136..50c97db5c 100644
--- a/jwql/tests/test_msata_monitor.py
+++ b/jwql/tests/test_msata_monitor.py
@@ -18,6 +18,8 @@
     """
 
 import os
+import warnings
+
 import pandas as pd
 import numpy as np
 import pytest
@@ -108,7 +110,7 @@ def define_testdata():
             number_status.append(1.0)
             status_colors.append('blue')
         # convert time string into an array of time (this is in UT with 0.0 milliseconds)
-        t = datetime.fromisoformat(do_str)
+        t = datetime.fromisoformat(do_str).timestamp()
         time_arr.append(t)
     # add these to the bokeh data structure
     msata_dict['time_arr'] = time_arr
@@ -313,7 +315,9 @@ def test_mk_plt_layout():
 
     ta.output_file_name = os.path.join(ta.output_dir, "msata_layout.html")
     ta.msata_data = define_testdata()
-    script, div = ta.mk_plt_layout()
+    with warnings.catch_warnings():
+        warnings.simplefilter('error')
+        script, div = ta.mk_plt_layout()
 
     # set group write permission for the test file
     # to make sure others can overwrite it
diff --git a/jwql/tests/test_wata_monitor.py b/jwql/tests/test_wata_monitor.py
index 017f37da8..0ed420ebc 100644
--- a/jwql/tests/test_wata_monitor.py
+++ b/jwql/tests/test_wata_monitor.py
@@ -102,7 +102,7 @@ def define_testdata():
             bool_status.append(0)
             status_colors.append('red')
         # convert time string into an array of time (this is in UT with 0.0 milliseconds)
-        t = datetime.fromisoformat(do_str)
+        t = datetime.fromisoformat(do_str).timestamp()
         time_arr.append(t)
     # add these to the bokeh data structure
     wata_dict['time_arr'] = time_arr
diff --git a/jwql/website/apps/jwql/templates/msata_monitor.html b/jwql/website/apps/jwql/templates/msata_monitor.html
index 0c3c597e1..4d8640916 100644
--- a/jwql/website/apps/jwql/templates/msata_monitor.html
+++ b/jwql/website/apps/jwql/templates/msata_monitor.html
@@ -14,7 +14,7 @@ <h1>{{ inst }} MSATA Monitor</h1>
         <!--Horizontal line-->
         <hr>
         <p id= "description">
-          This monitor uses all MSATA data available and the plots include all data.
+          This monitor uses all MSATA data available.
           The plots are updated regularly from new data in MAST.
         </p>
 
diff --git a/jwql/website/apps/jwql/templates/wata_monitor.html b/jwql/website/apps/jwql/templates/wata_monitor.html
index e61588af3..662bb2542 100644
--- a/jwql/website/apps/jwql/templates/wata_monitor.html
+++ b/jwql/website/apps/jwql/templates/wata_monitor.html
@@ -14,7 +14,7 @@ <h1>{{ inst }} WATA Monitor</h1>
         <!--Horizontal line-->
         <hr>
         <p id= "description">
-          This monitor uses all WATA data available and the plots include all data.
+          This monitor uses all WATA data available.
           The plots are updated regularly from new data in MAST.
         </p>
 

From 55e69676b79ade21a51895a4ac015b99c1a57f3e Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 26 May 2023 12:23:19 -0400
Subject: [PATCH 111/256] Add execute permission for ta monitor scripts

---
 .../nirspec_monitors/ta_monitors/msata_monitor.py                 | 0
 .../nirspec_monitors/ta_monitors/wata_monitor.py                  | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
 mode change 100644 => 100755 jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
old mode 100644
new mode 100755
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
old mode 100644
new mode 100755

From bb89aaba220b10cd948981c565dffbb87baa4f3b Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 26 May 2023 14:31:01 -0400
Subject: [PATCH 112/256] Improve tooltip readability

---
 .../ta_monitors/msata_monitor.py              | 97 ++++++++++++-------
 .../ta_monitors/wata_monitor.py               | 36 ++++---
 2 files changed, 84 insertions(+), 49 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index 206c9a7a5..bc21a4643 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -38,28 +38,27 @@
 import os
 import logging
 import shutil
+from random import randint
 
 import numpy as np
 import pandas as pd
-from bs4 import BeautifulSoup
-from datetime import datetime
 from astropy.time import Time
 from astropy.io import fits
-from random import randint
-from sqlalchemy.sql.expression import and_
-from bokeh.plotting import figure, output_file, show, save
-from bokeh.models import ColumnDataSource, Range1d
-from bokeh.models.tools import HoverTool
-from bokeh.layouts import gridplot
-from bokeh.models import Span, Label
 from bokeh.embed import components
+from bokeh.plotting import figure, output_file, save
+from bokeh.layouts import gridplot
+from bokeh.models import ColumnDataSource, Range1d, Span, Label
+from bokeh.models.tools import HoverTool
+from bs4 import BeautifulSoup
+from datetime import datetime
+from sqlalchemy.sql.expression import and_
 
 # jwql imports
-from jwql.utils.logging_functions import log_info, log_fail
-from jwql.utils import monitor_utils
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.database.database_interface import session, engine
 from jwql.database.database_interface import NIRSpecTAQueryHistory, NIRSpecTAStats
+from jwql.utils import monitor_utils
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser
 
 
@@ -279,19 +278,21 @@ def plt_status(self):
             self.source.data["number_status"] = number_status
             self.source.data["status_colors"] = status_colors
         # create a new bokeh plot
-        plot = figure(title="MSATA Status [Succes=1, In_Progress=0.5, Fail=0]", x_axis_label='Time',
+        plot = figure(title="MSATA Status [Success=1, In Progress=0.5, Fail=0]", x_axis_label='Time',
                       y_axis_label='MSATA Status', x_axis_type='datetime',)
         plot.y_range = Range1d(-0.5, 1.5)
         plot.circle(x='time_arr', y='number_status', source=self.source,
                     color='status_colors', size=7, fill_alpha=0.3)
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('TA status', '@ta_status'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
                           ('Date-Obs', '@date_obs'),
-                          ('Subarray', '@subarray')]
+                          ('Subarray', '@subarray'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -326,7 +327,8 @@ def plt_residual_offsets(self):
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -334,7 +336,8 @@ def plt_residual_offsets(self):
                           ('Subarray', '@subarray'),
                           ('LS roll offset', '@lsrolloffset'),
                           ('LS V2 offset', '@lsv2offset'),
-                          ('LS V3 offset', '@lsv3offset')]
+                          ('LS V3 offset', '@lsv3offset'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -362,7 +365,8 @@ def plt_v2offset_time(self):
         hflabel = Label(x=time_arr[-1], y=-1 * v2halffacet[0], y_units='data', text='-V2 half-facet value')
         plot.add_layout(hflabel)
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -370,7 +374,8 @@ def plt_v2offset_time(self):
                           ('Subarray', '@subarray'),
                           ('LS roll offset', '@lsrolloffset'),
                           ('LS V2 offset', '@lsv2offset'),
-                          ('LS V3 offset', '@lsv3offset')]
+                          ('LS V3 offset', '@lsv3offset'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -398,7 +403,8 @@ def plt_v3offset_time(self):
         hflabel = Label(x=time_arr[-1], y=-1 * v3halffacet[0], y_units='data', text='-V3 half-facet value')
         plot.add_layout(hflabel)
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -406,7 +412,8 @@ def plt_v3offset_time(self):
                           ('Subarray', '@subarray'),
                           ('LS roll offset', '@lsrolloffset'),
                           ('LS V2 offset', '@lsv2offset'),
-                          ('LS V3 offset', '@lsv3offset')]
+                          ('LS V3 offset', '@lsv3offset'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -433,7 +440,8 @@ def plt_lsv2v3offsetsigma(self):
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -443,7 +451,8 @@ def plt_lsv2v3offsetsigma(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V2 sigma', '@lsv2sigma'),
                           ('LS V3 offset', '@lsv3offset'),
-                          ('LS V3 sigma', '@lsv3sigma')]
+                          ('LS V3 sigma', '@lsv3sigma'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -492,7 +501,8 @@ def plt_res_offsets_corrected(self):
         hflabel = Label(x=xstart / 3.0, y=ystart, y_units='data', text='-V2, -V3 half-facets values')
         plot.add_layout(hflabel)
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -502,7 +512,8 @@ def plt_res_offsets_corrected(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('V2 half-facet', '@v2halffacet'),
-                          ('V3 half-facet', '@v3halffacet')]
+                          ('V3 half-facet', '@v3halffacet'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -526,14 +537,16 @@ def plt_v2offsigma_time(self):
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
                           ('Date-Obs', '@date_obs'),
                           ('Subarray', '@subarray'),
                           ('LS V2 offset', '@lsv2offset'),
-                          ('LS V2 sigma', '@lsv2sigma')]
+                          ('LS V2 sigma', '@lsv2sigma'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -557,7 +570,8 @@ def plt_v3offsigma_time(self):
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -565,7 +579,8 @@ def plt_v3offsigma_time(self):
                           ('Subarray', '@subarray'),
                           ('LS roll offset', '@lsrolloffset'),
                           ('LS V3 offset', '@lsv3offset'),
-                          ('LS V3 sigma', '@lsv3sigma')]
+                          ('LS V3 sigma', '@lsv3sigma'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -595,7 +610,8 @@ def plt_roll_offset(self):
         plot.add_layout(arlabel)
         plot.renderers.extend([hline, arlinepos, arlineneg])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -603,7 +619,8 @@ def plt_roll_offset(self):
                           ('Subarray', '@subarray'),
                           ('LS roll offset', '@lsrolloffset'),
                           ('LS V2 offset', '@lsv2offset'),
-                          ('LS V3 offset', '@lsv3offset')]
+                          ('LS V3 offset', '@lsv3offset'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -627,7 +644,8 @@ def plt_lsoffsetmag(self):
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -636,7 +654,8 @@ def plt_lsoffsetmag(self):
                           ('LS roll offset', '@lsrolloffset'),
                           ('LS slew mag offset', '@lsoffsetmag'),
                           ('LS V2 offset', '@lsv2offset'),
-                          ('LS V3 offset', '@lsv3offset')]
+                          ('LS V3 offset', '@lsv3offset'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -682,7 +701,8 @@ def plt_tot_number_of_stars(self):
                       color='black', size=7, fill_alpha=0.3)
         plot.y_range = Range1d(0.0, 40.0)
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('Detector', '@detector'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -692,7 +712,8 @@ def plt_tot_number_of_stars(self):
                           ('LS roll offset', '@lsrolloffset'),
                           ('LS slew mag offset', '@lsoffsetmag'),
                           ('LS V2 offset', '@lsv2offset'),
-                          ('LS V3 offset', '@lsv3offset')]
+                          ('LS V3 offset', '@lsv3offset'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -774,14 +795,16 @@ def plt_mags_time(self):
         plot.y_range = Range1d(-1000.0, 62000.0)
         # add hover
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@vid'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@vid'),
                           ('Detector', '@det'),
                           ('Star No.', '@star_no'),
                           ('LS Status', '@status'),
                           ('Date-Obs', '@dobs'),
                           ('Box peak', '@peaks'),
                           ('Measured V2', '@stars_v2'),
-                          ('Measured V3', '@stars_v3')]
+                          ('Measured V3', '@stars_v3'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index 6f53cf7fe..01e52a44c 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -240,12 +240,14 @@ def plt_status(self):
                     color='status_colors', size=7, fill_alpha=0.3)
         # output_file("wata_status.html")
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('TA status', '@ta_status'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
                           ('Date-Obs', '@date_obs'),
-                          ('Magnitude', '@star_mag')]
+                          ('Magnitude', '@star_mag'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -271,12 +273,14 @@ def plt_residual_offsets(self):
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('TA status', '@ta_status'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
                           ('Date-Obs', '@date_obs'),
-                          ('Magnitude', '@star_mag')]
+                          ('Magnitude', '@star_mag'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -300,12 +304,14 @@ def plt_v2offset_time(self):
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('TA status', '@ta_status'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
                           ('Date-Obs', '@date_obs'),
-                          ('Magnitude', '@star_mag')]
+                          ('Magnitude', '@star_mag'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -329,12 +335,14 @@ def plt_v3offset_time(self):
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('TA status', '@ta_status'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
                           ('Date-Obs', '@date_obs'),
-                          ('Magnitude', '@star_mag')]
+                          ('Magnitude', '@star_mag'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -446,12 +454,14 @@ def plt_mag_time(self):
         plot.y_range = Range1d(-1000.0, 62000.0)
         # add hover
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('TA status', '@ta_status'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
                           ('Date-Obs', '@date_obs'),
-                          ('Box peak', '@max_val_box')]
+                          ('Box peak', '@max_val_box'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot
@@ -510,7 +520,8 @@ def plt_centroid(self):
         plot.x_range = Range1d(0.0, 32.0)
         plot.y_range = Range1d(0.0, 32.0)
         hover = HoverTool()
-        hover.tooltips = [('Visit ID', '@visit_id'),
+        hover.tooltips = [('File name', '@filename'),
+                          ('Visit ID', '@visit_id'),
                           ('TA status', '@ta_status'),
                           ('Filter', '@tafilter'),
                           ('Readout', '@readout'),
@@ -519,7 +530,8 @@ def plt_centroid(self):
                           ('Box Centr Col', '@corr_col'),
                           ('Box Centr Row', '@corr_row'),
                           ('Det Centr Col', '@detector_final_col'),
-                          ('Det Centr Row', '@detector_final_row')]
+                          ('Det Centr Row', '@detector_final_row'),
+                          ('--------', '----------------')]
 
         plot.add_tools(hover)
         return plot

From 4fe907f58a77ff70ed73877c3441b6e95ebc452e Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 26 May 2023 16:48:10 -0400
Subject: [PATCH 113/256] Reformat TA monitor instructions

---
 .../apps/jwql/templates/msata_monitor.html    | 35 ++++++++++---------
 .../apps/jwql/templates/wata_monitor.html     | 22 ++++++------
 2 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/jwql/website/apps/jwql/templates/msata_monitor.html b/jwql/website/apps/jwql/templates/msata_monitor.html
index 28843e938..b9038c1f5 100644
--- a/jwql/website/apps/jwql/templates/msata_monitor.html
+++ b/jwql/website/apps/jwql/templates/msata_monitor.html
@@ -13,23 +13,24 @@
 		<h1>{{ inst }} MSATA Monitor</h1>
         <!--Horizontal line-->
         <hr>
-        <p id= "description">
-          This monitor uses all MSATA data available.
-          The plots are updated regularly from new data in MAST.
-          <h2>Instructions</h2>
-          For general MSATA monitoring you only need to look at the first two plots
-          below: 1. MSATA Status and 2. MSATA Least Squares Residual V2-V3 Offsets.
-          Plot 1 shows the successful Visit IDs (blue points, which always correspond
-          to the second position - 1/2 shutter moved from the first), the unsuccessful
-          ones (red points), and the in-progress images (gray points, which always
-          correspond to the first position of the TA). MSATA can do up-to two tries
-          so you may see 4 entries for the same program ID in the gray points.
-          Plot 2 shows the V2-V3 least square residual offsets, the closer to the
-          purple cross (the half facet coordinates) the better. The default limits of
-          the plot are the acceptable values, if you see any point at the edges or
-          outside the limits of (-0.5, 0.5) for both axis, please let any of the
-          NIRSpec TA experts: Tracy Beck, Charles Proffit, and/or Tony Keyes.
-        </p>
+        <div id= "description">
+            <p>This monitor uses all MSATA data available.
+            The plots are updated regularly from new data in MAST.</p>
+
+            <h2>Instructions</h2>
+            <p>For general MSATA monitoring, you only need to look at the first two plots
+            below: 1. MSATA Status and 2. MSATA Least Squares Residual V2-V3 Offsets.</p>
+            <p>Plot 1 shows the successful Visit IDs (blue points, which always correspond
+            to the second position - 1/2 shutter moved from the first), the unsuccessful
+            ones (red points), and the in-progress images (gray points, which always
+            correspond to the first position of the TA). MSATA can do up-to two tries,
+            so you may see 4 entries for the same program ID in the gray points.</p>
+            <p>Plot 2 shows the V2-V3 least square residual offsets: the closer to the
+            purple cross (the half facet coordinates) the better. The default limits of
+            the plot are the acceptable values. If you see any point at the edges or
+            outside the limits of (-0.5, 0.5) for both axis, please let any of the
+            NIRSpec TA experts know: Tracy Beck, Charles Proffit, and/or Tony Keyes.</p>
+        </div>
 
         <!-- Loading animation -->
         <div id="loading">
diff --git a/jwql/website/apps/jwql/templates/wata_monitor.html b/jwql/website/apps/jwql/templates/wata_monitor.html
index ae8375722..142a43c69 100644
--- a/jwql/website/apps/jwql/templates/wata_monitor.html
+++ b/jwql/website/apps/jwql/templates/wata_monitor.html
@@ -13,20 +13,20 @@
 		<h1>{{ inst }} WATA Monitor</h1>
         <!--Horizontal line-->
         <hr>
-        <p id= "description">
-          This monitor uses all WATA data available.
-          The plots are updated regularly from new data in MAST.
+        <div id= "description">
+          <p>This monitor uses all WATA data available.
+          The plots are updated regularly from new data in MAST.</p>
 
           <h2>Instructions</h2>
-          For general WATA monitoring you only need to look at the first two plots
-          below: 1. WATA Status and 2. WATA Residual V2-V3 Offsets.
-          Plot 1 shows the successful Visit IDs (blue points) and the unsuccessful
-          ones (red points).
-          Plot 2 shows the V2-V3 residual offsets, the closer to (0,0) the better. The
-          default limits of the plot are the acceptable values, if you see any point
+          <p>For general WATA monitoring, you only need to look at the first two plots
+          below: 1. WATA Status and 2. WATA Residual V2-V3 Offsets.</p>
+          <p>Plot 1 shows the successful Visit IDs (blue points) and the unsuccessful
+          ones (red points).</p>
+          <p>Plot 2 shows the V2-V3 residual offsets: the closer to (0,0) the better. The
+          default limits of the plot are the acceptable values. If you see any point
           at the edges or outside the limits of (-0.5, 0.5) for both axis, please let
-          any of the NIRSpec TA experts: Tracy Beck, Charles Proffit, and/or Tony Keyes.
-        </p>
+          any of the NIRSpec TA experts know: Tracy Beck, Charles Proffit, and/or Tony Keyes.</p>
+        </div>
 
         <!-- Loading animation -->
         <div id="loading">

From b72cd1c34e78977d3e4db325d8dd0ae877379a67 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 30 May 2023 10:00:32 -0400
Subject: [PATCH 114/256] Add test for alternate query method

---
 jwql/tests/test_msata_monitor.py | 12 +++++++++++-
 jwql/tests/test_wata_monitor.py  | 14 ++++++++++++--
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/jwql/tests/test_msata_monitor.py b/jwql/tests/test_msata_monitor.py
index 50c97db5c..2a423d0ce 100644
--- a/jwql/tests/test_msata_monitor.py
+++ b/jwql/tests/test_msata_monitor.py
@@ -128,10 +128,20 @@ def test_mast_query_ta():
     query_start = 59833.0
     query_end = 59844.6
 
+    # query mast
     result = monitor_utils.mast_query_ta('nirspec', 'NRS_FULL_MSA', query_start, query_end)
-
     assert len(result) == 4
 
+    # query local model
+    alternate = monitor_utils.model_query_ta('nirspec', 'NRS_FULL_MSA', query_start, query_end)
+    assert len(alternate) == len(result)
+
+    # check that filenames match up - model returns rootfiles, mast returns filenames
+    result = sorted(result, key=lambda x: x['filename'])
+    alternate = sorted(alternate, key=lambda x: x['root_name'])
+    for i, rootfile in enumerate(alternate):
+        assert rootfile['root_name'] in result[i]['filename']
+
 
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_most_recent_search():
diff --git a/jwql/tests/test_wata_monitor.py b/jwql/tests/test_wata_monitor.py
index 0ed420ebc..a3bcc8157 100644
--- a/jwql/tests/test_wata_monitor.py
+++ b/jwql/tests/test_wata_monitor.py
@@ -114,15 +114,25 @@ def define_testdata():
 
 
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
-def test_mast_query_ta():
+def test_query_ta():
     """Test the ``query_mast`` function"""
 
     query_start = 59833.0
     query_end = 59844.6
 
+    # query mast
     result = monitor_utils.mast_query_ta('nirspec', 'NRS_S1600A1_SLIT', query_start, query_end)
+    assert len(result) == 16
 
-    assert len(result) >= 16
+    # query local model
+    alternate = monitor_utils.model_query_ta('nirspec', 'NRS_S1600A1_SLIT', query_start, query_end)
+    assert len(alternate) == len(result)
+
+    # check that filenames match up - model returns rootfiles, mast returns filenames
+    result = sorted(result, key=lambda x: x['filename'])
+    alternate = sorted(alternate, key=lambda x: x['root_name'])
+    for i, rootfile in enumerate(alternate):
+        assert rootfile['root_name'] in result[i]['filename']
 
 
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')

From 65c4caa934e03df4842f0fcb7d94e505ac1ad670 Mon Sep 17 00:00:00 2001
From: Bradley Sappington
 <101193271+BradleySappington@users.noreply.github.com>
Date: Wed, 31 May 2023 09:52:15 -0400
Subject: [PATCH 115/256] Update README.md

Remove 3.8 reference in example
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 2e458391c..5909b8889 100644
--- a/README.md
+++ b/README.md
@@ -86,13 +86,13 @@ source activate base/root
 Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
 
 ```
-conda env create -f environment_python_3_8.yml
+conda env create -f environment_python_3_9.yml
 ```
 
 or
 
 ```
-conda env create -f environment_python_3_9.yml
+conda env create -f environment_python_3_10.yml
 ```
 
 ### Configuration File

From 78708ad940aec91d46f0dfb3baf01ddf9423d90c Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Wed, 31 May 2023 10:09:47 -0500
Subject: [PATCH 116/256] Updating requirements and conda files to match

---
 environment_python_3.10.yml |  2 +-
 environment_python_3.9.yml  |  2 +-
 requirements.txt            | 32 ++++++++++++++++----------------
 rtd_requirements.txt        |  7 +++----
 4 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 8e9de9f20..207f9d658 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -47,7 +47,7 @@ dependencies:
   - pyyaml=6.0
   - redis
   - ruff=0.0.269
-  - scipy=1.10.1
+  - scipy=1.9.3
   - setuptools=67.7.2
   - sphinx=6.2.1
   - sphinx_rtd_theme=1.2.0
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index 27d1b79f9..cf20b4108 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -47,7 +47,7 @@ dependencies:
   - pyyaml=6.0
   - redis
   - ruff=0.0.269
-  - scipy=1.10.1
+  - scipy=1.9.3
   - setuptools=67.7.2
   - sphinx=6.2.1
   - sphinx_rtd_theme=1.2.0
diff --git a/requirements.txt b/requirements.txt
index d4bc70951..f5e4571cf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,37 +1,37 @@
-astropy==5.2.1
+astropy==5.2.2
 astroquery==0.4.6
-bandit==1.7.4
-beautifulsoup4==4.11.2
+bandit==1.7.5
+beautifulsoup4==4.12.2
 bokeh==2.4.3
 celery==5.2.7
-cryptography==39.0.1
-django==4.1.7
-flake8==6.0.0
+cryptography==40.0.2
+django==4.2.1
 inflection==0.5.1
-ipython==8.10.0
+ipython==8.13.2
 jinja2==3.1.2
 jsonschema==4.17.3
-jwst==1.10.0
+jwst==1.10.2
 matplotlib==3.7.0
 nodejs==0.1.1
-numpy==1.24.2
+numpy==1.24.3
 numpydoc==1.5.0
-pandas==2.0.0
+pandas==2.0.1
 psycopg2-binary==2.9.3
 pysiaf==0.19.1
 pysqlite3==0.5.0
-pytest==7.2.1
+pytest==7.3.1
 pytest-cov==4.0.0
 pytest-mock==3.10.0
-pyvo==1.4
+pyvo==1.4.1
 pyyaml==6.0
 redis==4.5.1
+ruff==0.0.269
 scipy==1.9.3
-selenium==4.8.0
-setuptools==67.3.1
-sphinx==6.1.3
+selenium==4.9.1
+setuptools==67.7.2
+sphinx==6.2.1
 sphinx_rtd_theme==1.2.0
-sqlalchemy==2.0.8
+sqlalchemy==2.0.15
 stdatamodels==1.3.1
 stsci_rtd_theme==1.0.0
 twine==4.0.2
diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index d3f595ab7..d2dd3c394 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -2,12 +2,11 @@ sphinx_automodapi==0.14.1
 bokeh==2.4.3
 celery==5.2.7
 cython==0.29.33
-django==4.1.7
+django==4.2.1
 docutils==0.18.1
-flake8==6.0.0
-jwst==1.10.0
+jwst==1.10.2
 pygments==2.14.0
-pytest==7.2.1
+pytest==7.3.1
 redis==4.5.1
 sphinx>=2
 sphinx_rtd_theme==1.2.0

From 85087f983a9279dafad10361ab4c46f4ed11753e Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Wed, 31 May 2023 11:11:20 -0500
Subject: [PATCH 117/256] Upgrading redis versions

---
 requirements.txt     | 2 +-
 rtd_requirements.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index f5e4571cf..501641339 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,7 @@ pytest-cov==4.0.0
 pytest-mock==3.10.0
 pyvo==1.4.1
 pyyaml==6.0
-redis==4.5.1
+redis==4.5.5
 ruff==0.0.269
 scipy==1.9.3
 selenium==4.9.1
diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index d2dd3c394..ba1ecf587 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -7,7 +7,7 @@ docutils==0.18.1
 jwst==1.10.2
 pygments==2.14.0
 pytest==7.3.1
-redis==4.5.1
+redis==4.5.5
 sphinx>=2
 sphinx_rtd_theme==1.2.0
 stsci_rtd_theme==1.0.0

From e5a23cd8b2971326daff908fb387ce007fa5ebf7 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 31 May 2023 13:25:45 -0400
Subject: [PATCH 118/256] Initial Commit

---
 jwql/jwql_monitors/clean_locks.py | 90 +++++++++++++++++++++++++++++++
 1 file changed, 90 insertions(+)
 create mode 100644 jwql/jwql_monitors/clean_locks.py

diff --git a/jwql/jwql_monitors/clean_locks.py b/jwql/jwql_monitors/clean_locks.py
new file mode 100644
index 000000000..6c6becba5
--- /dev/null
+++ b/jwql/jwql_monitors/clean_locks.py
@@ -0,0 +1,90 @@
+#! /usr/bin/env python
+
+"""Evaluate and clean all *.lock files in the current directory.
+
+This script is used as protection for scripts using the `jwql/utils/protect_module.py::lock_module` decorator
+
+There are infrequent instances where the MODULE_NAME.lock file will linger despite the process no longer running.
+This script will
+    Cycle through all *.lock files generated from the lock_module decorator
+    Retreive their associated PIDs
+    Delete any lock file who's associated PID is no longer running.
+    Email confirmation of any files that were deleted
+
+Authors
+-------
+
+    - Bradley Sappington
+
+
+Use
+---
+
+    This script is intended to be run as an hourly cronjob as protection for generate_preview_images.py
+
+    ::
+
+        python clean_locks.py
+"""
+import glob
+import os
+
+from psutil import pid_exists
+
+# SAPP TODO - MAKE NOTE TO add psutil to env in dev/test/prod
+# SAPP TODO - Consider making LOCK_FILE_PID_KEY in protect_module.py to keep this method safe if there is ever a change to "Process Id = ".
+
+
+def clean_lock_file(filename):
+    try:
+        pid = retreive_pid_from_lock_file(filename)
+        if not pid_exists(pid):
+            # if PID associated with the lock file is no longer running, then lock file should not exist, delete it.
+            if os.path.exists(filename):
+                os.remove(filename)
+                print(f"File '{filename}' deleted successfully.")  # SAPP TODO -DEL THIS
+                return f"File '{filename}' was never deleted and associated PID no longer running, deleted successfully."
+            else:
+                print(f"File '{filename}' does not exist.")  # SAPP TODO -DEL THIS
+    except SyntaxError as e:
+        return e
+
+
+def get_all_lock_files(search_path):
+    if search_path is None:
+        search_path = "."
+
+    lock_files = glob.glob(search_path + "/*.lock")
+    return lock_files
+
+
+def retreive_pid_from_lock_file(filename):
+    # Lock file format is established in `jwql/utils/protect_module.py::lock_module`
+
+    with open(filename, 'r') as file:
+        for line in file:
+            if "Process Id = " in line:
+                number_index = line.index("Process Id = ") + len("Process Id = ")
+                number = line[number_index:].strip()
+                print(f"YES PID FOUND IN {filename}!!!!")  # SAPP TODO -DEL THIS
+                return int(number)
+
+    print(f"NO PID FOUND IN {filename}")  # SAPP TODO -DEL THIS
+    raise SyntaxError(f"No PID found in {filename}")
+
+
+if __name__ == '__main__':
+    email_data = []
+
+    lock_files = get_all_lock_files()
+
+    for file in lock_files:
+        notify_str = clean_lock_file(file)
+        if len(notify_str):
+            email_data.append(notify_str)
+
+    # Email all lock files that were deleted
+    # SAPP TODO - create email to send out!
+    print("EMAIL THIS: ")
+    for line in email_data:
+        print(line)

From d83f72205db6f94a8bc040ee4850331794abc9a4 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 31 May 2023 13:28:23 -0400
Subject: [PATCH 119/256] default search_path until params

---
 jwql/jwql_monitors/clean_locks.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/jwql/jwql_monitors/clean_locks.py b/jwql/jwql_monitors/clean_locks.py
index 6c6becba5..7fe243d48 100644
--- a/jwql/jwql_monitors/clean_locks.py
+++ b/jwql/jwql_monitors/clean_locks.py
@@ -50,10 +50,7 @@ def clean_lock_file(filename):
         return e
 
 
-def get_all_lock_files(search_path):
-    if search_path is None:
-        search_path = "."
-
+def get_all_lock_files(search_path="."):
     lock_files = glob.glob(search_path + "/*.lock")
     return lock_files
 

From ad2bc703832a225792cbf31a883c46fe499e4066 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 31 May 2023 14:36:59 -0400
Subject: [PATCH 120/256] add (p)ath argument

---
 jwql/jwql_monitors/clean_locks.py | 63 +++++++++++++++++++++++++------
 1 file changed, 52 insertions(+), 11 deletions(-)

diff --git a/jwql/jwql_monitors/clean_locks.py b/jwql/jwql_monitors/clean_locks.py
index 7fe243d48..b51e10366 100644
--- a/jwql/jwql_monitors/clean_locks.py
+++ b/jwql/jwql_monitors/clean_locks.py
@@ -20,18 +20,20 @@
 Use
 ---
 
-    This script is intended to be run as an hourly cronjob as protection for generate_preview_images.py
+    This script is intended to be run as a frequent cronjob as protection for generate_preview_images.py
 
     ::
 
         python clean_locks.py
+        or
+        python clean_locks.py -p "/directory/to/check"
 """
+import argparse
 import glob
 import os
 
 from psutil import pid_exists
 
-# SAPP TODO - MAKE NOTE TO add psutil to env in dev/test/prod
 # SAPP TODO - Consider making LOCK_FILE_PID_KEY in protect_module.py to keep this method safe if there is ever a change to "Process Id = ".
 
 
@@ -44,10 +46,9 @@ def clean_lock_file(filename):
                 os.remove(filename)
                 print(f"File '{filename}' deleted successfully.")  # SAPP TODO -DEL THIS
                 return f"File '{filename}' was never deleted and associated PID no longer running, deleted successfully."
-            else:
-                print(f"File '{filename}' does not exist.")  # SAPP TODO -DEL THIS
+        return ""
     except SyntaxError as e:
-        return e
+        return str(e)
 
 
 def get_all_lock_files(search_path="."):
@@ -55,25 +56,50 @@ def get_all_lock_files(search_path="."):
     return lock_files
 
 
+def parse_args():
+    """ Parse command line arguments.
+
+    Parameters:
+
+    Returns:
+        arguments: argparse.Namespace object
+            An object containing all of the added arguments.
+
+    Outputs:
+        Nothing
+    """
+    path_help = ('Directory where you would like to search for *.lock files.'
+                 ' Default is current directory')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-p',
+                        type=str,
+                        help=path_help,
+                        action='store',
+                        required=False,
+                        default='.')
+
+    arguments = parser.parse_args()
+
+    return arguments
+
+
 def retreive_pid_from_lock_file(filename):
     # Lock file format is established in `jwql/utils/protect_module.py::lock_module`
-
     with open(filename, 'r') as file:
         for line in file:
             if "Process Id = " in line:
                 number_index = line.index("Process Id = ") + len("Process Id = ")
                 number = line[number_index:].strip()
-                print(f"YES PID FOUND IN {filename}!!!!")  # SAPP TODO -DEL THIS
                 return int(number)
 
-    print(f"NO PID FOUND IN {filename}")  # SAPP TODO -DEL THIS
-    raise SyntaxError(f"No PID found in {filename}")
+    raise SyntaxError(f"No PID found in {filename} - Please manually investigate and delete if appropriate")
 
 
 if __name__ == '__main__':
     email_data = []
-
-    lock_files = get_all_lock_files()
+    args = parse_args()
+    lock_files = get_all_lock_files(args.p)
 
     for file in lock_files:
         notify_str = clean_lock_file(file)
@@ -85,3 +111,18 @@ def retreive_pid_from_lock_file(filename):
     print("EMAIL THIS: ")
     for line in email_data:
         print(line)
+
+        # SAPP TODO - implement this
+"""
+        deliverer = '{}@stsci.edu'.format(user)
+
+        message = MIMEText(success_message)
+        message['Subject'] = 'JWQL ACTION FROM `CLEAN_LOCKS` SCRIPT'
+        message['From'] = deliverer
+        message['To'] = 'redcat@stsci.edu'
+        message['CC'] = deliverer
+
+        s = smtplib.SMTP('smtp.stsci.edu')
+        s.send_message(message)
+        s.quit()
+    """

From 43d1836cab4cdd428cb62c81a3080f91ab75196f Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 31 May 2023 14:48:17 -0400
Subject: [PATCH 121/256] make PID_LOCKFILE_KEY constant for modules

---
 jwql/jwql_monitors/clean_locks.py | 8 +++-----
 jwql/utils/protect_module.py      | 7 ++++++-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/jwql/jwql_monitors/clean_locks.py b/jwql/jwql_monitors/clean_locks.py
index b51e10366..8e01d7fc3 100644
--- a/jwql/jwql_monitors/clean_locks.py
+++ b/jwql/jwql_monitors/clean_locks.py
@@ -32,10 +32,9 @@
 import glob
 import os
 
+from jwql.utils.protect_module import PID_LOCKFILE_KEY
 from psutil import pid_exists
 
-# SAPP TODO - Consider making LOCK_FILE_PID_KEY in protect_module.py to keep this method safe if there is ever a change to "Process Id = ".
-
 
 def clean_lock_file(filename):
     try:
@@ -44,7 +43,6 @@ def clean_lock_file(filename):
             # if PID associated with the lock file is no longer running, then lock file should not exist, delete it.
             if os.path.exists(filename):
                 os.remove(filename)
-                print(f"File '{filename}' deleted successfully.")  # SAPP TODO -DEL THIS
                 return f"File '{filename}' was never deleted and associated PID no longer running, deleted successfully."
         return ""
     except SyntaxError as e:
@@ -88,8 +86,8 @@ def retreive_pid_from_lock_file(filename):
     # Lock file format is established in `jwql/utils/protect_module.py::lock_module`
     with open(filename, 'r') as file:
         for line in file:
-            if "Process Id = " in line:
-                number_index = line.index("Process Id = ") + len("Process Id = ")
+            if PID_LOCKFILE_KEY in line:
+                number_index = line.index(PID_LOCKFILE_KEY) + len(PID_LOCKFILE_KEY)
                 number = line[number_index:].strip()
                 return int(number)
 
diff --git a/jwql/utils/protect_module.py b/jwql/utils/protect_module.py
index ed55b7c02..9ea60790a 100644
--- a/jwql/utils/protect_module.py
+++ b/jwql/utils/protect_module.py
@@ -58,6 +58,10 @@ def protected_code():
 from functools import wraps
 
 
+# This key is defined here because it is utilized in other modules
+PID_LOCKFILE_KEY = "Process Id = "
+
+
 def lock_module(func):
     """Decorator to prevent more than 1 instance of a module.
 
@@ -76,6 +80,7 @@ def lock_module(func):
         The wrapped function.
     """
 
+
     @wraps(func)
     def wrapped(*args, **kwargs):
 
@@ -96,7 +101,7 @@ def wrapped(*args, **kwargs):
         else:
             try:
                 with open(module_lock, "w") as lock_file:
-                    lock_file.write(f"Process Id = {os.getpid()}\n")
+                    lock_file.write(f"{PID_LOCKFILE_KEY}{os.getpid()}\n")
                 return func(*args, **kwargs)
             finally:
                 try:

From 8620e6a73a9987534abcafea830a0ca797c9999d Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 31 May 2023 15:11:27 -0400
Subject: [PATCH 122/256] Update Messages

---
 jwql/jwql_monitors/clean_locks.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/jwql/jwql_monitors/clean_locks.py b/jwql/jwql_monitors/clean_locks.py
index 8e01d7fc3..c2a1e109c 100644
--- a/jwql/jwql_monitors/clean_locks.py
+++ b/jwql/jwql_monitors/clean_locks.py
@@ -43,7 +43,8 @@ def clean_lock_file(filename):
             # if PID associated with the lock file is no longer running, then lock file should not exist, delete it.
             if os.path.exists(filename):
                 os.remove(filename)
-                return f"File '{filename}' was never deleted and associated PID no longer running, deleted successfully."
+                return (f"DELETED FILE `{filename}` - This file's associated PID was no longer running.\n"
+                        f"This lock file would have blocked future instances of `{filename.replace('.lock', '.py')}`")
         return ""
     except SyntaxError as e:
         return str(e)
@@ -95,22 +96,21 @@ def retreive_pid_from_lock_file(filename):
 
 
 if __name__ == '__main__':
-    email_data = []
+    email_data = "The JWQL project CLEAN_LOCKS.PY has recently run and performed actions that you should be aware of:\n"
     args = parse_args()
     lock_files = get_all_lock_files(args.p)
 
     for file in lock_files:
         notify_str = clean_lock_file(file)
         if len(notify_str):
-            email_data.append(notify_str)
+            email_data = email_data + notify_str + "\n"
 
     # Email all lock files that were deleted
     # SAPP TODO - create email to send out!
     print("EMAIL THIS: ")
-    for line in email_data:
-        print(line)
+    print(email_data)
 
-        # SAPP TODO - implement this
+    # SAPP TODO - implement this
 """
         deliverer = '{}@stsci.edu'.format(user)
 

From ff1057d5cc6e0b82962689bdad74de7a2e3d6694 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 31 May 2023 15:11:59 -0400
Subject: [PATCH 123/256] fix lint error

---
 jwql/utils/protect_module.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/jwql/utils/protect_module.py b/jwql/utils/protect_module.py
index 9ea60790a..ea5ef295e 100644
--- a/jwql/utils/protect_module.py
+++ b/jwql/utils/protect_module.py
@@ -80,7 +80,6 @@ def lock_module(func):
         The wrapped function.
     """
 
-
     @wraps(func)
     def wrapped(*args, **kwargs):
 

From 4c6c645d2a7fdaf6d9826aaa915e5fbb29899bc4 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 31 May 2023 17:11:56 -0400
Subject: [PATCH 124/256] Add testing email

---
 jwql/jwql_monitors/clean_locks.py | 93 ++++++++++++++++++++++---------
 1 file changed, 67 insertions(+), 26 deletions(-)

diff --git a/jwql/jwql_monitors/clean_locks.py b/jwql/jwql_monitors/clean_locks.py
index c2a1e109c..f2e47d23d 100644
--- a/jwql/jwql_monitors/clean_locks.py
+++ b/jwql/jwql_monitors/clean_locks.py
@@ -29,9 +29,13 @@
         python clean_locks.py -p "/directory/to/check"
 """
 import argparse
+import getpass
 import glob
 import os
+import smtplib
+import socket
 
+from email.mime.text import MIMEText
 from jwql.utils.protect_module import PID_LOCKFILE_KEY
 from psutil import pid_exists
 
@@ -51,6 +55,19 @@ def clean_lock_file(filename):
 
 
 def get_all_lock_files(search_path="."):
+    '''This function returns a list of all lock files in a given directory.
+
+    Parameters
+    ----------
+    search_path, optional
+        The directory path where the function will search for lock files. By default, it is set to the
+        current directory (".") but it can be changed to any other directory path.
+
+    Returns
+    -------
+        list: file paths for all files with the extension ".lock" in the search path
+
+    '''
     lock_files = glob.glob(search_path + "/*.lock")
     return lock_files
 
@@ -58,14 +75,14 @@ def get_all_lock_files(search_path="."):
 def parse_args():
     """ Parse command line arguments.
 
-    Parameters:
+    Parameters
+    ----------
+        None
 
-    Returns:
-        arguments: argparse.Namespace object
-            An object containing all of the added arguments.
+    Returns
+    ----------
+        argparse.Namespace object: An object containing all of the added arguments.
 
-    Outputs:
-        Nothing
     """
     path_help = ('Directory where you would like to search for *.lock files.'
                  ' Default is current directory')
@@ -84,6 +101,21 @@ def parse_args():
 
 
 def retreive_pid_from_lock_file(filename):
+    '''This function retrieves a process ID from a lock file.
+
+    Parameters
+    ----------
+    filename
+        The filename parameter is a string that represents the name of the lock file from which the process
+        ID (PID) needs to be retrieved.
+
+    Returns
+    -------
+        int: The process ID (PID)
+        or
+        SyntaxError: Indicating that the file should be manually investigated and deleted if appropriate.
+
+    '''
     # Lock file format is established in `jwql/utils/protect_module.py::lock_module`
     with open(filename, 'r') as file:
         for line in file:
@@ -95,32 +127,41 @@ def retreive_pid_from_lock_file(filename):
     raise SyntaxError(f"No PID found in {filename} - Please manually investigate and delete if appropriate")
 
 
+def send_notification(message):
+    '''Sends an email notification to JWQL team alerting them of script actually solving issue (or not)
+
+    Parameters
+    ----------
+    message
+        The message to be included in the email notification that will be sent out.
+
+    '''
+
+    user = getpass.getuser()
+    hostname = socket.gethostname()
+    deliverer = '{}@stsci.edu'.format(user)
+
+    message = MIMEText(message)
+    message['Subject'] = f'JWQL ALERT FOR `CLEAN_LOCKS` SCRIPT ON {hostname}'
+    message['From'] = deliverer
+    message['To'] = 'bsappington@stsci.edu'  # SAPP TODO - CHANGE TO JWQL@stsci.edu after testing
+
+    s = smtplib.SMTP('smtp.stsci.edu')
+    s.send_message(message)
+    s.quit()
+
+
 if __name__ == '__main__':
-    email_data = "The JWQL project CLEAN_LOCKS.PY has recently run and performed actions that you should be aware of:\n"
+    email_data = "The JWQL project's CLEAN_LOCKS.PY has recently run and performed actions that you should be aware of:\n\n"
     args = parse_args()
     lock_files = get_all_lock_files(args.p)
+    notify_team = False
 
     for file in lock_files:
         notify_str = clean_lock_file(file)
         if len(notify_str):
+            notify_team = True
             email_data = email_data + notify_str + "\n"
 
-    # Email all lock files that were deleted
-    # SAPP TODO - create email to send out!
-    print("EMAIL THIS: ")
-    print(email_data)
-
-    # SAPP TODO - implement this
-"""
-        deliverer = '{}@stsci.edu'.format(user)
-
-        message = MIMEText(success_message)
-        message['Subject'] = 'JWQL ACTION FROM `CLEAN_LOCKS` SCRIPT'
-        message['From'] = deliverer
-        message['To'] = 'redcat@stsci.edu'
-        message['CC'] = deliverer
-
-        s = smtplib.SMTP('smtp.stsci.edu')
-        s.send_message(message)
-        s.quit()
-    """
+    if notify_team:
+        send_notification(email_data)

From 3a96d9873c943ccd9f9255853ba8243f7fa576d5 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Thu, 1 Jun 2023 13:48:36 -0400
Subject: [PATCH 125/256] put cleaning code in protect_module.py

---
 jwql/jwql_monitors/clean_locks.py |   4 ++
 jwql/utils/protect_module.py      | 107 ++++++++++++++++++++++++++----
 2 files changed, 97 insertions(+), 14 deletions(-)

diff --git a/jwql/jwql_monitors/clean_locks.py b/jwql/jwql_monitors/clean_locks.py
index f2e47d23d..5aa167d20 100644
--- a/jwql/jwql_monitors/clean_locks.py
+++ b/jwql/jwql_monitors/clean_locks.py
@@ -28,6 +28,8 @@
         or
         python clean_locks.py -p "/directory/to/check"
 """
+# SAPP TODO - NO LONGER NEED THIS MODULE!
+
 import argparse
 import getpass
 import glob
@@ -36,12 +38,14 @@
 import socket
 
 from email.mime.text import MIMEText
+from jwql.utils import permissions
 from jwql.utils.protect_module import PID_LOCKFILE_KEY
 from psutil import pid_exists
 
 
 def clean_lock_file(filename):
     try:
+        permissions.set_permissions(filename)
         pid = retreive_pid_from_lock_file(filename)
         if not pid_exists(pid):
             # if PID associated with the lock file is no longer running, then lock file should not exist, delete it.
diff --git a/jwql/utils/protect_module.py b/jwql/utils/protect_module.py
index ea5ef295e..79495554c 100644
--- a/jwql/utils/protect_module.py
+++ b/jwql/utils/protect_module.py
@@ -52,14 +52,90 @@ def protected_code():
 
     None
 """
-
-import os
 import inspect
+import getpass
+import os
+import smtplib
+import socket
+
+from email.mime.text import MIMEText
 from functools import wraps
+from psutil import pid_exists
+
+_PID_LOCKFILE_KEY = "Process Id = "
+
+
+def _clean_lock_file(filename, module):
+    locked = True
+    try:
+        pid = _retreive_pid_from_lock_file(filename)
+        notify_str = ""
+        if not pid_exists(pid):
+            # if PID associated with the lock file is no longer running, then lock file should not exist, delete it.
+            if os.path.exists(filename):
+                try:
+                    os.remove(filename)
+                    notify_str = (f"DELETED FILE `{filename}`\nThis file's associated PID was no longer running.\n\n"
+                                  f"This implies the previous instance of {module} may not have completed successfully.\n\n"
+                                  f"New instance starting now.")
+                    locked = False
+                except Exception as e:
+                    notify_str = f"Exception {e} \n {type(e).__name__}\n{e.args}\n\n"
+                    notify_str = notify_str + filename + " delete failed, Please Manually Delete"
+        return notify_str, locked
+    except SyntaxError as e:
+        return str(e), locked
 
 
-# This key is defined here because it is utilized in other modules
-PID_LOCKFILE_KEY = "Process Id = "
+def _retreive_pid_from_lock_file(filename):
+    '''This function retrieves a process ID from a lock file.
+
+    Parameters
+    ----------
+    filename
+        The filename parameter is a string that represents the name of the lock file from which the process
+        ID (PID) needs to be retrieved.
+
+    Returns
+    -------
+        int: The process ID (PID)
+        or
+        SyntaxError: Indicating that the file should be manually investigated and deleted if appropriate.
+
+    '''
+    # Lock file format is established in `jwql/utils/protect_module.py::lock_module`
+    with open(filename, 'r') as file:
+        for line in file:
+            if _PID_LOCKFILE_KEY in line:
+                number_index = line.index(_PID_LOCKFILE_KEY) + len(_PID_LOCKFILE_KEY)
+                number = line[number_index:].strip()
+                return int(number)
+
+    raise SyntaxError(f"No PID found in {filename} - Please manually investigate and delete if appropriate")
+
+
+def _send_notification(message):
+    '''Sends an email notification to JWQL team alerting them of script actually solving issue (or not)
+
+    Parameters
+    ----------
+    message
+        The message to be included in the email notification that will be sent out.
+
+    '''
+
+    user = getpass.getuser()
+    hostname = socket.gethostname()
+    deliverer = '{}@stsci.edu'.format(user)
+
+    message = MIMEText(message)
+    message['Subject'] = f'JWQL ALERT FOR LOCK_MODULE ON {hostname}'
+    message['From'] = deliverer
+    message['To'] = 'bsappington@stsci.edu'  # SAPP TODO - CHANGE TO JWQL@stsci.edu after testing
+
+    s = smtplib.SMTP('smtp.stsci.edu')
+    s.send_message(message)
+    s.quit()
 
 
 def lock_module(func):
@@ -84,6 +160,8 @@ def lock_module(func):
     def wrapped(*args, **kwargs):
 
         # Get the module name of the calling method
+        existing_lock = False
+        notify_str = ""
         frame = inspect.stack()[1]
         mod = inspect.getmodule(frame[0])
         module = mod.__file__
@@ -92,21 +170,22 @@ def wrapped(*args, **kwargs):
         module_lock = module.replace('.py', '.lock')
 
         if os.path.exists(module_lock):
-            indent = " " * 4
-            print(indent + "ERROR!! Instance of protected module already running for:")
-            print(indent + module)
-            print(indent + f"Check PID in {module_lock} and verify process is still running")
-            print(indent + "If logged PID is not currently running, delete lock file and run again")
-        else:
+            notify_str, existing_lock = _clean_lock_file(module_lock, module)
+        if not existing_lock:
+            if notify_str:
+                _send_notification(notify_str)
+                notify_str = ""
+
             try:
                 with open(module_lock, "w") as lock_file:
-                    lock_file.write(f"{PID_LOCKFILE_KEY}{os.getpid()}\n")
+                    lock_file.write(f"{_PID_LOCKFILE_KEY}{os.getpid()}\n")
                 return func(*args, **kwargs)
             finally:
                 try:
                     os.remove(module_lock)
                 except Exception as e:
-                    print(e, type(e).__name__, e.args)
-                    print(module_lock + ' delete failed, Please Manually Delete')
-                    pass
+                    notify_str = f"Exception {e} \n {type(e).__name__}\n{e.args}\n"
+                    notify_str = notify_str + module_lock + " delete failed, please investigate cause"
+        if len(notify_str):
+            _send_notification(notify_str)
     return wrapped

From 5ad7fd97a222c7e68e66b5f4f46c8dc794f4da58 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Thu, 1 Jun 2023 15:09:48 -0400
Subject: [PATCH 126/256] update email and remove old solution

---
 jwql/jwql_monitors/clean_locks.py | 171 ------------------------------
 jwql/utils/protect_module.py      |   4 +-
 2 files changed, 3 insertions(+), 172 deletions(-)
 delete mode 100644 jwql/jwql_monitors/clean_locks.py

diff --git a/jwql/jwql_monitors/clean_locks.py b/jwql/jwql_monitors/clean_locks.py
deleted file mode 100644
index 5aa167d20..000000000
--- a/jwql/jwql_monitors/clean_locks.py
+++ /dev/null
@@ -1,171 +0,0 @@
-#! /usr/bin/env python
-
-"""Evaluate and clean all *.lock files in the current directory.
-
-This script is used as protection for scripts using the `jwql/utils/protect_module.py::lock_module` decorator
-
-There are infrequent instances where the MODULE_NAME.lock file will linger despite the process no longer running.
-This script will
-    Cycle through all *.lock files generated from the lock_module decorator
-    Retreive their associated PIDs
-    Delete any lock file who's associated PID is no longer running.
-    Email confirmation of any files that were deleted
-
-Authors
--------
-
-    - Bradley Sappington
-
-
-Use
----
-
-    This script is intended to be run as a frequent cronjob as protection for generate_preview_images.py
-
-    ::
-
-        python clean_locks.py
-        or
-        python clean_locks.py -p "/directory/to/check"
-"""
-# SAPP TODO - NO LONGER NEED THIS MODULE!
-
-import argparse
-import getpass
-import glob
-import os
-import smtplib
-import socket
-
-from email.mime.text import MIMEText
-from jwql.utils import permissions
-from jwql.utils.protect_module import PID_LOCKFILE_KEY
-from psutil import pid_exists
-
-
-def clean_lock_file(filename):
-    try:
-        permissions.set_permissions(filename)
-        pid = retreive_pid_from_lock_file(filename)
-        if not pid_exists(pid):
-            # if PID associated with the lock file is no longer running, then lock file should not exist, delete it.
-            if os.path.exists(filename):
-                os.remove(filename)
-                return (f"DELETED FILE `{filename}` - This file's associated PID was no longer running.\n"
-                        f"This lock file would have blocked future instances of `{filename.replace('.lock', '.py')}`")
-        return ""
-    except SyntaxError as e:
-        return str(e)
-
-
-def get_all_lock_files(search_path="."):
-    '''This function returns a list of all lock files in a given directory.
-
-    Parameters
-    ----------
-    search_path, optional
-        The directory path where the function will search for lock files. By default, it is set to the
-        current directory (".") but it can be changed to any other directory path.
-
-    Returns
-    -------
-        list: file paths for all files with the extension ".lock" in the search path
-
-    '''
-    lock_files = glob.glob(search_path + "/*.lock")
-    return lock_files
-
-
-def parse_args():
-    """ Parse command line arguments.
-
-    Parameters
-    ----------
-        None
-
-    Returns
-    ----------
-        argparse.Namespace object: An object containing all of the added arguments.
-
-    """
-    path_help = ('Directory where you would like to search for *.lock files.'
-                 ' Default is current directory')
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-p',
-                        type=str,
-                        help=path_help,
-                        action='store',
-                        required=False,
-                        default='.')
-
-    arguments = parser.parse_args()
-
-    return arguments
-
-
-def retreive_pid_from_lock_file(filename):
-    '''This function retrieves a process ID from a lock file.
-
-    Parameters
-    ----------
-    filename
-        The filename parameter is a string that represents the name of the lock file from which the process
-        ID (PID) needs to be retrieved.
-
-    Returns
-    -------
-        int: The process ID (PID)
-        or
-        SyntaxError: Indicating that the file should be manually investigated and deleted if appropriate.
-
-    '''
-    # Lock file format is established in `jwql/utils/protect_module.py::lock_module`
-    with open(filename, 'r') as file:
-        for line in file:
-            if PID_LOCKFILE_KEY in line:
-                number_index = line.index(PID_LOCKFILE_KEY) + len(PID_LOCKFILE_KEY)
-                number = line[number_index:].strip()
-                return int(number)
-
-    raise SyntaxError(f"No PID found in {filename} - Please manually investigate and delete if appropriate")
-
-
-def send_notification(message):
-    '''Sends an email notification to JWQL team alerting them of script actually solving issue (or not)
-
-    Parameters
-    ----------
-    message
-        The message to be included in the email notification that will be sent out.
-
-    '''
-
-    user = getpass.getuser()
-    hostname = socket.gethostname()
-    deliverer = '{}@stsci.edu'.format(user)
-
-    message = MIMEText(message)
-    message['Subject'] = f'JWQL ALERT FOR `CLEAN_LOCKS` SCRIPT ON {hostname}'
-    message['From'] = deliverer
-    message['To'] = 'bsappington@stsci.edu'  # SAPP TODO - CHANGE TO JWQL@stsci.edu after testing
-
-    s = smtplib.SMTP('smtp.stsci.edu')
-    s.send_message(message)
-    s.quit()
-
-
-if __name__ == '__main__':
-    email_data = "The JWQL project's CLEAN_LOCKS.PY has recently run and performed actions that you should be aware of:\n\n"
-    args = parse_args()
-    lock_files = get_all_lock_files(args.p)
-    notify_team = False
-
-    for file in lock_files:
-        notify_str = clean_lock_file(file)
-        if len(notify_str):
-            notify_team = True
-            email_data = email_data + notify_str + "\n"
-
-    if notify_team:
-        send_notification(email_data)
diff --git a/jwql/utils/protect_module.py b/jwql/utils/protect_module.py
index 79495554c..aa2d25230 100644
--- a/jwql/utils/protect_module.py
+++ b/jwql/utils/protect_module.py
@@ -9,6 +9,8 @@
 
 The file will also contain the process id for reference, in case a lock file exists and
 the user does not think it should (i.e. module exited unexpectedly without proper closure)
+If this scenario arises and the locked module is run again, it will verify that the PID in the current lock file is not actively running.
+If the PID is not actively running the module will delete the file, alert the dev team, and carry on with a new lock file/PID.
 
 This decorator is designed for use with JWQL Monitors and Generate functions.
 It should decorate a function called "protected_code" which contains the main functionality where locking is required.
@@ -131,7 +133,7 @@ def _send_notification(message):
     message = MIMEText(message)
     message['Subject'] = f'JWQL ALERT FOR LOCK_MODULE ON {hostname}'
     message['From'] = deliverer
-    message['To'] = 'bsappington@stsci.edu'  # SAPP TODO - CHANGE TO JWQL@stsci.edu after testing
+    message['To'] = 'jwql@stsci.edu'
 
     s = smtplib.SMTP('smtp.stsci.edu')
     s.send_message(message)

From 7e9f4579e363496773e0dde8885315db297c7633 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 2 Jun 2023 13:07:53 -0400
Subject: [PATCH 127/256] Update test_protect_module.py for updates

---
 jwql/tests/test_protect_module.py | 46 +++++++++++++++++++++++++++----
 1 file changed, 41 insertions(+), 5 deletions(-)

diff --git a/jwql/tests/test_protect_module.py b/jwql/tests/test_protect_module.py
index a7406e47a..6d4e78187 100644
--- a/jwql/tests/test_protect_module.py
+++ b/jwql/tests/test_protect_module.py
@@ -13,13 +13,19 @@
     These tests can be run via the command line (omit the -s to
     suppress verbose output to stdout):
 
+    !!!WARNING IF TESTING LOCALLY!!!
+    These tests will call protect_module.py::lock_module.py and will sent emails to jwql.stsci.edu!
+    Please locally change the `message['To']` recipient to an appropriate address while testing to
+    avoid spamming/confusing the team.
+
+
     ::
 
         pytest -s test_protect_module.py
 """
 import os
 from pytest import fixture, mark
-from jwql.utils.protect_module import lock_module
+from jwql.utils.protect_module import lock_module, _PID_LOCKFILE_KEY
 
 # Determine if tests are being run on Github Actions
 ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
@@ -44,7 +50,7 @@ def protected_code_entered():
 
 @mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_lock_module_create_destroy_file(module_lock):
-    """Test if that wrapper will create and destroy a lock file named by module """
+    """Test wrapper will create and destroy a lock file named by module if no other lock exists """
 
     # Ensure lock file does not exist
     if os.path.exists(module_lock):
@@ -56,11 +62,41 @@ def test_lock_module_create_destroy_file(module_lock):
 
 
 @mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
-def test_lock_module_wont_run_locked(module_lock):
-    """Test if that wrapper will create and destroy a lock file named by module """
+def test_already_locked_module_wont_run_with_bad_lock_file(module_lock):
+    """Test when wrapper encounters poorly formatted lock file,
+       it will not run, not delete lock file, and will send warn email """
     # create locked file in advance of calling protected code
     with open(module_lock, "w"):
         entered_protected_code = protected_code_entered()
-    os.remove(module_lock)
+    # assert the invalid lock file was never deleted
+    assert os.path.exists(module_lock)
     # assert that we never entered protected code
     assert entered_protected_code is None
+    # clean the test
+    os.remove(module_lock)
+
+
+@mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
+def test_already_locked_module_wont_run_with_legit_lock_file(module_lock):
+    """Test wrapper will not run if it encounters a correclty formatted lock_file """
+    # create locked file with running PID in advance of calling protected code
+    with open(module_lock, "w") as lock_file:
+        lock_file.write(f"{_PID_LOCKFILE_KEY}{os.getpid()}\n")
+    entered_protected_code = protected_code_entered()
+    # assert that we never entered protected code because the PID is currently running
+    assert entered_protected_code is None
+    # clean the test
+    os.remove(module_lock)
+
+
+@mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
+def test_lock_module_handles_stale_lock_files(module_lock):
+    """Test wrapper will correctly identify stale lock file,
+       delete it, and continue to run successfully """
+    # create locked file with not running PID in advance of calling protected code
+    with open(module_lock, "w") as lock_file:
+        lock_file.write(f"{_PID_LOCKFILE_KEY}12345\n")
+    file_created = protected_code_verify_file_exists_true(module_lock)
+    file_exists = os.path.exists(module_lock)
+    # Assert that lock file was created in wrapper, and removed upon exit
+    assert (file_created and not file_exists) is True
\ No newline at end of file

From 602f1c5408d1e678d6835be492911c6ef396448d Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 2 Jun 2023 14:44:53 -0400
Subject: [PATCH 128/256] Add shared selection tools and a date range filter

---
 .../ta_monitors/msata_monitor.py              | 309 +++++++++++++-----
 .../ta_monitors/wata_monitor.py               | 221 +++++++++----
 jwql/tests/test_msata_monitor.py              |  37 ++-
 jwql/tests/test_wata_monitor.py               |  30 +-
 4 files changed, 436 insertions(+), 161 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index bc21a4643..11bb6666c 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -38,6 +38,7 @@
 import os
 import logging
 import shutil
+from datetime import datetime, timezone, timedelta
 from random import randint
 
 import numpy as np
@@ -45,12 +46,13 @@
 from astropy.time import Time
 from astropy.io import fits
 from bokeh.embed import components
-from bokeh.plotting import figure, output_file, save
-from bokeh.layouts import gridplot
-from bokeh.models import ColumnDataSource, Range1d, Span, Label
-from bokeh.models.tools import HoverTool
+from bokeh.layouts import gridplot, layout
+from bokeh.models import (
+    ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView,
+    Span, Label, DateRangeSlider)
+from bokeh.models.tools import HoverTool, BoxSelectTool
+from bokeh.plotting import figure, save, output_file
 from bs4 import BeautifulSoup
-from datetime import datetime
 from sqlalchemy.sql.expression import and_
 
 # jwql imports
@@ -136,6 +138,13 @@ def __init__(self):
                                'planned_v3': {'loc': 'ta_table', 'alt_key': None, 'name': 'planned_v3', 'type': float},
                                'FITTARGS': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'stars_in_fit', 'type': int}}
 
+        # initialize attributes to be set later
+        self.source = None
+        self.share_tools = []
+        self.date_range = None
+        self.date_filter = None
+        self.date_view = None
+
     def get_tainfo_from_fits(self, fits_file):
         """ Get the TA information from the fits file
         Parameters
@@ -240,6 +249,19 @@ def get_msata_data(self, new_filenames):
         msata_df = pd.DataFrame(msata_dict)
         return msata_df, no_ta_ext_msgs
 
+    def add_time_column(self):
+        """Add time column to data source, to be used by all plots."""
+        date_obs = self.source.data['date_obs']
+        if 'time_arr' not in self.source.data:
+            time_arr = []
+            for do_str in date_obs:
+                # convert time string into an array of time (this is in UT)
+                t = datetime.fromisoformat(do_str)
+                time_arr.append(t)
+
+            # add to the bokeh data structure
+            self.source.data["time_arr"] = time_arr
+
     def plt_status(self):
         """ Plot the MSATA status versus time.
         Parameters
@@ -249,18 +271,12 @@ def plt_status(self):
         -------
             plot: bokeh plot object
         """
-        # ta_status, date_obs = source.data['ta_status'], source.data['date_obs']
-        date_obs = self.source.data['date_obs']
         ta_status = self.source.data['ta_status']
         # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            time_arr = self.source.data['time_arr']
-            bool_status = self.source.data['bool_status']
-            status_colors = self.source.data['status_colors']
-        except KeyError:
-            # bokeh does not like to plot strings, turn  into numbers
-            number_status, time_arr, status_colors = [], [], []
-            for tas, do_str in zip(ta_status, date_obs):
+        if 'bool_status' not in self.source.data:
+            # bokeh does not like to plot strings, turn into numbers
+            number_status, status_colors = [], []
+            for tas in ta_status:
                 if tas.lower() == 'unsuccessful':
                     number_status.append(0.0)
                     status_colors.append('red')
@@ -270,19 +286,19 @@ def plt_status(self):
                 else:
                     number_status.append(1.0)
                     status_colors.append('blue')
-                # convert time string into an array of time (this is in UT with 0.0 milliseconds)
-                t = datetime.fromisoformat(do_str)
-                time_arr.append(t)
+
             # add these to the bokeh data structure
-            self.source.data["time_arr"] = time_arr
             self.source.data["number_status"] = number_status
             self.source.data["status_colors"] = status_colors
+
         # create a new bokeh plot
         plot = figure(title="MSATA Status [Success=1, In Progress=0.5, Fail=0]", x_axis_label='Time',
                       y_axis_label='MSATA Status', x_axis_type='datetime',)
         plot.y_range = Range1d(-0.5, 1.5)
         plot.circle(x='time_arr', y='number_status', source=self.source,
-                    color='status_colors', size=7, fill_alpha=0.3)
+                    color='status_colors', size=7, fill_alpha=0.3, view=self.date_view)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -293,8 +309,11 @@ def plt_status(self):
                           ('Date-Obs', '@date_obs'),
                           ('Subarray', '@subarray'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_residual_offsets(self):
@@ -311,7 +330,8 @@ def plt_residual_offsets(self):
                       x_axis_label='Least Squares Residual V2 Offset',
                       y_axis_label='Least Squares Residual V3 Offset')
         plot.circle(x='lsv2offset', y='lsv3offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+
         v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet']
         xstart, ystart, ray_length = -1 * v2halffacet[0], -1 * v3halffacet[0], 0.05
         plot.ray(x=xstart - ray_length / 2.0, y=ystart, length=ray_length, angle_units="deg",
@@ -322,10 +342,13 @@ def plt_residual_offsets(self):
         plot.add_layout(hflabel)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -338,8 +361,11 @@ def plt_residual_offsets(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v2offset_time(self):
@@ -355,8 +381,9 @@ def plt_v2offset_time(self):
         plot = figure(title="MSATA Least Squares V2 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V2 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv2offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         time_arr, v2halffacet = self.source.data['time_arr'], self.source.data['v2halffacet']
@@ -364,6 +391,8 @@ def plt_v2offset_time(self):
         plot.renderers.extend([hline, hfline])
         hflabel = Label(x=time_arr[-1], y=-1 * v2halffacet[0], y_units='data', text='-V2 half-facet value')
         plot.add_layout(hflabel)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -376,8 +405,11 @@ def plt_v2offset_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v3offset_time(self):
@@ -393,8 +425,9 @@ def plt_v3offset_time(self):
         plot = figure(title="MSATA Least Squares V3 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V3 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv3offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         time_arr, v3halffacet = self.source.data['time_arr'], self.source.data['v3halffacet']
@@ -402,6 +435,8 @@ def plt_v3offset_time(self):
         plot.renderers.extend([hline, hfline])
         hflabel = Label(x=time_arr[-1], y=-1 * v3halffacet[0], y_units='data', text='-V3 half-facet value')
         plot.add_layout(hflabel)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -414,8 +449,11 @@ def plt_v3offset_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_lsv2v3offsetsigma(self):
@@ -432,13 +470,16 @@ def plt_lsv2v3offsetsigma(self):
                       x_axis_label='Least Squares Residual V2 Sigma Offset',
                       y_axis_label='Least Squares Residual V3 Sigma Offset')
         plot.circle(x='lsv2sigma', y='lsv3sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.x_range = Range1d(-0.1, 0.1)
         plot.y_range = Range1d(-0.1, 0.1)
+
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -453,8 +494,11 @@ def plt_lsv2v3offsetsigma(self):
                           ('LS V3 offset', '@lsv3offset'),
                           ('LS V3 sigma', '@lsv3sigma'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_res_offsets_corrected(self):
@@ -466,29 +510,30 @@ def plt_res_offsets_corrected(self):
         -------
             plot: bokeh plot object
         """
-        # create a new bokeh plot
         lsv2offset, lsv3offset = self.source.data['lsv2offset'], self.source.data['lsv3offset']
         v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet']
-        # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            v2_half_fac_corr = self.source.data['v2_half_fac_corr']
-            v3_half_fac_corr = self.source.data['v3_half_fac_corr']
-        except KeyError:
+
+        # check if this column exists in the data already, else create it
+        if 'v2_half_fac_corr' not in self.source.data:
             v2_half_fac_corr, v3_half_fac_corr = [], []
             for idx, v2hf in enumerate(v2halffacet):
                 v3hf = v3halffacet[idx]
                 v2_half_fac_corr.append(lsv2offset[idx] + v2hf)
                 v3_half_fac_corr.append(lsv3offset[idx] + v3hf)
+
             # add these to the bokeh data structure
             self.source.data["v2_half_fac_corr"] = v2_half_fac_corr
             self.source.data["v3_half_fac_corr"] = v3_half_fac_corr
+
+        # create a new bokeh plot
         plot = figure(title="MSATA Least Squares Residual V2-V3 Offsets Half-facet corrected",
                       x_axis_label='Least Squares Residual V2 Offset + half-facet',
                       y_axis_label='Least Squares Residual V3 Offset + half-facet')
         plot.circle(x='v2_half_fac_corr', y='v3_half_fac_corr', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
@@ -500,6 +545,8 @@ def plt_res_offsets_corrected(self):
                  angle=90, line_color='purple', line_width=3)
         hflabel = Label(x=xstart / 3.0, y=ystart, y_units='data', text='-V2, -V3 half-facets values')
         plot.add_layout(hflabel)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -514,15 +561,16 @@ def plt_res_offsets_corrected(self):
                           ('V2 half-facet', '@v2halffacet'),
                           ('V3 half-facet', '@v3halffacet'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v2offsigma_time(self):
-        """ Plot the residual Least Squares V2 sigma Offset versus time
-        Parameters
-        ----------
-            None
+        """Plot the residual Least Squares V2 sigma Offset versus time
+
         Returns
         -------
             plot: bokeh plot object
@@ -531,11 +579,14 @@ def plt_v2offsigma_time(self):
         plot = figure(title="MSATA Least Squares V2 Sigma Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V2 Sigma Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv2sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.1, 0.1)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -547,28 +598,34 @@ def plt_v2offsigma_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V2 sigma', '@lsv2sigma'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v3offsigma_time(self):
-        """ Plot the residual Least Squares V3 Offset versus time
-        Parameters
-        ----------
-            None
+        """Plot the residual Least Squares V3 Offset versus time
+
         Returns
         -------
             p: bokeh plot object
         """
         # create a new bokeh plot
-        plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time", x_axis_label='Time',
-                      y_axis_label='Least Squares Residual V3 Sigma Offset', x_axis_type='datetime')
+        plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time",
+                      x_axis_label='Time',
+                      y_axis_label='Least Squares Residual V3 Sigma Offset',
+                      x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv3sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.1, 0.1)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -581,8 +638,11 @@ def plt_v3offsigma_time(self):
                           ('LS V3 offset', '@lsv3offset'),
                           ('LS V3 sigma', '@lsv3sigma'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_roll_offset(self):
@@ -598,10 +658,12 @@ def plt_roll_offset(self):
         plot = figure(title="MSATA Least Squares Roll Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual Roll Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsrolloffset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-600.0, 600.0)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
+
         # Maximum accepted roll line and label
         time_arr = self.source.data['time_arr']
         arlinepos = Span(location=120, dimension='width', line_color='green', line_width=3)
@@ -609,6 +671,8 @@ def plt_roll_offset(self):
         arlabel = Label(x=time_arr[-1], y=125, y_units='data', text='Max accepted roll')
         plot.add_layout(arlabel)
         plot.renderers.extend([hline, arlinepos, arlineneg])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -621,8 +685,11 @@ def plt_roll_offset(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_lsoffsetmag(self):
@@ -638,11 +705,14 @@ def plt_lsoffsetmag(self):
         plot = figure(title="MSATA Least Squares Total Magnitude of the Linear V2, V3 Offset Slew vs Time", x_axis_label='Time',
                       y_axis_label='sqrt((V2_off)**2 + (V3_off)**2)', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsoffsetmag', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -656,8 +726,11 @@ def plt_lsoffsetmag(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_tot_number_of_stars(self):
@@ -672,34 +745,34 @@ def plt_tot_number_of_stars(self):
         # get the number of stars per array
         visit_id = self.source.data['visit_id']
         reference_star_number = self.source.data['reference_star_number']
-        stars_in_fit = self.source.data['stars_in_fit']
-        date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr']
-        # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            tot_number_of_stars = self.source.data['tot_number_of_stars']
-            colors_list = self.source.data['colors_list']
-        except KeyError:
+
+        # check if this column exists in the data already, else create it
+        if 'tot_number_of_stars' not in self.source.data:
             # create the list of color per visit and tot_number_of_stars
             colors_list, tot_number_of_stars = [], []
             color_dict = {}
-            for i, _ in enumerate(visit_id):
+            for i, vid in enumerate(visit_id):
                 tot_stars = len(reference_star_number[i])
                 tot_number_of_stars.append(tot_stars)
                 ci = '#%06X' % randint(0, 0xFFFFFF)
-                if visit_id[i] not in color_dict:
-                    color_dict[visit_id[i]] = ci
-                colors_list.append(color_dict[visit_id[i]])
+                if vid not in color_dict:
+                    color_dict[vid] = ci
+                colors_list.append(color_dict[vid])
+
             # add these to the bokeh data structure
             self.source.data["tot_number_of_stars"] = tot_number_of_stars
             self.source.data["colors_list"] = colors_list
+
         # create a new bokeh plot
         plot = figure(title="Total Number of Measurements vs Time", x_axis_label='Time',
                       y_axis_label='Total number of measurements', x_axis_type='datetime')
         plot.circle(x='time_arr', y='tot_number_of_stars', source=self.source,
-                    color='colors_list', size=7, fill_alpha=0.3)
+                    color='colors_list', size=7, fill_alpha=0.3, view=self.date_view)
         plot.triangle(x='time_arr', y='stars_in_fit', source=self.source,
-                      color='black', size=7, fill_alpha=0.3)
+                      color='black', size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(0.0, 40.0)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -714,8 +787,8 @@ def plt_tot_number_of_stars(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
         return plot
 
     def plt_mags_time(self):
@@ -735,22 +808,24 @@ def plt_mags_time(self):
         planned_v2 = self.source.data['planned_v2']
         planned_v3 = self.source.data['planned_v3']
         reference_star_number = self.source.data['reference_star_number']
-        visits_stars_mags = self.source.data['reference_star_mag']
         box_peak_value = self.source.data['box_peak_value']
         date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr']
         colors_list = self.source.data['colors_list']
         detector_list = self.source.data['detector']
+        filename = self.source.data['filename']
+
         # create the structure matching the number of visits and reference stars
         new_colors_list, vid, dobs, tarr, star_no, status = [], [], [], [], [], []
-        peaks, stars_v2, stars_v3, det = [], [], [], []
+        peaks, stars_v2, stars_v3, det, fnames = [], [], [], [], []
         for i, _ in enumerate(visit_id):
-            v, d, t, c, s, x, y, dt = [], [], [], [], [], [], [], []
+            v, d, t, c, s, x, y, dt, fn = [], [], [], [], [], [], [], [], []
             for j in range(len(reference_star_number[i])):
                 v.append(visit_id[i])
                 d.append(date_obs[i])
                 t.append(time_arr[i])
                 c.append(colors_list[i])
                 dt.append(detector_list[i])
+                fn.append(filename[i])
                 if 'not_removed' in lsf_removed_status[i][j]:
                     s.append('SUCCESS')
                     x.append(planned_v2[i][j])
@@ -769,17 +844,29 @@ def plt_mags_time(self):
             stars_v3.extend(y)
             peaks.extend(box_peak_value[i])
             det.extend(dt)
+            fnames.extend(fn)
+
         # now create the mini ColumnDataSource for this particular plot
         mini_source = {'vid': vid, 'star_no': star_no, 'status': status,
-                       'dobs': dobs, 'tarr': tarr, 'det': det,
+                       'dobs': dobs, 'time_arr': tarr, 'det': det, 'filename': fnames,
                        'peaks': peaks, 'colors_list': new_colors_list,
                        'stars_v2': stars_v2, 'stars_v3': stars_v3}
         mini_source = ColumnDataSource(data=mini_source)
-        # create a the bokeh plot
-        plot = figure(title="MSATA Counts vs Time", x_axis_label='Time', y_axis_label='box_peak [Counts]',
+
+        # hook up the date range slider to this source as well
+        callback = CustomJS(args=dict(s=mini_source), code="""
+            s.change.emit();
+        """)
+        self.date_range.js_on_change('value', callback)
+        mini_view = CDSView(source=mini_source, filters=[self.date_filter])
+
+        # create the bokeh plot
+        plot = figure(title="MSATA Counts vs Time", x_axis_label='Time',
+                      y_axis_label='box_peak [Counts]',
                       x_axis_type='datetime')
-        plot.circle(x='tarr', y='peaks', source=mini_source,
-                    color='colors_list', size=7, fill_alpha=0.3)
+        plot.circle(x='time_arr', y='peaks', source=mini_source,
+                    color='colors_list', size=7, fill_alpha=0.3, view=mini_view)
+
         # add count saturation warning lines
         loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0
         hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3)
@@ -793,7 +880,8 @@ def plt_mags_time(self):
         plot.add_layout(label2)
         plot.add_layout(label3)
         plot.y_range = Range1d(-1000.0, 62000.0)
-        # add hover
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@vid'),
@@ -805,18 +893,69 @@ def plt_mags_time(self):
                           ('Measured V2', '@stars_v2'),
                           ('Measured V3', '@stars_v3'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
         return plot
 
+    def setup_date_range(self):
+        """Set up a date range filter, defaulting to the last week of data."""
+        end_date = datetime.now(tz=timezone.utc)
+        one_week_ago = end_date.date() - timedelta(days=7)
+        first_data_point = np.min(self.source.data['time_arr']).date()
+        last_data_point = np.max(self.source.data['time_arr']).date()
+        if last_data_point < one_week_ago:
+            # keep at least one point in the plot if there was
+            # no TA data this week
+            start_date = last_data_point
+        else:
+            start_date = one_week_ago
+
+        # allowed range is from the first ever data point to today
+        self.date_range = DateRangeSlider(
+            title="Date range displayed", start=first_data_point,
+            end=end_date, value=(start_date, end_date), step=1)
+
+        callback = CustomJS(args=dict(s=self.source), code="""
+            s.change.emit();
+        """)
+        self.date_range.js_on_change('value', callback)
+
+        self.date_filter = CustomJSFilter(args=dict(slider=self.date_range), code="""
+                var indices = [];
+                var start = slider.value[0];
+                var end = slider.value[1];
+
+                for (var i=0; i < source.get_length(); i++) {
+                    if (source.data['time_arr'][i] >= start 
+                            && source.data['time_arr'][i] <= end) {
+                        indices.push(true);
+                    } else {
+                        indices.push(false);
+                    }
+                }
+                return indices;
+                """)
+        self.date_view = CDSView(source=self.source, filters=[self.date_filter])
+
     def mk_plt_layout(self):
         """Create the bokeh plot layout"""
         self.source = ColumnDataSource(data=self.msata_data)
+
         # make sure all arrays are lists in order to later be able to read the data
         # from the html file
         for item in self.source.data:
             if not isinstance(self.source.data[item], (str, float, int, list)):
                 self.source.data[item] = self.source.data[item].tolist()
+
+        # add a time array to the data source
+        self.add_time_column()
+
+        # set up selection tools to share
+        self.share_tools = [BoxSelectTool()]
+
+        # set up a date range filter widget
+        self.setup_date_range()
+
         # set the output html file name and create the plot grid
         output_file(self.output_file_name)
         p1 = self.plt_status()
@@ -831,13 +970,15 @@ def mk_plt_layout(self):
         p10 = self.plt_lsoffsetmag()
         p12 = self.plt_tot_number_of_stars()
         p11 = self.plt_mags_time()
+
         # make grid
         grid = gridplot([p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12],
                         ncols=2, merge_tools=False)
-        # show(grid)
-        save(grid)
-        # return the needed components for embeding the results in the MSATA html template
-        script, div = components(grid)
+        box_layout = layout(children=[self.date_range, grid])
+        save(box_layout)
+
+        # return the needed components for embedding the results in the MSATA html template
+        script, div = components(box_layout)
         return script, div
 
     def identify_tables(self):
@@ -961,7 +1102,7 @@ def prev_data2expected_format(self, prev_data_dict):
             Date of the latest observation in the previously plotted data
         """
         # remember that the time array created is in milliseconds, removing to get time object
-        time_in_millis = max(prev_data_dict['tarr'])
+        time_in_millis = max(prev_data_dict['time_arr'])
         latest_prev_obs = Time(time_in_millis / 1000., format='unix')
         latest_prev_obs = latest_prev_obs.mjd
         prev_data_expected_cols = {}
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index 01e52a44c..28b533b1e 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -34,21 +34,22 @@
 import os
 import logging
 import shutil
+from datetime import datetime, timezone, timedelta
 
 import numpy as np
 import pandas as pd
-from bs4 import BeautifulSoup
-from datetime import datetime
 from astropy.time import Time
 from astropy.io import fits
-from sqlalchemy.sql.expression import and_
-from bokeh.io import output_file
-from bokeh.plotting import figure, show, save
-from bokeh.models import ColumnDataSource, Range1d
-from bokeh.models.tools import HoverTool
-from bokeh.layouts import gridplot
-from bokeh.models import Span, Label
 from bokeh.embed import components
+from bokeh.io import output_file
+from bokeh.layouts import gridplot, layout
+from bokeh.models import (
+    ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView,
+    Span, Label, DateRangeSlider)
+from bokeh.models.tools import HoverTool, BoxSelectTool
+from bokeh.plotting import figure, save
+from bs4 import BeautifulSoup
+from sqlalchemy.sql.expression import and_
 
 # jwql imports
 from jwql.utils.logging_functions import log_info, log_fail
@@ -130,6 +131,12 @@ def __init__(self):
                                'SAM_X': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_x'},
                                'SAM_Y': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_y'}}
 
+        # initialize attributes to be set later
+        self.source = None
+        self.share_tools = []
+        self.date_range = None
+        self.date_view = None
+
     def get_tainfo_from_fits(self, fits_file):
         """ Get the TA information from the fits file
         Parameters
@@ -200,6 +207,19 @@ def get_wata_data(self, new_filenames):
         wata_df = pd.DataFrame(wata_dict)
         return wata_df, no_ta_ext_msgs
 
+    def add_time_column(self):
+        """Add time column to data source, to be used by all plots."""
+        date_obs = self.source.data['date_obs']
+        if 'time_arr' not in self.source.data:
+            time_arr = []
+            for do_str in date_obs:
+                # convert time string into an array of time (this is in UT)
+                t = datetime.fromisoformat(do_str)
+                time_arr.append(t)
+
+            # add to the bokeh data structure
+            self.source.data["time_arr"] = time_arr
+
     def plt_status(self):
         """ Plot the WATA status (passed = 0 or failed = 1).
         Parameters
@@ -209,36 +229,32 @@ def plt_status(self):
         -------
             plot: bokeh plot object
         """
-        ta_status, date_obs = self.source.data['ta_status'], self.source.data['date_obs']
-        # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            time_arr = self.source.data['time_arr']
-            bool_status = self.source.data['bool_status']
-            status_colors = self.source.data['status_colors']
-        except KeyError:
-            # bokeh does not like to plot strings, turn  into binary type
-            bool_status, time_arr, status_colors = [], [], []
-            for tas, do_str in zip(ta_status, date_obs):
+        ta_status = self.source.data['ta_status']
+
+        # check if this column exists in the data already, else create it
+        if 'bool_status' not in self.source.data:
+            # bokeh does not like to plot strings, turn into binary type
+            bool_status, status_colors = [], []
+            for tas in ta_status:
                 if 'unsuccessful' not in tas.lower():
                     bool_status.append(1)
                     status_colors.append('blue')
                 else:
                     bool_status.append(0)
                     status_colors.append('red')
-                # convert time string into an array of time (this is in UT)
-                t = datetime.fromisoformat(do_str)
-                time_arr.append(t)
+
             # add these to the bokeh data structure
-            self.source.data["time_arr"] = time_arr
             self.source.data["ta_status_bool"] = bool_status
             self.source.data["status_colors"] = status_colors
+
         # create a new bokeh plot
-        plot = figure(title="WATA Status [Succes=1, Fail=0]", x_axis_label='Time',
+        plot = figure(title="WATA Status [Success=1, Fail=0]", x_axis_label='Time',
                       y_axis_label='WATA Status', x_axis_type='datetime',)
         plot.y_range = Range1d(-0.5, 1.5)
         plot.circle(x='time_arr', y='ta_status_bool', source=self.source,
-                    color='status_colors', size=7, fill_alpha=0.3)
-        # output_file("wata_status.html")
+                    color='status_colors', size=7, fill_alpha=0.3, view=self.date_view)
+
+        # make tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -250,6 +266,10 @@ def plt_status(self):
                           ('--------', '----------------')]
 
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_residual_offsets(self):
@@ -265,13 +285,16 @@ def plt_residual_offsets(self):
         plot = figure(title="WATA Residual V2-V3 Offsets", x_axis_label='Residual V2 Offset',
                       y_axis_label='Residual V3 Offset')
         plot.circle(x='v2_offset', y='v3_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -281,8 +304,11 @@ def plt_residual_offsets(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v2offset_time(self):
@@ -298,11 +324,14 @@ def plt_v2offset_time(self):
         plot = figure(title="WATA V2 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Residual V2 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='v2_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -312,8 +341,11 @@ def plt_v2offset_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v3offset_time(self):
@@ -329,11 +361,14 @@ def plt_v3offset_time(self):
         plot = figure(title="WATA V3 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Residual V3 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='v3_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -343,8 +378,11 @@ def plt_v3offset_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_mag_time(self):
@@ -358,15 +396,9 @@ def plt_mag_time(self):
         """
         # calculate the pseudo magnitudes
         max_val_box, time_arr = self.source.data['max_val_box'], self.source.data['time_arr']
-        # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            nrsrapid_f140x = self.source.data["nrsrapid_f140x"]
-            nrsrapid_f110w = self.source.data["nrsrapid_f110w"]
-            nrsrapid_clear = self.source.data["nrsrapid_clear"]
-            nrsrapidd6_f140x = self.source.data["nrsrapidd6_f140x"]
-            nrsrapidd6_f110w = self.source.data["nrsrapidd6_f110w"]
-            nrsrapidd6_clear = self.source.data["nrsrapidd6_clear"]
-        except KeyError:
+
+        # check if this column exists in the data already, else create it
+        if "nrsrapid_f140x" not in self.source.data:
             # create the arrays per filter and readout pattern
             nrsrapid_f140x, nrsrapid_f110w, nrsrapid_clear = [], [], []
             nrsrapidd6_f140x, nrsrapidd6_f110w, nrsrapidd6_clear = [], [], []
@@ -417,6 +449,7 @@ def plt_mag_time(self):
                         nrsrapidd6_f140x.append(np.NaN)
                         nrsrapidd6_f110w.append(np.NaN)
                         nrsrapidd6_clear.append(val)
+
             # add to the bokeh data structure
             self.source.data["nrsrapid_f140x"] = nrsrapid_f140x
             self.source.data["nrsrapid_f110w"] = nrsrapid_f110w
@@ -424,27 +457,30 @@ def plt_mag_time(self):
             self.source.data["nrsrapidd6_f140x"] = nrsrapidd6_f140x
             self.source.data["nrsrapidd6_f110w"] = nrsrapidd6_f110w
             self.source.data["nrsrapidd6_clear"] = nrsrapidd6_clear
+
         # create a new bokeh plot
         plot = figure(title="WATA Counts vs Time", x_axis_label='Time',
                       y_axis_label='box_peak [Counts]', x_axis_type='datetime')
         plot.circle(x='time_arr', y='nrsrapid_f140x', source=self.source,
-                    color="purple", size=7, fill_alpha=0.4)
+                    color="purple", size=7, fill_alpha=0.4, view=self.date_view)
         plot.circle(x='time_arr', y='nrsrapidd6_f140x', source=self.source,
-                    color="purple", size=12, fill_alpha=0.4)
+                    color="purple", size=12, fill_alpha=0.4, view=self.date_view)
         plot.triangle(x='time_arr', y='nrsrapid_f110w', source=self.source,
-                      color="orange", size=8, fill_alpha=0.4)
+                      color="orange", size=8, fill_alpha=0.4, view=self.date_view)
         plot.triangle(x='time_arr', y='nrsrapidd6_f110w', source=self.source,
-                      color="orange", size=13, fill_alpha=0.4)
+                      color="orange", size=13, fill_alpha=0.4, view=self.date_view)
         plot.square(x='time_arr', y='nrsrapid_clear', source=self.source,
-                    color="gray", size=7, fill_alpha=0.4)
+                    color="gray", size=7, fill_alpha=0.4, view=self.date_view)
         plot.square(x='time_arr', y='nrsrapidd6_clear', source=self.source,
-                    color="gray", size=12, fill_alpha=0.4)
+                    color="gray", size=12, fill_alpha=0.4, view=self.date_view)
+
         # add count saturation warning lines
         loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0
         hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3)
         hline2 = Span(location=loc2, dimension='width', line_color='yellow', line_width=3)
         hline3 = Span(location=loc3, dimension='width', line_color='red', line_width=3)
         plot.renderers.extend([hline1, hline2, hline3])
+
         label1 = Label(x=time_arr[-1], y=loc1, y_units='data', text='45000 counts')
         label2 = Label(x=time_arr[-1], y=loc2, y_units='data', text='50000 counts')
         label3 = Label(x=time_arr[-1], y=loc3, y_units='data', text='60000 counts')
@@ -452,7 +488,8 @@ def plt_mag_time(self):
         plot.add_layout(label2)
         plot.add_layout(label3)
         plot.y_range = Range1d(-1000.0, 62000.0)
-        # add hover
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -462,12 +499,15 @@ def plt_mag_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Box peak', '@max_val_box'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
-    def get_unsucessful_ta(self, arr_name):
-        """ Find unsucessful TAs in this set (to be plotted in red)
+    def get_unsuccessful_ta(self, arr_name):
+        """ Find unsuccessful TAs in this set (to be plotted in red)
         Parameters
         ----------
             arr_name: str, name of the array of interest
@@ -497,16 +537,16 @@ def plt_centroid(self):
             plot: bokeh plot object
         """
         # get the failed TAs to plot in red
-        try:
-            corr_col_failed = self.source.data["corr_col_failed"]
-        except KeyError:
-            corr_col_failed, corr_col_not_failed = self.get_unsucessful_ta('corr_col')
-            corr_row_failed, corr_row_not_failed = self.get_unsucessful_ta('corr_row')
+        if "corr_col_failed" not in self.source.data:
+            corr_col_failed, corr_col_not_failed = self.get_unsuccessful_ta('corr_col')
+            corr_row_failed, corr_row_not_failed = self.get_unsuccessful_ta('corr_row')
+
             # add these to the bokeh data structure
             self.source.data["corr_col_failed"] = corr_col_failed
             self.source.data["corr_col_not_failed"] = corr_col_not_failed
             self.source.data["corr_row_failed"] = corr_row_failed
             self.source.data["corr_row_not_failed"] = corr_row_not_failed
+
         # create a new bokeh plot
         plot = figure(title="WATA Centroid", x_axis_label='Column',
                       y_axis_label='Row')
@@ -514,11 +554,13 @@ def plt_centroid(self):
         plot.x_range = Range1d(limits[0], limits[1])
         plot.y_range = Range1d(limits[0], limits[1])
         plot.circle(x='corr_col_not_failed', y='corr_row_not_failed', source=self.source,
-                    color="blue", size=7, fill_alpha=0.5)
+                    color="blue", size=7, fill_alpha=0.5, view=self.date_view)
         plot.circle(x='corr_col_failed', y='corr_row_failed', source=self.source,
-                    color="red", size=7, fill_alpha=0.5)
+                    color="red", size=7, fill_alpha=0.5, view=self.date_view)
         plot.x_range = Range1d(0.0, 32.0)
         plot.y_range = Range1d(0.0, 32.0)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -532,18 +574,72 @@ def plt_centroid(self):
                           ('Det Centr Col', '@detector_final_col'),
                           ('Det Centr Row', '@detector_final_row'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
+    def setup_date_range(self):
+        """Set up a date range filter, defaulting to the last week of data."""
+        end_date = datetime.now(tz=timezone.utc)
+        one_week_ago = end_date.date() - timedelta(days=7)
+        first_data_point = np.min(self.source.data['time_arr']).date()
+        last_data_point = np.max(self.source.data['time_arr']).date()
+        if last_data_point < one_week_ago:
+            # keep at least one point in the plot if there was
+            # no TA data this week
+            start_date = last_data_point
+        else:
+            start_date = one_week_ago
+
+        # allowed range is from the first ever data point to today
+        self.date_range = DateRangeSlider(
+            title="Date range displayed", start=first_data_point,
+            end=end_date, value=(start_date, end_date), step=1)
+
+        callback = CustomJS(args=dict(s=self.source), code="""
+            s.change.emit();
+        """)
+        self.date_range.js_on_change('value', callback)
+
+        filt = CustomJSFilter(args=dict(slider=self.date_range), code="""
+                var indices = [];
+                var start = slider.value[0];
+                var end = slider.value[1];
+
+                for (var i=0; i < source.get_length(); i++) {
+                    if (source.data['time_arr'][i] >= start 
+                            && source.data['time_arr'][i] <= end) {
+                        indices.push(true);
+                    } else {
+                        indices.push(false);
+                    }
+                }
+                return indices;
+                """)
+        self.date_view = CDSView(source=self.source, filters=[filt])
+
     def mk_plt_layout(self):
         """Create the bokeh plot layout"""
         self.source = ColumnDataSource(data=self.wata_data)
+
         # make sure all arrays are lists in order to later be able to read the data
         # from the html file
         for item in self.source.data:
             if not isinstance(self.source.data[item], (str, float, int, list)):
                 self.source.data[item] = self.source.data[item].tolist()
+
+        # add a time array to the data source
+        self.add_time_column()
+
+        # set up selection tools to share
+        self.share_tools = [BoxSelectTool()]
+
+        # set up a date range filter widget
+        self.setup_date_range()
+
         # set the output html file name and create the plot grid
         output_file(self.output_file_name)
         p1 = self.plt_status()
@@ -552,11 +648,14 @@ def mk_plt_layout(self):
         p4 = self.plt_v3offset_time()
         p5 = self.plt_centroid()
         p6 = self.plt_mag_time()
+
         # make grid
         grid = gridplot([p1, p2, p3, p4, p5, p6], ncols=2, merge_tools=False)
-        save(grid)
+        box_layout = layout(children=[self.date_range, grid])
+        save(box_layout)
+
         # return the needed components for embeding the results in the WATA html template
-        script, div = components(grid)
+        script, div = components(box_layout)
         return script, div
 
     def identify_tables(self):
diff --git a/jwql/tests/test_msata_monitor.py b/jwql/tests/test_msata_monitor.py
index 2a423d0ce..09be53586 100644
--- a/jwql/tests/test_msata_monitor.py
+++ b/jwql/tests/test_msata_monitor.py
@@ -87,7 +87,7 @@ def define_testdata():
                 }
     # add info from ta_table
     num_refstars = msata_dict['num_refstars'][0]
-    msata_dict['box_peak_value'] =  [[8000 for _ in range(num_refstars)]]
+    msata_dict['box_peak_value'] = [[8000 for _ in range(num_refstars)]]
     msata_dict['reference_star_mag'] = [[-999 for _ in range(num_refstars)]]
     msata_dict['convergence_status'] = [['SUCCESS' for _ in range(num_refstars)]]
     msata_dict['reference_star_number'] = [[i for i in range(num_refstars)]]
@@ -97,8 +97,9 @@ def define_testdata():
     msata_dict['lsf_removed_y'] = [[-999.0 for _ in range(num_refstars)]]
     msata_dict['planned_v2'] = [[-999.0 for _ in range(num_refstars)]]
     msata_dict['planned_v3'] = [[-999.0 for _ in range(num_refstars)]]
+
     # create the additional arrays
-    number_status, time_arr, status_colors = [], [], []
+    number_status, status_colors = [], []
     for tas, do_str in zip(msata_dict['ta_status'], msata_dict['date_obs']):
         if tas.lower() == 'unsuccessful':
             number_status.append(0.0)
@@ -109,14 +110,12 @@ def define_testdata():
         else:
             number_status.append(1.0)
             status_colors.append('blue')
-        # convert time string into an array of time (this is in UT with 0.0 milliseconds)
-        t = datetime.fromisoformat(do_str).timestamp()
-        time_arr.append(t)
+
     # add these to the bokeh data structure
-    msata_dict['time_arr'] = time_arr
     msata_dict['number_status'] = number_status
     msata_dict['status_colors'] = status_colors
-    # crate the dataframe
+
+    # create the dataframe
     msata_data = pd.DataFrame(msata_dict)
     return msata_data
 
@@ -163,6 +162,8 @@ def test_plt_status():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_status()
 
     assert bokeh_plot_type == type(result)
@@ -175,6 +176,8 @@ def test_plt_residual_offsets():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_residual_offsets()
 
     assert bokeh_plot_type == type(result)
@@ -187,6 +190,8 @@ def test_plt_v2offset_time():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_v2offset_time()
 
     assert bokeh_plot_type == type(result)
@@ -199,6 +204,8 @@ def test_plt_v3offset_time():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_v3offset_time()
 
     assert bokeh_plot_type == type(result)
@@ -211,6 +218,8 @@ def test_plt_lsv2v3offsetsigma():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_lsv2v3offsetsigma()
 
     assert bokeh_plot_type == type(result)
@@ -223,6 +232,8 @@ def test_plt_res_offsets_corrected():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_res_offsets_corrected()
 
     assert bokeh_plot_type == type(result)
@@ -235,6 +246,8 @@ def test_plt_v2offsigma_time():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_v2offsigma_time()
 
     assert bokeh_plot_type == type(result)
@@ -247,6 +260,8 @@ def test_plt_v3offsigma_time():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_v3offsigma_time()
 
     assert bokeh_plot_type == type(result)
@@ -259,6 +274,8 @@ def test_plt_roll_offset():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_roll_offset()
 
     assert bokeh_plot_type == type(result)
@@ -271,6 +288,8 @@ def test_plt_lsoffsetmag():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_lsoffsetmag()
 
     assert bokeh_plot_type == type(result)
@@ -283,6 +302,8 @@ def test_plt_tot_number_of_stars():
     ta = MSATA()
     msata_data = define_testdata()
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_tot_number_of_stars()
 
     assert bokeh_plot_type == type(result)
@@ -308,6 +329,8 @@ def test_plt_mags_time():
     msata_data['tot_number_of_stars'] = tot_number_of_stars
     msata_data['colors_list'] = colors_list
     ta.source = ColumnDataSource(data=msata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_mags_time()
 
     assert bokeh_plot_type == type(result)
diff --git a/jwql/tests/test_wata_monitor.py b/jwql/tests/test_wata_monitor.py
index a3bcc8157..def2c9ba2 100644
--- a/jwql/tests/test_wata_monitor.py
+++ b/jwql/tests/test_wata_monitor.py
@@ -93,7 +93,7 @@ def define_testdata():
                     'sam_y': [-999.0],
                 }
     # create the additional arrays
-    bool_status, time_arr, status_colors = [], [], []
+    bool_status, status_colors = [], []
     for tas, do_str in zip(wata_dict['ta_status'], wata_dict['date_obs']):
         if 'unsuccessful' not in tas.lower():
             bool_status.append(1)
@@ -101,14 +101,12 @@ def define_testdata():
         else:
             bool_status.append(0)
             status_colors.append('red')
-        # convert time string into an array of time (this is in UT with 0.0 milliseconds)
-        t = datetime.fromisoformat(do_str).timestamp()
-        time_arr.append(t)
+
     # add these to the bokeh data structure
-    wata_dict['time_arr'] = time_arr
     wata_dict['ta_status_bool'] = bool_status
     wata_dict['status_colors'] = status_colors
-    # crate the dataframe
+
+    # create the dataframe
     wata_data = pd.DataFrame(wata_dict)
     return wata_data
 
@@ -155,6 +153,8 @@ def test_plt_status():
     ta = WATA()
     wata_data = define_testdata()
     ta.source = ColumnDataSource(data=wata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_status()
 
     assert bokeh_plot_type == type(result)
@@ -167,6 +167,8 @@ def test_plt_residual_offsets():
     ta = WATA()
     wata_data = define_testdata()
     ta.source = ColumnDataSource(data=wata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_residual_offsets()
 
     assert bokeh_plot_type == type(result)
@@ -179,6 +181,8 @@ def test_plt_v2offset_time():
     ta = WATA()
     wata_data = define_testdata()
     ta.source = ColumnDataSource(data=wata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_v2offset_time()
 
     assert bokeh_plot_type == type(result)
@@ -191,6 +195,8 @@ def test_plt_v3offset_time():
     ta = WATA()
     wata_data = define_testdata()
     ta.source = ColumnDataSource(data=wata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
     result = ta.plt_v3offset_time()
 
     assert bokeh_plot_type == type(result)
@@ -203,6 +209,9 @@ def test_plt_mag_time():
     ta = WATA()
     wata_data = define_testdata()
     ta.source = ColumnDataSource(data=wata_data)
+    ta.add_time_column()
+    ta.setup_date_range()
+
     # create the arrays per filter and readout pattern
     nrsrapid_f140x, nrsrapid_f110w, nrsrapid_clear = [], [], []
     nrsrapidd6_f140x, nrsrapidd6_f110w, nrsrapidd6_clear = [], [], []
@@ -267,13 +276,16 @@ def test_plt_mag_time():
 
 
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
-def test_get_unsucessful_ta():
-    """Test the ``get_unsucessful_ta`` function"""
+def test_get_unsuccessful_ta():
+    """Test the ``get_unsuccessful_ta`` function"""
 
     ta = WATA()
     wata_data = define_testdata()
     ta.source = ColumnDataSource(data=wata_data)
-    list_failed, list_else = ta.get_unsucessful_ta('ta_status_bool')
+    ta.add_time_column()
+    ta.setup_date_range()
+
+    list_failed, list_else = ta.get_unsuccessful_ta('ta_status_bool')
 
     assert list_else[0] == ta.source.data['ta_status_bool'][0]
     assert np.isnan(list_failed[0])

From 97bb9677306d8fadfa955c933c2dc12a2972bb8e Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 2 Jun 2023 15:26:14 -0400
Subject: [PATCH 129/256] Add possible exception for group permissions

---
 jwql/utils/permissions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/utils/permissions.py b/jwql/utils/permissions.py
index b3117e006..2524b1c7c 100644
--- a/jwql/utils/permissions.py
+++ b/jwql/utils/permissions.py
@@ -240,7 +240,7 @@ def set_permissions(pathname, owner=DEFAULT_OWNER, mode=DEFAULT_MODES, group=DEF
             os.chmod(pathname, mode_to_use)
             # change group but not owner
             os.chown(pathname, -1, grp.getgrnam(group).gr_gid)
-        except PermissionError:
+        except (PermissionError, KeyError):
             pass
 
     if verbose:

From 9bed53f94f9cd971bb8b3be9a6a00aaf2e5226d9 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 2 Jun 2023 15:40:14 -0400
Subject: [PATCH 130/256] Dont email during test

---
 jwql/tests/test_protect_module.py | 24 +++++++++++++-----------
 jwql/utils/protect_module.py      | 28 +++++++++++++++-------------
 2 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/jwql/tests/test_protect_module.py b/jwql/tests/test_protect_module.py
index 6d4e78187..d0da6ab79 100644
--- a/jwql/tests/test_protect_module.py
+++ b/jwql/tests/test_protect_module.py
@@ -13,17 +13,13 @@
     These tests can be run via the command line (omit the -s to
     suppress verbose output to stdout):
 
-    !!!WARNING IF TESTING LOCALLY!!!
-    These tests will call protect_module.py::lock_module.py and will sent emails to jwql.stsci.edu!
-    Please locally change the `message['To']` recipient to an appropriate address while testing to
-    avoid spamming/confusing the team.
-
-
     ::
 
         pytest -s test_protect_module.py
 """
 import os
+from jwql.utils import protect_module as pm
+
 from pytest import fixture, mark
 from jwql.utils.protect_module import lock_module, _PID_LOCKFILE_KEY
 
@@ -38,6 +34,12 @@ def module_lock():
     return module_lock
 
 
+@fixture
+def do_not_email():
+    pm.ALERT_EMAIL = False
+    return pm.ALERT_EMAIL
+
+
 @lock_module
 def protected_code_verify_file_exists_true(module_lock):
     return os.path.exists(module_lock)
@@ -49,7 +51,7 @@ def protected_code_entered():
 
 
 @mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
-def test_lock_module_create_destroy_file(module_lock):
+def test_lock_module_create_destroy_file(module_lock, do_not_email):
     """Test wrapper will create and destroy a lock file named by module if no other lock exists """
 
     # Ensure lock file does not exist
@@ -62,7 +64,7 @@ def test_lock_module_create_destroy_file(module_lock):
 
 
 @mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
-def test_already_locked_module_wont_run_with_bad_lock_file(module_lock):
+def test_already_locked_module_wont_run_with_bad_lock_file(module_lock, do_not_email):
     """Test when wrapper encounters poorly formatted lock file,
        it will not run, not delete lock file, and will send warn email """
     # create locked file in advance of calling protected code
@@ -77,7 +79,7 @@ def test_already_locked_module_wont_run_with_bad_lock_file(module_lock):
 
 
 @mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
-def test_already_locked_module_wont_run_with_legit_lock_file(module_lock):
+def test_already_locked_module_wont_run_with_legit_lock_file(module_lock, do_not_email):
     """Test wrapper will not run if it encounters a correclty formatted lock_file """
     # create locked file with running PID in advance of calling protected code
     with open(module_lock, "w") as lock_file:
@@ -90,7 +92,7 @@ def test_already_locked_module_wont_run_with_legit_lock_file(module_lock):
 
 
 @mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
-def test_lock_module_handles_stale_lock_files(module_lock):
+def test_lock_module_handles_stale_lock_files(module_lock, do_not_email):
     """Test wrapper will correctly identify stale lock file,
        delete it, and continue to run successfully """
     # create locked file with not running PID in advance of calling protected code
@@ -99,4 +101,4 @@ def test_lock_module_handles_stale_lock_files(module_lock):
     file_created = protected_code_verify_file_exists_true(module_lock)
     file_exists = os.path.exists(module_lock)
     # Assert that lock file was created in wrapper, and removed upon exit
-    assert (file_created and not file_exists) is True
\ No newline at end of file
+    assert (file_created and not file_exists) is True
diff --git a/jwql/utils/protect_module.py b/jwql/utils/protect_module.py
index aa2d25230..acaaaa365 100644
--- a/jwql/utils/protect_module.py
+++ b/jwql/utils/protect_module.py
@@ -64,7 +64,9 @@ def protected_code():
 from functools import wraps
 from psutil import pid_exists
 
+_ALERT_ADDRESS = "jwql@stsci.edu"
 _PID_LOCKFILE_KEY = "Process Id = "
+ALERT_EMAIL = True  # Global to be turned off in tests
 
 
 def _clean_lock_file(filename, module):
@@ -125,19 +127,19 @@ def _send_notification(message):
         The message to be included in the email notification that will be sent out.
 
     '''
-
-    user = getpass.getuser()
-    hostname = socket.gethostname()
-    deliverer = '{}@stsci.edu'.format(user)
-
-    message = MIMEText(message)
-    message['Subject'] = f'JWQL ALERT FOR LOCK_MODULE ON {hostname}'
-    message['From'] = deliverer
-    message['To'] = 'jwql@stsci.edu'
-
-    s = smtplib.SMTP('smtp.stsci.edu')
-    s.send_message(message)
-    s.quit()
+    if ALERT_EMAIL:
+        user = getpass.getuser()
+        hostname = socket.gethostname()
+        deliverer = '{}@stsci.edu'.format(user)
+
+        message = MIMEText(message)
+        message['Subject'] = f'JWQL ALERT FOR LOCK_MODULE ON {hostname}'
+        message['From'] = deliverer
+        message['To'] = _ALERT_ADDRESS
+
+        s = smtplib.SMTP('smtp.stsci.edu')
+        s.send_message(message)
+        s.quit()
 
 
 def lock_module(func):

From 89b378d9cdbf86ce644a1ffdecb5642ae7822971 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Mon, 5 Jun 2023 10:30:19 -0400
Subject: [PATCH 131/256] Fix issue confusing filenames on monitor re-run

---
 .../nirspec_monitors/ta_monitors/msata_monitor.py         | 8 ++++----
 jwql/tests/test_msata_monitor.py                          | 5 ++++-
 jwql/tests/test_wata_monitor.py                           | 5 ++++-
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index 11bb6666c..cad0b2c1c 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -61,11 +61,11 @@
 from jwql.utils import monitor_utils
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.logging_functions import log_info, log_fail
-from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
 
 class MSATA():
-    """ Class for executint the NIRSpec MSATA monitor.
+    """ Class for executing the NIRSpec MSATA monitor.
 
     This class will search for new MSATA current files in the file systems
     for NIRSpec and will run the monitor on these files. The monitor will
@@ -848,7 +848,7 @@ def plt_mags_time(self):
 
         # now create the mini ColumnDataSource for this particular plot
         mini_source = {'vid': vid, 'star_no': star_no, 'status': status,
-                       'dobs': dobs, 'time_arr': tarr, 'det': det, 'filename': fnames,
+                       'dobs': dobs, 'time_arr': tarr, 'det': det, 'fname': fnames,
                        'peaks': peaks, 'colors_list': new_colors_list,
                        'stars_v2': stars_v2, 'stars_v3': stars_v3}
         mini_source = ColumnDataSource(data=mini_source)
@@ -883,7 +883,7 @@ def plt_mags_time(self):
 
         # add tooltips
         hover = HoverTool()
-        hover.tooltips = [('File name', '@filename'),
+        hover.tooltips = [('File name', '@fname'),
                           ('Visit ID', '@vid'),
                           ('Detector', '@det'),
                           ('Star No.', '@star_no'),
diff --git a/jwql/tests/test_msata_monitor.py b/jwql/tests/test_msata_monitor.py
index 09be53586..2b2e57e96 100644
--- a/jwql/tests/test_msata_monitor.py
+++ b/jwql/tests/test_msata_monitor.py
@@ -129,7 +129,10 @@ def test_mast_query_ta():
 
     # query mast
     result = monitor_utils.mast_query_ta('nirspec', 'NRS_FULL_MSA', query_start, query_end)
-    assert len(result) == 4
+
+    # eliminate duplicates (sometimes rate files are returned with cal files)
+    result = [r for r in result if r['productLevel'] == '2b']
+    assert len(result) >= 4
 
     # query local model
     alternate = monitor_utils.model_query_ta('nirspec', 'NRS_FULL_MSA', query_start, query_end)
diff --git a/jwql/tests/test_wata_monitor.py b/jwql/tests/test_wata_monitor.py
index def2c9ba2..6d5c096f9 100644
--- a/jwql/tests/test_wata_monitor.py
+++ b/jwql/tests/test_wata_monitor.py
@@ -120,11 +120,14 @@ def test_query_ta():
 
     # query mast
     result = monitor_utils.mast_query_ta('nirspec', 'NRS_S1600A1_SLIT', query_start, query_end)
+
+    # eliminate duplicates (sometimes rate files are returned with cal files)
+    result = [r for r in result if r['productLevel'] == '2b']
     assert len(result) == 16
 
     # query local model
     alternate = monitor_utils.model_query_ta('nirspec', 'NRS_S1600A1_SLIT', query_start, query_end)
-    assert len(alternate) == len(result)
+    assert len(alternate) == 16
 
     # check that filenames match up - model returns rootfiles, mast returns filenames
     result = sorted(result, key=lambda x: x['filename'])

From b311879650fc1af2c699940b1c2821f701d412f2 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Mon, 5 Jun 2023 10:46:48 -0400
Subject: [PATCH 132/256] Clean up style and spelling

---
 .../nirspec_monitors/ta_monitors/msata_monitor.py     |  7 ++++---
 .../nirspec_monitors/ta_monitors/wata_monitor.py      | 11 ++++++-----
 jwql/tests/test_msata_monitor.py                      |  2 +-
 jwql/tests/test_wata_monitor.py                       |  2 +-
 jwql/utils/monitor_utils.py                           |  2 +-
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index cad0b2c1c..55abcf95e 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -24,6 +24,7 @@
 Author
 ______
     - Maria Pena-Guerrero
+    - Melanie Clarke
 
 Use
 ---
@@ -1025,7 +1026,7 @@ def get_data_from_html(self, html_file):
             File created by the monitor script
         Returns
         -------
-        prev_data_dict: dictionary
+        prev_data_dict: dict
             Dictionary containing all data used in the plots
         """
 
@@ -1192,13 +1193,13 @@ def update_ta_success_txtfile(self):
         output_success_ta_txtfile = os.path.join(self.output_dir, "msata_success.txt")
         # check if previous file exsists and read the data from it
         if os.path.isfile(output_success_ta_txtfile):
-            # now rename the the previous file, for backup
+            # now rename the previous file, for backup
             os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_msata_success.txt"))
         # get the new data
         ta_success, ta_inprogress, ta_failure = [], [], []
         filenames, ta_status = self.msata_data.loc[:,'filename'], self.msata_data.loc[:,'ta_status']
         for fname, ta_stat in zip(filenames, ta_status):
-            # select the appriopriate list to append to
+            # select the appropriate list to append to
             if ta_stat == 'SUCCESSFUL':
                 ta_success.append(fname)
             elif ta_stat == 'IN_PROGRESS':
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index 28b533b1e..f248a1d09 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -10,7 +10,7 @@
 
 
 """
-This module contains the code for NIRSpec the Wide Aperture Target
+This module contains the code for the NIRSpec Wide Aperture Target
 Acquisition (WATA) monitor, which monitors the TA offsets.
 
 This monitor displays the comparison of desired versus measured TA.
@@ -20,6 +20,7 @@
 Author
 ______
     - Maria Pena-Guerrero
+    - Melanie Clarke
 
 Use
 ---
@@ -61,7 +62,7 @@
 
 
 class WATA():
-    """ Class for executint the NIRSpec WATA monitor.
+    """ Class for executing the NIRSpec WATA monitor.
 
     This class will search for new WATA current files in the file systems
     for NIRSpec and will run the monitor on these files. The monitor will
@@ -828,15 +829,15 @@ def update_ta_success_txtfile(self):
             Nothing
         """
         output_success_ta_txtfile = os.path.join(self.output_dir, "wata_success.txt")
-        # check if previous file exsists and read the data from it
+        # check if previous file exists and read the data from it
         if os.path.isfile(output_success_ta_txtfile):
-            # now rename the the previous file, for backup
+            # now rename the previous file, for backup
             os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_wata_success.txt"))
         # get the new data
         ta_success, ta_failure = [], []
         filenames, ta_status = self.wata_data.loc[:,'filename'], self.wata_data.loc[:,'ta_status']
         for fname, ta_stat in zip(filenames, ta_status):
-            # select the appriopriate list to append to
+            # select the appropriate list to append to
             if ta_stat == 'SUCCESSFUL':
                 ta_success.append(fname)
             else:
diff --git a/jwql/tests/test_msata_monitor.py b/jwql/tests/test_msata_monitor.py
index 2b2e57e96..fc0286d6c 100644
--- a/jwql/tests/test_msata_monitor.py
+++ b/jwql/tests/test_msata_monitor.py
@@ -132,7 +132,7 @@ def test_mast_query_ta():
 
     # eliminate duplicates (sometimes rate files are returned with cal files)
     result = [r for r in result if r['productLevel'] == '2b']
-    assert len(result) >= 4
+    assert len(result) == 4
 
     # query local model
     alternate = monitor_utils.model_query_ta('nirspec', 'NRS_FULL_MSA', query_start, query_end)
diff --git a/jwql/tests/test_wata_monitor.py b/jwql/tests/test_wata_monitor.py
index 6d5c096f9..0b9099beb 100644
--- a/jwql/tests/test_wata_monitor.py
+++ b/jwql/tests/test_wata_monitor.py
@@ -127,7 +127,7 @@ def test_query_ta():
 
     # query local model
     alternate = monitor_utils.model_query_ta('nirspec', 'NRS_S1600A1_SLIT', query_start, query_end)
-    assert len(alternate) == 16
+    assert len(alternate) == len(result)
 
     # check that filenames match up - model returns rootfiles, mast returns filenames
     result = sorted(result, key=lambda x: x['filename'])
diff --git a/jwql/utils/monitor_utils.py b/jwql/utils/monitor_utils.py
index 3b5297cc4..f5c673cbd 100644
--- a/jwql/utils/monitor_utils.py
+++ b/jwql/utils/monitor_utils.py
@@ -208,7 +208,7 @@ def mast_query_ta(instrument, aperture, start_date, end_date, readpatt=None):
 
         # Create dictionary of parameters to add
         parameters = {"date_obs_mjd": {"min": start_date, "max": end_date},
-                      "apername": aperture, "exp_type": template_name }
+                      "apername": aperture, "exp_type": template_name}
 
         if readpatt is not None:
             parameters["readpatt"] = readpatt

From 2ba25dea20d370f5c589b69597657c99378dc74f Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Mon, 5 Jun 2023 16:50:32 -0400
Subject: [PATCH 133/256] Adding proposal category and look status to query
 page

---
 jwql/utils/constants.py                       |  16 ++-
 jwql/website/apps/jwql/data_containers.py     |  44 +++++--
 jwql/website/apps/jwql/forms.py               |  29 +++--
 .../apps/jwql/templates/jwql_query.html       | 116 +++++++++++-------
 jwql/website/apps/jwql/views.py               |  12 +-
 5 files changed, 150 insertions(+), 67 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 4ab6733da..239e9bcd8 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -320,6 +320,9 @@
 JWST_MAST_SERVICES = ['Mast.Jwst.Filtered.{}'.format(value.title()) for value in
                       JWST_INSTRUMENT_NAMES]
 
+# Possible values for look status filter
+LOOK_OPTIONS = ['New', 'Viewed']
+
 # Maximum number of records returned by MAST for a single query
 MAST_QUERY_LIMIT = 500000
 
@@ -418,6 +421,9 @@
 # The complete name will have "_{instrument.lower}.txt" added to the end of this.
 PREVIEW_IMAGE_LISTFILE = 'preview_image_inventory'
 
+# All possible proposal categories
+PROPOSAL_CATEGORIES = ['AR', 'CAL', 'COM', 'DD', 'ENG', 'GO', 'GTO', 'NASA', 'SURVEY']
+
 PUPILS_PER_INSTRUMENT = {'nircam': ['CLEAR', 'FLAT', 'F162M', 'F164N', 'GDHS0', 'GDHS60', 'MASKBAR', 'MASKIPR', 'MASKRND',
                                     'PINHOLES', 'WLM8', 'WLP8', 'F323N', 'F405N', 'F466N', 'F470N', 'GRISMC', 'GRISMR', 'GRISMV2', 'GRISMV3'],
                          'niriss': ['CLEARP', 'F090W', 'F115W', 'F140M', 'F150W', 'F158M', 'F200W', 'GR700XD', 'NRM'],
@@ -430,15 +436,16 @@
 class QUERY_CONFIG_KEYS:
     ANOMALIES = "ANOMALIES"
     APERTURES = "APERTURES"
+    CAT_TYPE = "CAT_TYPE"
     DETECTORS = "DETECTORS"
-    EXP_TIME_MAX = "EXP_TIME_MAX"
-    EXP_TIME_MIN = "EXP_TIME_MIN"
     EXP_TYPES = "EXP_TYPES"
     FILTERS = "FILTERS"
     GRATINGS = "GRATINGS"
+    LOOK_STATUS = "LOOK_STATUS"
     INSTRUMENTS = "INSTRUMENTS"
     PUPILS = "PUPILS"
     READ_PATTS = "READ_PATTS"
+    SORT_TYPE = "SORT_TYPE"
     SUBARRAYS = "SUBARRAYS"
     THUMBNAILS = "THUMBNAILS"
 
@@ -448,8 +455,6 @@ class QUERY_CONFIG_KEYS:
     QUERY_CONFIG_KEYS.ANOMALIES: {},
     QUERY_CONFIG_KEYS.APERTURES: {},
     QUERY_CONFIG_KEYS.DETECTORS: {},
-    QUERY_CONFIG_KEYS.EXP_TIME_MAX: ['999999999999999'],
-    QUERY_CONFIG_KEYS.EXP_TIME_MIN: ['0'],
     QUERY_CONFIG_KEYS.EXP_TYPES: {},
     QUERY_CONFIG_KEYS.FILTERS: {},
     QUERY_CONFIG_KEYS.GRATINGS: {},
@@ -496,6 +501,9 @@ class QUERY_CONFIG_KEYS:
                               'nirspec': ['exp_type', 'filter', 'grating',
                                           'read_patt_num', 'viewed']}
 
+# Possible values for sort order
+SORT_OPTIONS = ['Ascending', 'Descending', 'Recent', 'Oldest']
+
 SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256']
 
 SUBARRAYS_PER_INSTRUMENT = {'nircam': ['FULL', 'FULLP', 'SUB640', 'SUB320', 'SUB160', 'SUB400P', 'SUB160P', 'SUB64P',
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 7b32f55d9..0a945efd7 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1507,6 +1507,17 @@ def get_rootnames_from_query(parameters):
         inst = inst.lower()
         current_ins_rootfileinfos = RootFileInfo.objects.filter(instrument=JWST_INSTRUMENT_NAMES_MIXEDCASE[inst])
 
+        # General fields
+        sort_type = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
+        look_status = parameters[QUERY_CONFIG_KEYS.LOOK_STATUS]
+        if len(look_status) == 1:
+            viewed = (look_status[0] == 'VIEWED')
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(viewed=viewed)
+        cat_type = parameters[QUERY_CONFIG_KEYS.CAT_TYPE]
+        if len(cat_type) > 0:
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(obsnum__proposal__category__in=cat_type)
+
+        # Instrument fields
         inst_anomalies = parameters[QUERY_CONFIG_KEYS.ANOMALIES][inst]
         inst_aperture = parameters[QUERY_CONFIG_KEYS.APERTURES][inst]
         inst_detector = parameters[QUERY_CONFIG_KEYS.DETECTORS][inst]
@@ -1535,25 +1546,36 @@ def get_rootnames_from_query(parameters):
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(subarray__in=inst_subarray)
 
         # TODO - This uncommented CODE is what we will use while DJANGO is still using the SQLITE3 DB.
-        #             ONCE DB IS MIGRATED TO POSTGRES WE CAN REPLACE THIS CODE WITH THE UNTESTED CODE COMMENTED OUT BELOW
+        #        ONCE DB IS MIGRATED TO POSTGRES WE CAN REPLACE THIS CODE WITH THE UNTESTED CODE COMMENTED OUT BELOW
         #      >>>  START CODE PRE DB MIGRATION HERE  <<<
         if (inst_anomalies != []):
             # If the rootfile info has any of the marked anomalies we want it
             # Make union of marked anomalies from our current query set, then save intersection.
-            anomaly_rootnames = []
-
+            anomaly_filters = None
             for anomaly in inst_anomalies:
                 anomaly_filter = "anomalies__" + str(anomaly).lower()
-                this_anomaly_rootnames = [name[0] for name in current_ins_rootfileinfos.filter(**{anomaly_filter: True}).values_list("root_name")]
-                anomaly_rootnames.extend(this_anomaly_rootnames)
-
-            filtered_rootnames.extend(anomaly_rootnames)
-        else:
-            rootnames = [name[0] for name in current_ins_rootfileinfos.values_list("root_name")]
-            filtered_rootnames.extend(rootnames)
+                this_anomaly_rootnames = current_ins_rootfileinfos.filter(**{anomaly_filter: True})
+                if anomaly_filters is None:
+                    anomaly_filters = this_anomaly_rootnames
+                else:
+                    anomaly_filters |= this_anomaly_rootnames
+            current_ins_rootfileinfos = anomaly_filters
+        #      >>>  END CODE PRE DB MIGRATION HERE  <<<
+
+        # sort as desired
+        if sort_type == 'DESCENDING':
+            current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('-root_name')
+        elif sort_type == 'ASCENDING':
+            current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('root_name')
+        elif sort_type == 'RECENT':
+            current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('-expstart', 'root_name')
+        elif sort_type == 'OLDEST':
+            current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('expstart', 'root_name')
+
+        rootnames = [name[0] for name in current_ins_rootfileinfos.values_list("root_name")]
+        filtered_rootnames.extend(rootnames)
 
     return list(set(filtered_rootnames))
-    #      >>>  END CODE PRE DB MIGRATION HERE  <<<
 
     # TODO - BELOW IS THE OUTLINE OF CODE WE WANT TO USE, HOWEVER THIS CAN'T BE IMPLEMENTED WITH DJANGO RUNNING SQLITE
     #             ONCE WE MIGRATE TO POSTGRES WE CAN IMPLEMENT THE BELOW FUNCTIONALITY WHICH SHOULD MAKE THIS CODE MOVE A LITTLE FASTER
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index f2fea0b38..029207de7 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -61,7 +61,8 @@ def view_function(request):
 
 from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, ANOMALIES_PER_INSTRUMENT, APERTURES_PER_INSTRUMENT, DETECTOR_PER_INSTRUMENT, EXP_TYPE_PER_INSTRUMENT,
                                   FILTERS_PER_INSTRUMENT, GENERIC_SUFFIX_TYPES, GRATING_PER_INSTRUMENT, GUIDER_FILENAME_TYPE, JWST_INSTRUMENT_NAMES_MIXEDCASE,
-                                  JWST_INSTRUMENT_NAMES_SHORTHAND, READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES, SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT)
+                                  JWST_INSTRUMENT_NAMES_SHORTHAND, READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES, SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT,
+                                  LOOK_OPTIONS, SORT_OPTIONS, PROPOSAL_CATEGORIES)
 from jwql.utils.utils import (get_config, get_rootnames_for_instrument_proposal, filename_parser, query_format)
 
 from wtforms import SubmitField, StringField
@@ -133,13 +134,27 @@ class JwqlQueryForm(BaseForm):
                 item = [query_format(anomaly), query_format(anomaly)]
                 params[instrument]['anomalies_list'].append(item)
 
-    # Anomaly Parameters
-    instrument = forms.MultipleChoiceField(required=False,
-                                           choices=[(inst, JWST_INSTRUMENT_NAMES_MIXEDCASE[inst]) for inst in JWST_INSTRUMENT_NAMES_MIXEDCASE],
-                                           widget=forms.CheckboxSelectMultiple)
-    exp_time_max = forms.DecimalField(required=False, initial="685")
-    exp_time_min = forms.DecimalField(required=False, initial="680")
+    # general parameters
+    instrument = forms.MultipleChoiceField(
+        required=False,
+        choices=[(inst, JWST_INSTRUMENT_NAMES_MIXEDCASE[inst]) for inst in JWST_INSTRUMENT_NAMES_MIXEDCASE],
+        widget=forms.CheckboxSelectMultiple)
 
+    look_choices = [(query_format(choice), query_format(choice)) for choice in LOOK_OPTIONS]
+    look_status = forms.MultipleChoiceField(
+        required=False, choices=look_choices, widget=forms.CheckboxSelectMultiple)
+
+    cat_choices = [(query_format(choice), query_format(choice)) for choice in PROPOSAL_CATEGORIES]
+    cat_type = forms.MultipleChoiceField(
+        required=False, choices=cat_choices, widget=forms.CheckboxSelectMultiple)
+
+    sort_choices = [(query_format(choice), query_format(choice)) for choice in SORT_OPTIONS]
+    sort_type = forms.ChoiceField(
+        required=True,
+        choices=sort_choices, initial=sort_choices[2],
+        widget=forms.RadioSelect)
+
+    # instrument specific parameters
     miri_aper = forms.MultipleChoiceField(required=False, choices=params['miri']['aperture_list'], widget=forms.CheckboxSelectMultiple)
     nirspec_aper = forms.MultipleChoiceField(required=False, choices=params['nirspec']['aperture_list'], widget=forms.CheckboxSelectMultiple)
     niriss_aper = forms.MultipleChoiceField(required=False, choices=params['niriss']['aperture_list'], widget=forms.CheckboxSelectMultiple)
diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index feac6ba96..9a8d3e724 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -6,7 +6,7 @@
     <style type="text/css">
         div#instrument_form_group { margin-right: 80px;
                                     margin-left: 80px; }
-        span#helpBlock.help-block {font-size: LARGER;}
+        span.help-block {font-size: LARGER;}
     </style>
 
     <title>Dynamic Query Form - JWQL</title>
@@ -75,21 +75,53 @@ <h1>Dynamic Query Form</h1>
                 <hr class="col-md-12">
                 <div class='instrument-form-group' id='instrument_form_group'>
                     <!-- Initial INSTRUMENT Checkbox -->
-                    <div class="col-md-2">
-                        <span id="helpBlock" class="help-block">Instruments</span>
-                        <div class="form-control" name='instrument' id='instrument'>
-                            {% for option in form.instrument %}
-                            <div class='radio'>
-                                {{ option }}
-                            </div>
-                            {% endfor %}
+                    <div style="display: flex" id="GENERAL">
+                        <div class="col-md-2">
+                            <span class="help-block">Instruments</span>
+                            <div class="form-control" id='instrument'>
+                                {% for option in form.instrument %}
+                                <div class='radio'>
+                                    {{ option }}
+                                </div>
+                                {% endfor %}
+                            </div>
+                        </div>
+                        <div class="col-md-2">
+                            <span class="help-block">Proposal Category</span>
+                            <div class="form-control" id='cat_type'>
+                                {% for option in form.cat_type %}
+                                <div class='radio'>
+                                    {{ option }}
+                                </div>
+                                {% endfor %}
+                            </div>
+                        </div>
+                        <div class="col-md-2">
+                            <span class="help-block">Look Status</span>
+                            <div class="form-control" id='look_status'>
+                                {% for option in form.look_status %}
+                                <div class='radio'>
+                                    {{ option }}
+                                </div>
+                                {% endfor %}
+                            </div>
+                        </div>
+                        <div class="col-md-2">
+                            <span class="help-block">Sort Type</span>
+                            <div class="form-control" id='sort_type'>
+                                {% for option in form.sort_type %}
+                                <div class='radio'>
+                                    {{ option }}
+                                </div>
+                                {% endfor %}
+                            </div>
                         </div>
                     </div>
 
                     <!-- FGS Fields -->
                     <div style="display: flex" id="FGS">
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">FGS Anomalies</span>
+                            <span class="help-block">FGS Anomalies</span>
                             <div class="form-control" id='fgs_anomalies'>
                                 {% for field in form.fgs_anomalies %}
                                 <div class="row-sm-10">
@@ -101,7 +133,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">FGS Apertures</span>
+                            <span class="help-block">FGS Apertures</span>
                             <div class="form-control" id='fgs_aper'>
                                 {% for field in form.fgs_aper %}
                                 <div class="row-sm-10">
@@ -113,7 +145,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">FGS Detectors</span>
+                            <span class="help-block">FGS Detectors</span>
                             <div class="form-control" id='fgs_detector'>
                                 {% for field in form.fgs_detector %}
                                 <div class="row-sm-10">
@@ -125,7 +157,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">FGS Exposure Types</span>
+                            <span class="help-block">FGS Exposure Types</span>
                             <div class="form-control" id='fgs_exptype'>
                                 {% for field in form.fgs_exptype %}
                                 <div class="row-sm-10">
@@ -137,7 +169,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">FGS Read Patterns</span>
+                            <span class="help-block">FGS Read Patterns</span>
                             <div class="form-control" id='fgs_readpatt'>
                                 {% for field in form.fgs_readpatt %}
                                 <div class="row-sm-10">
@@ -152,7 +184,7 @@ <h1>Dynamic Query Form</h1>
                     <!-- MIRI Fields -->
                     <div style="display: flex" id="MIRI">
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Anomalies</span>
+                            <span class="help-block">MIRI Anomalies</span>
                             <div class="form-control" id='miri_anomalies'>
                                 {% for field in form.miri_anomalies %}
                                 <div class="row-sm-10">
@@ -164,7 +196,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Detectors</span>
+                            <span class="help-block">MIRI Detectors</span>
                             <div class="form-control" id='miri_detector'>
                                 {% for field in form.miri_detector %}
                                 <div class="row-2">
@@ -176,7 +208,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Exposure Types</span>
+                            <span class="help-block">MIRI Exposure Types</span>
                             <div class="form-control" id='miri_exptype'>
                                 {% for field in form.miri_exptype %}
                                 <div class="row-2">
@@ -188,7 +220,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Filters</span>
+                            <span class="help-block">MIRI Filters</span>
                             <div class="form-control" id='miri_filt'>
                                 {% for field in form.miri_filt %}
                                 <div class="row-2">
@@ -200,7 +232,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Read Patterns</span>
+                            <span class="help-block">MIRI Read Patterns</span>
                             <div class="form-control" id='miri_readpatt'>
                                 {% for field in form.miri_readpatt %}
                                 <div class="row-2">
@@ -212,7 +244,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">MIRI Subarrays</span>
+                            <span class="help-block">MIRI Subarrays</span>
                             <div class="form-control" id='miri_subarray'>
                                 {% for field in form.miri_subarray %}
                                 <div class="row-2">
@@ -226,7 +258,7 @@ <h1>Dynamic Query Form</h1>
                     <!-- NIRCAM Fields -->
                     <div style="display: flex" id="NIRCAM">
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Anomalies</span>
+                            <span class="help-block">NIRCam Anomalies</span>
                             <div class="form-control" id='nircam_anomalies'>
                                 {% for field in form.nircam_anomalies %}
                                 <div class="row-sm-10">
@@ -238,7 +270,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Detectors</span>
+                            <span class="help-block">NIRCam Detectors</span>
                             <div class="form-control" id='nircam_detector'>
                                 {% for field in form.nircam_detector %}
                                 <div class="row-4">
@@ -250,7 +282,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Exposure Types</span>
+                            <span class="help-block">NIRCam Exposure Types</span>
                             <div class="form-control" id='nircam_exptype'>
                                 {% for field in form.nircam_exptype %}
                                 <div class="row-4">
@@ -262,7 +294,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Filters</span>
+                            <span class="help-block">NIRCam Filters</span>
                             <div class="form-control" id='nircam_filt'>
                                 {% for field in form.nircam_filt %}
                                 <div class="row-sm-10">
@@ -274,7 +306,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Pupils</span>
+                            <span class="help-block">NIRCam Pupils</span>
                             <div class="form-control" id='nircam_pupil'>
                                 {% for field in form.nircam_pupil %}
                                 <div class="row-4">
@@ -286,7 +318,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Read Patterns</span>
+                            <span class="help-block">NIRCam Read Patterns</span>
                             <div class="form-control" id='nircam_readpatt'>
                                 {% for field in form.nircam_readpatt %}
                                 <div class="row-4">
@@ -298,7 +330,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRCam Subarrays</span>
+                            <span class="help-block">NIRCam Subarrays</span>
                             <div class="form-control" id='nircam_subarray'>
                                 {% for field in form.nircam_subarray %}
                                 <div class="row-4">
@@ -313,7 +345,7 @@ <h1>Dynamic Query Form</h1>
                     <!-- NIRISS Fields -->
                     <div style="display: flex" id="NIRISS">
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Anomalies</span>
+                            <span class="help-block">NIRISS Anomalies</span>
                             <div class="form-control" id='niriss_anomalies'>
                                 {% for field in form.niriss_anomalies %}
                                 <div class="row-sm-10">
@@ -325,7 +357,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Detectors</span>
+                            <span class="help-block">NIRISS Detectors</span>
                             <div class="form-control" id='niriss_detector'>
                                 {% for field in form.niriss_detector %}
                                 <div class="row-sm-10">
@@ -337,7 +369,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Exposure Types</span>
+                            <span class="help-block">NIRISS Exposure Types</span>
                             <div class="form-control" id='niriss_exptype'>
                                 {% for field in form.niriss_exptype %}
                                 <div class="row-sm-10">
@@ -349,7 +381,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Filters</span>
+                            <span class="help-block">NIRISS Filters</span>
                             <div class="form-control" id='niriss_filt'>
                                 {% for field in form.niriss_filt %}
                                 <div class="row-sm-10">
@@ -361,7 +393,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Pupils</span>
+                            <span class="help-block">NIRISS Pupils</span>
                             <div class="form-control" id='niriss_pupil'>
                                 {% for field in form.niriss_pupil %}
                                 <div class="row-sm-10">
@@ -373,7 +405,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Read Patterns</span>
+                            <span class="help-block">NIRISS Read Patterns</span>
                             <div class="form-control" id='niriss_readpatt'>
                                 {% for field in form.niriss_readpatt %}
                                 <div class="row-sm-10">
@@ -385,7 +417,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRISS Subarrays</span>
+                            <span class="help-block">NIRISS Subarrays</span>
                             <div class="form-control" id='niriss_subarray'>
                                 {% for field in form.niriss_subarray %}
                                 <div class="row-sm-10">
@@ -400,7 +432,7 @@ <h1>Dynamic Query Form</h1>
                     <!-- NIRSPEC Fields -->
                     <div style="display: flex" id="NIRSPEC">
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Anomalies</span>
+                            <span class="help-block">NIRSpec Anomalies</span>
                             <div class="form-control" id='nirspec_anomalies'>
                                 {% for field in form.nirspec_anomalies %}
                                 <div class="row-sm-10">
@@ -412,7 +444,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Apertures</span>
+                            <span class="help-block">NIRSpec Apertures</span>
                             <div class="form-control" id='nirspec_aper'>
                                 {% for field in form.nirspec_aper %}
                                 <div class="row-sm-10">
@@ -424,7 +456,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Detectors</span>
+                            <span class="help-block">NIRSpec Detectors</span>
                             <div class="form-control" id='nirspec_detector'>
                                 {% for field in form.nirspec_detector %}
                                 <div class="row-sm-10">
@@ -436,7 +468,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Exposure Types</span>
+                            <span class="help-block">NIRSpec Exposure Types</span>
                             <div class="form-control" id='nirspec_exptype'>
                                 {% for field in form.nirspec_exptype %}
                                 <div class="row-sm-10">
@@ -448,7 +480,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Filters</span>
+                            <span class="help-block">NIRSpec Filters</span>
                             <div class="form-control" id='nirspec_filt'>
                                 {% for field in form.nirspec_filt %}
                                 <div class="row-sm-10">
@@ -460,7 +492,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Gratings</span>
+                            <span class="help-block">NIRSpec Gratings</span>
                             <div class="form-control" id='nirspec_grating'>
                                 {% for field in form.nirspec_grating %}
                                 <div class="row-sm-10">
@@ -472,7 +504,7 @@ <h1>Dynamic Query Form</h1>
                         </section>
 
                         <section class="col-md">
-                            <span id="helpBlock" class="help-block">NIRSpec Read Patterns</span>
+                            <span class="help-block">NIRSpec Read Patterns</span>
                             <div class="form-control" id='nirspec_readpatt'>
                                 {% for field in form.nirspec_readpatt %}
                                 <div class="row-sm-10">
@@ -492,4 +524,4 @@ <h1>Dynamic Query Form</h1>
             </form>
         </form>
     </div>
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 8b4712a90..9d5b5e02c 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -53,6 +53,7 @@
 
 from bokeh.layouts import layout
 from bokeh.embed import components
+from django.core.paginator import Paginator
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import redirect, render
 from sqlalchemy import inspect
@@ -123,6 +124,9 @@ def jwql_query(request):
 
             parameters = QUERY_CONFIG_TEMPLATE.copy()
             parameters[QUERY_CONFIG_KEYS.INSTRUMENTS] = form.cleaned_data['instrument']
+            parameters[QUERY_CONFIG_KEYS.LOOK_STATUS] = form.cleaned_data['look_status']
+            parameters[QUERY_CONFIG_KEYS.CAT_TYPE] = form.cleaned_data['cat_type']
+            parameters[QUERY_CONFIG_KEYS.SORT_TYPE] = form.cleaned_data['sort_type']
             parameters[QUERY_CONFIG_KEYS.ANOMALIES] = all_anomalies
             parameters[QUERY_CONFIG_KEYS.APERTURES] = all_apers
             parameters[QUERY_CONFIG_KEYS.FILTERS] = all_filters
@@ -132,8 +136,6 @@ def jwql_query(request):
             parameters[QUERY_CONFIG_KEYS.GRATINGS] = all_gratings
             parameters[QUERY_CONFIG_KEYS.SUBARRAYS] = all_subarrays
             parameters[QUERY_CONFIG_KEYS.PUPILS] = all_pupils
-            parameters[QUERY_CONFIG_KEYS.EXP_TIME_MIN] = str(form.cleaned_data['exp_time_min'])
-            parameters[QUERY_CONFIG_KEYS.EXP_TIME_MAX] = str(form.cleaned_data['exp_time_max'])
 
             # save the query config settings to a session
             request.session['query_config'] = parameters
@@ -208,8 +210,12 @@ def save_page_navigation_data_ajax(request):
     if request.method == 'POST':
         navigate_dict = request.POST.get('navigate_dict')
         # Save session in form {rootname:expstart}
-        rootname_expstarts = dict(item.split("=") for item in navigate_dict.split(","))
+        rootname_expstarts = dict()
+        for item in navigate_dict.split(','):
+            rootname, expstart = item.split("=")
+            rootname_expstarts[rootname] = float(expstart)
         request.session['navigation_data'] = rootname_expstarts
+
     context = {'item': request.session['navigation_data']}
     return JsonResponse(context, json_dumps_params={'indent': 2})
 

From e630c0e8d48ff65df24dd71d3df0f660081e36d8 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Mon, 5 Jun 2023 17:58:40 -0400
Subject: [PATCH 134/256] Add pagination for query results

---
 jwql/utils/constants.py                       |  2 ++
 jwql/website/apps/jwql/data_containers.py     | 10 +++----
 jwql/website/apps/jwql/forms.py               |  2 +-
 jwql/website/apps/jwql/static/js/jwql.js      | 30 +++++++++++++++++--
 .../apps/jwql/templates/query_submit.html     |  5 ++--
 jwql/website/apps/jwql/views.py               | 23 +++++++++++---
 6 files changed, 58 insertions(+), 14 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 239e9bcd8..6048dcf8e 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -454,6 +454,7 @@ class QUERY_CONFIG_KEYS:
 QUERY_CONFIG_TEMPLATE = {
     QUERY_CONFIG_KEYS.ANOMALIES: {},
     QUERY_CONFIG_KEYS.APERTURES: {},
+    QUERY_CONFIG_KEYS.CAT_TYPE: [],
     QUERY_CONFIG_KEYS.DETECTORS: {},
     QUERY_CONFIG_KEYS.EXP_TYPES: {},
     QUERY_CONFIG_KEYS.FILTERS: {},
@@ -461,6 +462,7 @@ class QUERY_CONFIG_KEYS:
     QUERY_CONFIG_KEYS.INSTRUMENTS: [],
     QUERY_CONFIG_KEYS.PUPILS: {},
     QUERY_CONFIG_KEYS.READ_PATTS: {},
+    QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent',
     QUERY_CONFIG_KEYS.SUBARRAYS: {},
     QUERY_CONFIG_KEYS.THUMBNAILS: []
     }
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 0a945efd7..d925285a1 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1563,14 +1563,14 @@ def get_rootnames_from_query(parameters):
         #      >>>  END CODE PRE DB MIGRATION HERE  <<<
 
         # sort as desired
-        if sort_type == 'DESCENDING':
-            current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('-root_name')
-        elif sort_type == 'ASCENDING':
+        if sort_type.upper() == 'ASCENDING':
             current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('root_name')
-        elif sort_type == 'RECENT':
+        elif sort_type.upper() == 'RECENT':
             current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('-expstart', 'root_name')
-        elif sort_type == 'OLDEST':
+        elif sort_type.upper() == 'OLDEST':
             current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('expstart', 'root_name')
+        else:
+            current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('-root_name')
 
         rootnames = [name[0] for name in current_ins_rootfileinfos.values_list("root_name")]
         filtered_rootnames.extend(rootnames)
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index 029207de7..6b875b0e1 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -148,7 +148,7 @@ class JwqlQueryForm(BaseForm):
     cat_type = forms.MultipleChoiceField(
         required=False, choices=cat_choices, widget=forms.CheckboxSelectMultiple)
 
-    sort_choices = [(query_format(choice), query_format(choice)) for choice in SORT_OPTIONS]
+    sort_choices = [(choice, choice) for choice in SORT_OPTIONS]
     sort_type = forms.ChoiceField(
         required=True,
         choices=sort_choices, initial=sort_choices[2],
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index e86dfa9c6..731b2ebe0 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1224,6 +1224,29 @@ function update_obs_options(data, inst, prop, observation) {
     $("#obs-list")[0].innerHTML = content;
 }
 
+/**
+ * Update the pagination div with navigation links
+ * @param {Object} data - The data returned by the query_submit AJAX method
+ */
+function update_pagination(data) {
+    var content = '<span class="step-links">'
+    if ('previous_page' in data) {
+        content += '<a href="?page=1">&laquo; first</a> ' +
+            '<a href="?page=' + data.previous_page + '">previous</a>';
+    }
+    content += '<span class="current">Page ' + data.current_page + ' of ' + data.total_pages + '</span>'
+
+    if ('next_page' in data) {
+        content += '<a href="?page=' + data.next_page + '">next</a> ' +
+            '<a href="?page=' + data.total_pages + '">last &raquo;</a>';
+    }
+    content += '</span>';
+    $("#pagination")[0].innerHTML = content;
+
+    // Add the total file count to the img_show_count banner
+    $("#query_total")[0].innerHTML = '<p>Query returned ' + data.total_files + ' files total.</p>';
+}
+
 /**
  * Updates the img_show_count component
  * @param {Integer} count - The count to display
@@ -1233,6 +1256,7 @@ function update_show_count(count, type) {
     var content = 'Showing <a id="img_shown">' + count + '</a> / <a id="img_total">' + count + '</a> <a id="img_type">' + type + '</a>';
     content += '<a href="https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/science_products.html" target="_blank" style="color: black">';
     content += '<span class="help-tip mx-2">i</span></a>';
+    content += '<span id="query_total"></span>';
     $("#img_show_count")[0].innerHTML = content;
 }
 
@@ -1426,10 +1450,11 @@ function update_thumbnails_per_observation_page(inst, proposal, observation, bas
  * Updates various components on the thumbnails anomaly query page
  * @param {String} base_url - The base URL for gathering data from the AJAX view.
  * @param {String} sort - Sort method string saved in session data image_sort
+ * @param {Int} page - Page number to load
  */
-function update_thumbnails_query_page(base_url, sort, group) {
+function update_thumbnails_query_page(base_url, sort, group, page) {
     $.ajax({
-        url: base_url + '/ajax/query_submit/',
+        url: base_url + '/ajax/query_submit/?page=' + page,
         success: function(data){
             // Perform various updates to divs
             var num_thumbnails = Object.keys(data.file_data).length;
@@ -1438,6 +1463,7 @@ function update_thumbnails_query_page(base_url, sort, group) {
             update_filter_options(data, base_url, 'thumbnail');
             update_group_options(data, base_url);
             update_sort_options(data, base_url);
+            update_pagination(data);
 
             // Do initial sort and group to match sort button display
             group_by_thumbnails(group, base_url);
diff --git a/jwql/website/apps/jwql/templates/query_submit.html b/jwql/website/apps/jwql/templates/query_submit.html
index 7bac5a290..1f349843c 100644
--- a/jwql/website/apps/jwql/templates/query_submit.html
+++ b/jwql/website/apps/jwql/templates/query_submit.html
@@ -53,8 +53,9 @@ <h3 id='title'>Images of Queried Instruments</h3>
         </div>
 
         <!-- Display the data -->
-        <div id='thumbnail-array'></div>
-        <script>update_thumbnails_query_page('{{base_url}}', '{{ sort }}', '{{ group }}');</script>
+        <div id="pagination" class="pagination"></div>
+        <div id="thumbnail-array"></div>
+        <script>update_thumbnails_query_page('{{base_url}}', '{{ sort }}', '{{ group }}', '{{ page }}');</script>
         <a id="no_thumbnails_msg" style='display: none'>No data match the selected criteria.</a>
 
         <hr>
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 9d5b5e02c..28b63cb34 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -478,10 +478,24 @@ def archive_thumbnails_query_ajax(request):
 
     parameters = request.session.get("query_config", QUERY_CONFIG_TEMPLATE.copy())
     filtered_rootnames = get_rootnames_from_query(parameters)
-    data = thumbnails_query_ajax(filtered_rootnames)
-    data['thumbnail_sort'] = request.session.get("image_sort", "Ascending")
+
+    paginator = Paginator(filtered_rootnames, 500)
+    page_number = request.GET.get("page", 1)
+    page_obj = paginator.get_page(page_number)
+
+    data = thumbnails_query_ajax(page_obj.object_list)
+    data['thumbnail_sort'] = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
     data['thumbnail_group'] = request.session.get("image_group", "Exposure")
 
+    # pass pagination info
+    if page_obj.has_previous():
+        data['previous_page'] = page_obj.previous_page_number()
+    data['current_page'] = page_obj.number
+    if page_obj.has_next():
+        data['next_page'] = page_obj.next_page_number()
+    data['total_pages'] = paginator.num_pages
+    data['total_files'] = paginator.count
+
     save_page_navigation_data(request, data)
     return JsonResponse(data, json_dumps_params={'indent': 2})
 
@@ -834,11 +848,12 @@ def query_submit(request):
     template = 'query_submit.html'
     sort_type = request.session.get('image_sort', 'Ascending')
     group_type = request.session.get('image_group', 'Exposure')
+    page_number = request.GET.get("page", 1)
     context = {'inst': '',
                'base_url': get_base_url(),
                'sort': sort_type,
-               'group': group_type
-               }
+               'group': group_type,
+               'page': page_number}
 
     return render(request, template, context)
 

From b3956ab12abea3a852c50410313bdb911d559635 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 6 Jun 2023 10:27:05 -0400
Subject: [PATCH 135/256] Add results per page configuration to query

---
 jwql/utils/constants.py                          |  2 ++
 jwql/website/apps/jwql/forms.py                  |  6 ++++++
 jwql/website/apps/jwql/static/js/jwql.js         |  2 +-
 jwql/website/apps/jwql/templates/jwql_query.html | 15 ++++++++++++---
 jwql/website/apps/jwql/views.py                  |  3 ++-
 5 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 6048dcf8e..ced832ddf 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -443,6 +443,7 @@ class QUERY_CONFIG_KEYS:
     GRATINGS = "GRATINGS"
     LOOK_STATUS = "LOOK_STATUS"
     INSTRUMENTS = "INSTRUMENTS"
+    NUM_PER_PAGE = "NUM_PER_PAGE"
     PUPILS = "PUPILS"
     READ_PATTS = "READ_PATTS"
     SORT_TYPE = "SORT_TYPE"
@@ -460,6 +461,7 @@ class QUERY_CONFIG_KEYS:
     QUERY_CONFIG_KEYS.FILTERS: {},
     QUERY_CONFIG_KEYS.GRATINGS: {},
     QUERY_CONFIG_KEYS.INSTRUMENTS: [],
+    QUERY_CONFIG_KEYS.NUM_PER_PAGE: 100,
     QUERY_CONFIG_KEYS.PUPILS: {},
     QUERY_CONFIG_KEYS.READ_PATTS: {},
     QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent',
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index 6b875b0e1..1163dafc5 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -154,6 +154,12 @@ class JwqlQueryForm(BaseForm):
         choices=sort_choices, initial=sort_choices[2],
         widget=forms.RadioSelect)
 
+    num_choices = [(50, 50), (100, 100), (200, 200), (500, 500)]
+    num_per_page = forms.ChoiceField(
+        required=True,
+        choices=num_choices, initial=num_choices[1],
+        widget=forms.RadioSelect)
+
     # instrument specific parameters
     miri_aper = forms.MultipleChoiceField(required=False, choices=params['miri']['aperture_list'], widget=forms.CheckboxSelectMultiple)
     nirspec_aper = forms.MultipleChoiceField(required=False, choices=params['nirspec']['aperture_list'], widget=forms.CheckboxSelectMultiple)
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 731b2ebe0..939aab5b9 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1244,7 +1244,7 @@ function update_pagination(data) {
     $("#pagination")[0].innerHTML = content;
 
     // Add the total file count to the img_show_count banner
-    $("#query_total")[0].innerHTML = '<p>Query returned ' + data.total_files + ' files total.</p>';
+    $("#query_total")[0].innerHTML = '<p>Query returned ' + data.total_files + ' activities total.</p>';
 }
 
 /**
diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index 9a8d3e724..2cd418947 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -61,10 +61,9 @@
 
     <main role="main" class="container">
         <h1>Dynamic Query Form</h1>
-        <p> Use this form to query the archive for any instrument or combination of instruments.
-            Enter the instruments for which you would like to view the archive and other constraints.
+        <p> Use this form to query the archive for data from any instrument or combination of instruments.</p>
+        <p> Select at least one instrument, then any other constraints desired.
             Feel free to leave fields blank if you do not want to further constrain the database search.
-            Note, however, that it may increase the run time if you search a wider array of images.
         </p>
     </main>
 
@@ -116,6 +115,16 @@ <h1>Dynamic Query Form</h1>
                                 {% endfor %}
                             </div>
                         </div>
+                        <div class="col-md-2">
+                            <span class="help-block">Results Per Page</span>
+                            <div class="form-control" id='num_per_page'>
+                                {% for option in form.num_per_page %}
+                                <div class='radio'>
+                                    {{ option }}
+                                </div>
+                                {% endfor %}
+                            </div>
+                        </div>
                     </div>
 
                     <!-- FGS Fields -->
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 28b63cb34..396b0b036 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -479,7 +479,8 @@ def archive_thumbnails_query_ajax(request):
     parameters = request.session.get("query_config", QUERY_CONFIG_TEMPLATE.copy())
     filtered_rootnames = get_rootnames_from_query(parameters)
 
-    paginator = Paginator(filtered_rootnames, 500)
+    paginator = Paginator(filtered_rootnames,
+                          parameters[QUERY_CONFIG_KEYS.NUM_PER_PAGE])
     page_number = request.GET.get("page", 1)
     page_obj = paginator.get_page(page_number)
 

From c65fc63b731e7eba706323d87eed5cc8cf9a73a0 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 6 Jun 2023 11:06:08 -0400
Subject: [PATCH 136/256] Link query sort preference to session sort preference

---
 jwql/website/apps/jwql/data_containers.py          |  8 ++++----
 jwql/website/apps/jwql/static/js/jwql.js           |  9 ++++-----
 jwql/website/apps/jwql/templates/query_submit.html |  2 +-
 jwql/website/apps/jwql/views.py                    | 14 ++++++++------
 4 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index d925285a1..a3a741847 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1495,8 +1495,8 @@ def get_rootnames_from_query(parameters):
 
     Returns
     -------
-    filtered_rootnames : query_set
-        A query_set of all rootfileinfos filtered from the given parameters
+    filtered_rootnames : list
+        A list of all root filenames filtered from the given parameters
     """
     # TODO - This code setup is temporary until the merge to Postgres is complete.
     # selected_rootfileinfos = RootFileInfo.objects.none()
@@ -1572,10 +1572,10 @@ def get_rootnames_from_query(parameters):
         else:
             current_ins_rootfileinfos = current_ins_rootfileinfos.order_by('-root_name')
 
-        rootnames = [name[0] for name in current_ins_rootfileinfos.values_list("root_name")]
+        rootnames = [name[0] for name in current_ins_rootfileinfos.values_list('root_name')]
         filtered_rootnames.extend(rootnames)
 
-    return list(set(filtered_rootnames))
+    return filtered_rootnames
 
     # TODO - BELOW IS THE OUTLINE OF CODE WE WANT TO USE, HOWEVER THIS CAN'T BE IMPLEMENTED WITH DJANGO RUNNING SQLITE
     #             ONCE WE MIGRATE TO POSTGRES WE CAN IMPLEMENT THE BELOW FUNCTIONALITY WHICH SHOULD MAKE THIS CODE MOVE A LITTLE FASTER
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 939aab5b9..9069b7bba 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1449,10 +1449,9 @@ function update_thumbnails_per_observation_page(inst, proposal, observation, bas
 /**
  * Updates various components on the thumbnails anomaly query page
  * @param {String} base_url - The base URL for gathering data from the AJAX view.
- * @param {String} sort - Sort method string saved in session data image_sort
  * @param {Int} page - Page number to load
  */
-function update_thumbnails_query_page(base_url, sort, group, page) {
+function update_thumbnails_query_page(base_url, page) {
     $.ajax({
         url: base_url + '/ajax/query_submit/?page=' + page,
         success: function(data){
@@ -1462,12 +1461,12 @@ function update_thumbnails_query_page(base_url, sort, group, page) {
             update_thumbnail_array(data);
             update_filter_options(data, base_url, 'thumbnail');
             update_group_options(data, base_url);
-            update_sort_options(data, base_url);
+            update_sort_options(data.thumbnail_sort, base_url);
             update_pagination(data);
 
             // Do initial sort and group to match sort button display
-            group_by_thumbnails(group, base_url);
-            sort_by_thumbnails(sort, base_url);
+            group_by_thumbnails(data.thumbnail_group, base_url);
+            sort_by_thumbnails(data.thumbnail_sort, base_url);
 
             // Replace loading screen with the proposal array div
             document.getElementById("loading").style.display = "none";
diff --git a/jwql/website/apps/jwql/templates/query_submit.html b/jwql/website/apps/jwql/templates/query_submit.html
index 1f349843c..0810aee00 100644
--- a/jwql/website/apps/jwql/templates/query_submit.html
+++ b/jwql/website/apps/jwql/templates/query_submit.html
@@ -55,7 +55,7 @@ <h3 id='title'>Images of Queried Instruments</h3>
         <!-- Display the data -->
         <div id="pagination" class="pagination"></div>
         <div id="thumbnail-array"></div>
-        <script>update_thumbnails_query_page('{{base_url}}', '{{ sort }}', '{{ group }}', '{{ page }}');</script>
+        <script>update_thumbnails_query_page('{{base_url}}', '{{ page }}');</script>
         <a id="no_thumbnails_msg" style='display: none'>No data match the selected criteria.</a>
 
         <hr>
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 396b0b036..b874ed89d 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -94,6 +94,7 @@ def jwql_query(request):
     """The anomaly query form page"""
 
     form = JwqlQueryForm(request.POST or None)
+    form.fields['sort_type'].initial = request.session.get('image_sort', 'Recent')
 
     if request.method == 'POST':
         if form.is_valid():
@@ -399,7 +400,7 @@ def archive_thumbnails_ajax(request, inst, proposal, observation=None):
     inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()]
 
     data = thumbnails_ajax(inst, proposal, obs_num=observation)
-    data['thumbnail_sort'] = request.session.get("image_sort", "Ascending")
+    data['thumbnail_sort'] = request.session.get("image_sort", "Recent")
     data['thumbnail_group'] = request.session.get("image_group", "Exposure")
 
     save_page_navigation_data(request, data)
@@ -443,7 +444,7 @@ def archive_thumbnails_per_observation(request, inst, proposal, observation):
 
     obs_list = sorted(list(set(all_obs)))
 
-    sort_type = request.session.get('image_sort', 'Ascending')
+    sort_type = request.session.get('image_sort', 'Recent')
     group_type = request.session.get('image_group', 'Exposure')
     template = 'thumbnails_per_obs.html'
     context = {'base_url': get_base_url(),
@@ -497,12 +498,13 @@ def archive_thumbnails_query_ajax(request):
     data['total_pages'] = paginator.num_pages
     data['total_files'] = paginator.count
 
+    request.session['image_sort'] = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
     save_page_navigation_data(request, data)
     return JsonResponse(data, json_dumps_params={'indent': 2})
 
 
 def dashboard(request):
-    """Generate the dashbaord page
+    """Generate the dashboard page
 
     Parameters
     ----------
@@ -847,7 +849,7 @@ def query_submit(request):
     """
 
     template = 'query_submit.html'
-    sort_type = request.session.get('image_sort', 'Ascending')
+    sort_type = request.session.get('image_sort', 'Recent')
     group_type = request.session.get('image_group', 'Exposure')
     page_number = request.GET.get("page", 1)
     context = {'inst': '',
@@ -1225,7 +1227,7 @@ def view_exposure(request, inst, group_root):
 
     # For time based sorting options, sort to "Recent" first to create sorting consistency when times are the same.
     # This is consistent with how Tinysort is utilized in jwql.js->sort_by_thumbnails
-    sort_type = request.session.get('image_sort', 'Ascending')
+    sort_type = request.session.get('image_sort', 'Recent')
     if sort_type in ['Descending']:
         matching_rootfiles = sorted(navigation_data, reverse=True)
     elif sort_type in ['Recent']:
@@ -1323,7 +1325,7 @@ def view_image(request, inst, file_root):
     # sorting consistency when times are the same.
     # This is consistent with how Tinysort is utilized in
     # jwql.js->sort_by_thumbnails
-    sort_type = request.session.get('image_sort', 'Ascending')
+    sort_type = request.session.get('image_sort', 'Recent')
     if sort_type in ['Descending']:
         file_root_list = sorted(navigation_data, reverse=True)
     elif sort_type in ['Recent']:

From 4dae5a7a03c6f5af9a5b9a8888112cfa48710984 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 6 Jun 2023 11:24:27 -0400
Subject: [PATCH 137/256] Tidy pagination style

---
 jwql/website/apps/jwql/static/js/jwql.js           | 12 ++++++------
 jwql/website/apps/jwql/templates/query_submit.html |  1 +
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 9069b7bba..d5e63a393 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1229,16 +1229,16 @@ function update_obs_options(data, inst, prop, observation) {
  * @param {Object} data - The data returned by the query_submit AJAX method
  */
 function update_pagination(data) {
-    var content = '<span class="step-links">'
+    var content = '<span class="step-links mx-2">'
     if ('previous_page' in data) {
-        content += '<a href="?page=1">&laquo; first</a> ' +
-            '<a href="?page=' + data.previous_page + '">previous</a>';
+        content += '<a class="mx-2" href="?page=1">&laquo; first</a> ' +
+            '<a class="mx-2" href="?page=' + data.previous_page + '">previous</a>';
     }
-    content += '<span class="current">Page ' + data.current_page + ' of ' + data.total_pages + '</span>'
+    content += '<span class="current mx-2">Page ' + data.current_page + ' of ' + data.total_pages + '</span>'
 
     if ('next_page' in data) {
-        content += '<a href="?page=' + data.next_page + '">next</a> ' +
-            '<a href="?page=' + data.total_pages + '">last &raquo;</a>';
+        content += '<a class="mx-2" href="?page=' + data.next_page + '">next</a> ' +
+            '<a class="mx-2" href="?page=' + data.total_pages + '">last &raquo;</a>';
     }
     content += '</span>';
     $("#pagination")[0].innerHTML = content;
diff --git a/jwql/website/apps/jwql/templates/query_submit.html b/jwql/website/apps/jwql/templates/query_submit.html
index 0810aee00..585e55111 100644
--- a/jwql/website/apps/jwql/templates/query_submit.html
+++ b/jwql/website/apps/jwql/templates/query_submit.html
@@ -54,6 +54,7 @@ <h3 id='title'>Images of Queried Instruments</h3>
 
         <!-- Display the data -->
         <div id="pagination" class="pagination"></div>
+        <hr>
         <div id="thumbnail-array"></div>
         <script>update_thumbnails_query_page('{{base_url}}', '{{ page }}');</script>
         <a id="no_thumbnails_msg" style='display: none'>No data match the selected criteria.</a>

From e165c1ca93b1767943ed059bb487c7f56ca3571a Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 6 Jun 2023 14:04:26 -0400
Subject: [PATCH 138/256] Add summary text for top-level query params

---
 jwql/utils/constants.py                          |  4 ++--
 jwql/website/apps/jwql/data_containers.py        |  6 +++---
 jwql/website/apps/jwql/forms.py                  |  2 +-
 jwql/website/apps/jwql/static/js/jwql.js         | 11 +++++++++--
 jwql/website/apps/jwql/templates/jwql_query.html |  4 ++--
 jwql/website/apps/jwql/views.py                  | 12 +++++++++++-
 6 files changed, 28 insertions(+), 11 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index ced832ddf..9e26a155a 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -436,7 +436,7 @@
 class QUERY_CONFIG_KEYS:
     ANOMALIES = "ANOMALIES"
     APERTURES = "APERTURES"
-    CAT_TYPE = "CAT_TYPE"
+    PROPOSAL_CATEGORY = "PROPOSAL_CATEGORY"
     DETECTORS = "DETECTORS"
     EXP_TYPES = "EXP_TYPES"
     FILTERS = "FILTERS"
@@ -455,7 +455,7 @@ class QUERY_CONFIG_KEYS:
 QUERY_CONFIG_TEMPLATE = {
     QUERY_CONFIG_KEYS.ANOMALIES: {},
     QUERY_CONFIG_KEYS.APERTURES: {},
-    QUERY_CONFIG_KEYS.CAT_TYPE: [],
+    QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY: [],
     QUERY_CONFIG_KEYS.DETECTORS: {},
     QUERY_CONFIG_KEYS.EXP_TYPES: {},
     QUERY_CONFIG_KEYS.FILTERS: {},
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index a3a741847..c3828b11a 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1513,9 +1513,9 @@ def get_rootnames_from_query(parameters):
         if len(look_status) == 1:
             viewed = (look_status[0] == 'VIEWED')
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(viewed=viewed)
-        cat_type = parameters[QUERY_CONFIG_KEYS.CAT_TYPE]
-        if len(cat_type) > 0:
-            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(obsnum__proposal__category__in=cat_type)
+        proposal_category = parameters[QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY]
+        if len(proposal_category) > 0:
+            current_ins_rootfileinfos = current_ins_rootfileinfos.filter(obsnum__proposal__category__in=proposal_category)
 
         # Instrument fields
         inst_anomalies = parameters[QUERY_CONFIG_KEYS.ANOMALIES][inst]
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index 1163dafc5..b0554b451 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -145,7 +145,7 @@ class JwqlQueryForm(BaseForm):
         required=False, choices=look_choices, widget=forms.CheckboxSelectMultiple)
 
     cat_choices = [(query_format(choice), query_format(choice)) for choice in PROPOSAL_CATEGORIES]
-    cat_type = forms.MultipleChoiceField(
+    proposal_category = forms.MultipleChoiceField(
         required=False, choices=cat_choices, widget=forms.CheckboxSelectMultiple)
 
     sort_choices = [(choice, choice) for choice in SORT_OPTIONS]
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index d5e63a393..173aae822 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1244,7 +1244,14 @@ function update_pagination(data) {
     $("#pagination")[0].innerHTML = content;
 
     // Add the total file count to the img_show_count banner
-    $("#query_total")[0].innerHTML = '<p>Query returned ' + data.total_files + ' activities total.</p>';
+    var query_summary = '<p><br>Query returned ' + data.total_files + ' activities total with parameters:</p><ul>';
+    for (var i = 0; i < Object.keys(data.query_config).length; i++) {
+        var param = Object.keys(data.query_config)[i];
+        query_summary += '<li>' + param.toUpperCase() + '=' +
+            data.query_config[param].toString().toUpperCase() + '</li>';
+    }
+    query_summary += '</p>';
+    $("#query_summary")[0].innerHTML = query_summary;
 }
 
 /**
@@ -1256,7 +1263,7 @@ function update_show_count(count, type) {
     var content = 'Showing <a id="img_shown">' + count + '</a> / <a id="img_total">' + count + '</a> <a id="img_type">' + type + '</a>';
     content += '<a href="https://jwst-pipeline.readthedocs.io/en/latest/jwst/data_products/science_products.html" target="_blank" style="color: black">';
     content += '<span class="help-tip mx-2">i</span></a>';
-    content += '<span id="query_total"></span>';
+    content += '<span id="query_summary"></span>';
     $("#img_show_count")[0].innerHTML = content;
 }
 
diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index 2cd418947..4cd308acd 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -87,8 +87,8 @@ <h1>Dynamic Query Form</h1>
                         </div>
                         <div class="col-md-2">
                             <span class="help-block">Proposal Category</span>
-                            <div class="form-control" id='cat_type'>
-                                {% for option in form.cat_type %}
+                            <div class="form-control" id='proposal_category'>
+                                {% for option in form.proposal_category %}
                                 <div class='radio'>
                                     {{ option }}
                                 </div>
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index b874ed89d..d073ec766 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -126,7 +126,7 @@ def jwql_query(request):
             parameters = QUERY_CONFIG_TEMPLATE.copy()
             parameters[QUERY_CONFIG_KEYS.INSTRUMENTS] = form.cleaned_data['instrument']
             parameters[QUERY_CONFIG_KEYS.LOOK_STATUS] = form.cleaned_data['look_status']
-            parameters[QUERY_CONFIG_KEYS.CAT_TYPE] = form.cleaned_data['cat_type']
+            parameters[QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY] = form.cleaned_data['proposal_category']
             parameters[QUERY_CONFIG_KEYS.SORT_TYPE] = form.cleaned_data['sort_type']
             parameters[QUERY_CONFIG_KEYS.ANOMALIES] = all_anomalies
             parameters[QUERY_CONFIG_KEYS.APERTURES] = all_apers
@@ -489,6 +489,16 @@ def archive_thumbnails_query_ajax(request):
     data['thumbnail_sort'] = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
     data['thumbnail_group'] = request.session.get("image_group", "Exposure")
 
+    # add top level parameters for summarizing
+    summary_keys = [QUERY_CONFIG_KEYS.INSTRUMENTS,
+                    QUERY_CONFIG_KEYS.SORT_TYPE, QUERY_CONFIG_KEYS.LOOK_STATUS,
+                    QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY]
+    data['query_config'] = {}
+    for key in summary_keys:
+        value = parameters[key]
+        if value:
+            data['query_config'][key] = value
+
     # pass pagination info
     if page_obj.has_previous():
         data['previous_page'] = page_obj.previous_page_number()

From f181f0a543c31a830b137f3c7a0676c3e13a8f17 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 6 Jun 2023 15:48:48 -0400
Subject: [PATCH 139/256] Fix for bad test PID

---
 jwql/tests/test_protect_module.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/tests/test_protect_module.py b/jwql/tests/test_protect_module.py
index d0da6ab79..3bcafa5c1 100644
--- a/jwql/tests/test_protect_module.py
+++ b/jwql/tests/test_protect_module.py
@@ -97,7 +97,7 @@ def test_lock_module_handles_stale_lock_files(module_lock, do_not_email):
        delete it, and continue to run successfully """
     # create locked file with not running PID in advance of calling protected code
     with open(module_lock, "w") as lock_file:
-        lock_file.write(f"{_PID_LOCKFILE_KEY}12345\n")
+        lock_file.write(f"{_PID_LOCKFILE_KEY}-9999\n")
     file_created = protected_code_verify_file_exists_true(module_lock)
     file_exists = os.path.exists(module_lock)
     # Assert that lock file was created in wrapper, and removed upon exit

From cd0c93a9e36923df4f37b7813237584c31701af5 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 6 Jun 2023 15:55:18 -0400
Subject: [PATCH 140/256] Fix argument to update_sort_options

---
 jwql/website/apps/jwql/static/js/jwql.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 173aae822..73f33eae5 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1468,7 +1468,7 @@ function update_thumbnails_query_page(base_url, page) {
             update_thumbnail_array(data);
             update_filter_options(data, base_url, 'thumbnail');
             update_group_options(data, base_url);
-            update_sort_options(data.thumbnail_sort, base_url);
+            update_sort_options(data, base_url);
             update_pagination(data);
 
             // Do initial sort and group to match sort button display

From a9229ca940f9c9836a2c870b0144c3ffe71cda5b Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Wed, 7 Jun 2023 10:23:49 -0400
Subject: [PATCH 141/256] Add all parameters to query summary

---
 jwql/utils/constants.py         | 27 ++++++++++++++-------------
 jwql/website/apps/jwql/views.py | 12 +++++++-----
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 9e26a155a..dee4123d2 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -433,41 +433,42 @@
 
 
 # Keep keys defined via class as they are used many places with potential mispellings
+# Keys are in sort order from general to instrument specific, then alphabetical
+# within instrument specific fields.
 class QUERY_CONFIG_KEYS:
+    INSTRUMENTS = "INSTRUMENTS"
+    PROPOSAL_CATEGORY = "PROPOSAL_CATEGORY"
+    LOOK_STATUS = "LOOK_STATUS"
+    NUM_PER_PAGE = "NUM_PER_PAGE"
+    SORT_TYPE = "SORT_TYPE"
     ANOMALIES = "ANOMALIES"
     APERTURES = "APERTURES"
-    PROPOSAL_CATEGORY = "PROPOSAL_CATEGORY"
     DETECTORS = "DETECTORS"
     EXP_TYPES = "EXP_TYPES"
     FILTERS = "FILTERS"
     GRATINGS = "GRATINGS"
-    LOOK_STATUS = "LOOK_STATUS"
-    INSTRUMENTS = "INSTRUMENTS"
-    NUM_PER_PAGE = "NUM_PER_PAGE"
     PUPILS = "PUPILS"
     READ_PATTS = "READ_PATTS"
-    SORT_TYPE = "SORT_TYPE"
     SUBARRAYS = "SUBARRAYS"
-    THUMBNAILS = "THUMBNAILS"
 
 
 # Template for parameters to be stored in "query_config" session for query_page
 QUERY_CONFIG_TEMPLATE = {
+    QUERY_CONFIG_KEYS.INSTRUMENTS: [],
+    QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY: [],
+    QUERY_CONFIG_KEYS.LOOK_STATUS: [],
+    QUERY_CONFIG_KEYS.NUM_PER_PAGE: 100,
+    QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent',
     QUERY_CONFIG_KEYS.ANOMALIES: {},
     QUERY_CONFIG_KEYS.APERTURES: {},
-    QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY: [],
     QUERY_CONFIG_KEYS.DETECTORS: {},
     QUERY_CONFIG_KEYS.EXP_TYPES: {},
     QUERY_CONFIG_KEYS.FILTERS: {},
     QUERY_CONFIG_KEYS.GRATINGS: {},
-    QUERY_CONFIG_KEYS.INSTRUMENTS: [],
-    QUERY_CONFIG_KEYS.NUM_PER_PAGE: 100,
     QUERY_CONFIG_KEYS.PUPILS: {},
     QUERY_CONFIG_KEYS.READ_PATTS: {},
-    QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent',
-    QUERY_CONFIG_KEYS.SUBARRAYS: {},
-    QUERY_CONFIG_KEYS.THUMBNAILS: []
-    }
+    QUERY_CONFIG_KEYS.SUBARRAYS: {}
+}
 
 # RAPID-style readout patterns for each instrument. Added so we can
 # differentiate in MAST searches for e.g. the dark current monitor
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index d073ec766..78b09cdba 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -490,13 +490,15 @@ def archive_thumbnails_query_ajax(request):
     data['thumbnail_group'] = request.session.get("image_group", "Exposure")
 
     # add top level parameters for summarizing
-    summary_keys = [QUERY_CONFIG_KEYS.INSTRUMENTS,
-                    QUERY_CONFIG_KEYS.SORT_TYPE, QUERY_CONFIG_KEYS.LOOK_STATUS,
-                    QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY]
     data['query_config'] = {}
-    for key in summary_keys:
+    for key in parameters:
         value = parameters[key]
-        if value:
+        if isinstance(value, dict):
+            for subkey in value:
+                subvalue = value[subkey]
+                if subvalue:
+                    data['query_config'][f'{key}_{subkey}'] = subvalue
+        elif value:
             data['query_config'][key] = value
 
     # pass pagination info

From b2e0d80805d6dc39948ccd3e1244d0dc37f23580 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Wed, 7 Jun 2023 16:07:13 -0400
Subject: [PATCH 142/256] Fix aspect ratio, default to linear zscale

---
 jwql/utils/interactive_preview_image.py       | 44 ++++++++++++++++---
 .../apps/jwql/templates/explore_image.html    |  8 ++--
 jwql/website/apps/jwql/views.py               |  4 +-
 3 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index 01c99e808..6ea0c1e9e 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -23,7 +23,9 @@
 from astropy.visualization import ZScaleInterval
 import numpy as np
 from bokeh.embed import components
-from bokeh.models import BasicTicker, ColorBar, HoverTool, LinearColorMapper, LogColorMapper, LogTicker
+from bokeh.models import (
+    BasicTicker, BoxZoomTool, ColorBar, HoverTool, LinearColorMapper, LogColorMapper,
+    LogTicker, WheelZoomTool)
 from bokeh.plotting import figure, output_file, show, save
 
 from jwst.datamodels import dqflags
@@ -80,9 +82,9 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contras
         self.save_html = save_html
 
         # Allow sending in of None without overriding defaults
-        if(group is None):
+        if group is None:
             group = -1
-        if(integ is None):
+        if integ is None:
             integ = 0
 
         # Determine the min and max values to use for the display
@@ -107,13 +109,13 @@ def create_bokeh_image(self):
         """Method to create the figure
         """
         limits = self.get_scale()
-        if limits[0] <= 0:
-            limits = (1e-4, limits[1])
         if self.low_lim is not None:
             limits = (self.low_lim, limits[1])
         if self.high_lim is not None:
             limits = (limits[0], self.high_lim)
         if self.scaling == 'log':
+            if limits[0] <= 0:
+                limits = (1e-4, limits[1])
             color_mapper = LogColorMapper(
                 palette="Viridis256", low=limits[0], high=limits[1])
             ticker = LogTicker()
@@ -128,13 +130,41 @@ def create_bokeh_image(self):
         if not self.show and self.save_html is not None:
             output_file(filename=self.save_html,
                         title=os.path.basename(self.filename))
-        fig = figure(tools='pan,box_zoom,reset,wheel_zoom,save')
+
+        # fix figure aspect from data aspect
+        if xd > yd:
+            plot_width = 800
+            plot_height = int(plot_width * yd / xd)
+            if plot_height < 400:
+                plot_height = 400
+        else:
+            plot_height = 800
+            plot_width = int(plot_height * xd / yd)
+            if plot_width < 400:
+                plot_width = 400
+
+        fig = figure(tools='pan,reset,save', match_aspect=True,
+                     plot_width=plot_width, plot_height=plot_height)
+        fig.add_tools(BoxZoomTool(match_aspect=True))
+        fig.add_tools(WheelZoomTool(zoom_on_axis=False))
+
         img = fig.image(source=info, image='image',
                         level="image", color_mapper=color_mapper)
-        color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker, title=self.signal_units, bar_line_color='black',
+        color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker,
+                             title=self.signal_units, bar_line_color='black',
                              minor_tick_line_color='black', major_tick_line_color='black')
         fig.add_layout(color_bar, 'below')
+
         fig.x_range.range_padding = fig.y_range.range_padding = 0
+        if xd >= yd:
+            fig.x_range.start = 0
+            fig.x_range.end = xd
+            fig.x_range.bounds = (0, xd)
+        if yd >= xd:
+            fig.y_range.start = 0
+            fig.y_range.end = yd
+            fig.y_range.bounds = (0, yd)
+
         if 'DQ' not in self.extname:
             hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"),
                                              ('Value', '@image{0.4f}')
diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html
index 205bffc2b..cced80f37 100644
--- a/jwql/website/apps/jwql/templates/explore_image.html
+++ b/jwql/website/apps/jwql/templates/explore_image.html
@@ -21,10 +21,10 @@ <h3>{{ file_root }}_{{ filetype }}.fits</h3>
         <h4>Image Settings</h4>
         <div class="mt-4">
             Scaling:&nbsp;&nbsp;
-            <input type="radio" name="scaling" value="log" id="log" checked="checked"> log &nbsp;&nbsp;&nbsp;&nbsp;
-            <input type="radio" name="scaling" value="lin" id="lin"> linear &nbsp;&nbsp;
+            <input type="radio" name="scaling" value="log" id="log"> log &nbsp;&nbsp;&nbsp;&nbsp;
+            <input type="radio" name="scaling" value="lin" id="lin" checked="checked"> linear &nbsp;&nbsp;
             <br>
-            Z-Scale Limits:&nbsp;&nbsp;
+            Scale Limits:&nbsp;&nbsp;
             <input type="number" id="low_lim" size="15" step="any">&nbsp;Low &nbsp;&nbsp;&nbsp;&nbsp;
             <input type="number" id="high_lim" size="15" step="any">&nbsp;High &nbsp;&nbsp;&nbsp;&nbsp;
             <br>
@@ -129,4 +129,4 @@ <h5>Submit Anomaly</h5>
     <script>explore_image_update_enable_options('{{ extension_ints }}', '{{ extension_groups }}');</script>
     <script>update_explore_image_page('{{inst}}', '{{file_root}}', '{{filetype}}', '{{base_url}}');</script>
 
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 8b4712a90..b13599f00 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -892,7 +892,7 @@ def view_header(request, inst, filename, filetype):
 
 
 def explore_image(request, inst, file_root, filetype):
-    """Generate the header view page
+    """Generate the explore image page.
 
     Parameters
     ----------
@@ -958,7 +958,7 @@ def explore_image(request, inst, file_root, filetype):
     return render(request, template, context)
 
 
-def explore_image_ajax(request, inst, file_root, filetype, scaling="log", low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None):
+def explore_image_ajax(request, inst, file_root, filetype, scaling="lin", low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None):
     """Generate the page listing all archived images in the database
     for a certain proposal
 

From 90b04d5e7ef340a66b13da3ef3f079b62ab2bd4e Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Thu, 8 Jun 2023 16:11:39 -0400
Subject: [PATCH 143/256] Move scaling and limit controls to client side

---
 .../ta_monitors/msata_monitor.py              | 322 +++++++++++++-----
 .../ta_monitors/wata_monitor.py               | 232 +++++++++----
 jwql/utils/interactive_preview_image.py       | 176 ++++++++--
 jwql/website/apps/jwql/static/css/jwql.css    |  19 +-
 jwql/website/apps/jwql/static/js/jwql.js      |   5 +-
 .../apps/jwql/templates/explore_image.html    |  93 +++--
 jwql/website/apps/jwql/urls.py                |   2 +-
 7 files changed, 607 insertions(+), 242 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index bc21a4643..55abcf95e 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -24,6 +24,7 @@
 Author
 ______
     - Maria Pena-Guerrero
+    - Melanie Clarke
 
 Use
 ---
@@ -38,6 +39,7 @@
 import os
 import logging
 import shutil
+from datetime import datetime, timezone, timedelta
 from random import randint
 
 import numpy as np
@@ -45,12 +47,13 @@
 from astropy.time import Time
 from astropy.io import fits
 from bokeh.embed import components
-from bokeh.plotting import figure, output_file, save
-from bokeh.layouts import gridplot
-from bokeh.models import ColumnDataSource, Range1d, Span, Label
-from bokeh.models.tools import HoverTool
+from bokeh.layouts import gridplot, layout
+from bokeh.models import (
+    ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView,
+    Span, Label, DateRangeSlider)
+from bokeh.models.tools import HoverTool, BoxSelectTool
+from bokeh.plotting import figure, save, output_file
 from bs4 import BeautifulSoup
-from datetime import datetime
 from sqlalchemy.sql.expression import and_
 
 # jwql imports
@@ -59,11 +62,11 @@
 from jwql.utils import monitor_utils
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.logging_functions import log_info, log_fail
-from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
 
 class MSATA():
-    """ Class for executint the NIRSpec MSATA monitor.
+    """ Class for executing the NIRSpec MSATA monitor.
 
     This class will search for new MSATA current files in the file systems
     for NIRSpec and will run the monitor on these files. The monitor will
@@ -136,6 +139,13 @@ def __init__(self):
                                'planned_v3': {'loc': 'ta_table', 'alt_key': None, 'name': 'planned_v3', 'type': float},
                                'FITTARGS': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'stars_in_fit', 'type': int}}
 
+        # initialize attributes to be set later
+        self.source = None
+        self.share_tools = []
+        self.date_range = None
+        self.date_filter = None
+        self.date_view = None
+
     def get_tainfo_from_fits(self, fits_file):
         """ Get the TA information from the fits file
         Parameters
@@ -240,6 +250,19 @@ def get_msata_data(self, new_filenames):
         msata_df = pd.DataFrame(msata_dict)
         return msata_df, no_ta_ext_msgs
 
+    def add_time_column(self):
+        """Add time column to data source, to be used by all plots."""
+        date_obs = self.source.data['date_obs']
+        if 'time_arr' not in self.source.data:
+            time_arr = []
+            for do_str in date_obs:
+                # convert time string into an array of time (this is in UT)
+                t = datetime.fromisoformat(do_str)
+                time_arr.append(t)
+
+            # add to the bokeh data structure
+            self.source.data["time_arr"] = time_arr
+
     def plt_status(self):
         """ Plot the MSATA status versus time.
         Parameters
@@ -249,18 +272,12 @@ def plt_status(self):
         -------
             plot: bokeh plot object
         """
-        # ta_status, date_obs = source.data['ta_status'], source.data['date_obs']
-        date_obs = self.source.data['date_obs']
         ta_status = self.source.data['ta_status']
         # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            time_arr = self.source.data['time_arr']
-            bool_status = self.source.data['bool_status']
-            status_colors = self.source.data['status_colors']
-        except KeyError:
-            # bokeh does not like to plot strings, turn  into numbers
-            number_status, time_arr, status_colors = [], [], []
-            for tas, do_str in zip(ta_status, date_obs):
+        if 'bool_status' not in self.source.data:
+            # bokeh does not like to plot strings, turn into numbers
+            number_status, status_colors = [], []
+            for tas in ta_status:
                 if tas.lower() == 'unsuccessful':
                     number_status.append(0.0)
                     status_colors.append('red')
@@ -270,19 +287,19 @@ def plt_status(self):
                 else:
                     number_status.append(1.0)
                     status_colors.append('blue')
-                # convert time string into an array of time (this is in UT with 0.0 milliseconds)
-                t = datetime.fromisoformat(do_str)
-                time_arr.append(t)
+
             # add these to the bokeh data structure
-            self.source.data["time_arr"] = time_arr
             self.source.data["number_status"] = number_status
             self.source.data["status_colors"] = status_colors
+
         # create a new bokeh plot
         plot = figure(title="MSATA Status [Success=1, In Progress=0.5, Fail=0]", x_axis_label='Time',
                       y_axis_label='MSATA Status', x_axis_type='datetime',)
         plot.y_range = Range1d(-0.5, 1.5)
         plot.circle(x='time_arr', y='number_status', source=self.source,
-                    color='status_colors', size=7, fill_alpha=0.3)
+                    color='status_colors', size=7, fill_alpha=0.3, view=self.date_view)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -293,8 +310,11 @@ def plt_status(self):
                           ('Date-Obs', '@date_obs'),
                           ('Subarray', '@subarray'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_residual_offsets(self):
@@ -311,7 +331,8 @@ def plt_residual_offsets(self):
                       x_axis_label='Least Squares Residual V2 Offset',
                       y_axis_label='Least Squares Residual V3 Offset')
         plot.circle(x='lsv2offset', y='lsv3offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+
         v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet']
         xstart, ystart, ray_length = -1 * v2halffacet[0], -1 * v3halffacet[0], 0.05
         plot.ray(x=xstart - ray_length / 2.0, y=ystart, length=ray_length, angle_units="deg",
@@ -322,10 +343,13 @@ def plt_residual_offsets(self):
         plot.add_layout(hflabel)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -338,8 +362,11 @@ def plt_residual_offsets(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v2offset_time(self):
@@ -355,8 +382,9 @@ def plt_v2offset_time(self):
         plot = figure(title="MSATA Least Squares V2 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V2 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv2offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         time_arr, v2halffacet = self.source.data['time_arr'], self.source.data['v2halffacet']
@@ -364,6 +392,8 @@ def plt_v2offset_time(self):
         plot.renderers.extend([hline, hfline])
         hflabel = Label(x=time_arr[-1], y=-1 * v2halffacet[0], y_units='data', text='-V2 half-facet value')
         plot.add_layout(hflabel)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -376,8 +406,11 @@ def plt_v2offset_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v3offset_time(self):
@@ -393,8 +426,9 @@ def plt_v3offset_time(self):
         plot = figure(title="MSATA Least Squares V3 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V3 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv3offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         time_arr, v3halffacet = self.source.data['time_arr'], self.source.data['v3halffacet']
@@ -402,6 +436,8 @@ def plt_v3offset_time(self):
         plot.renderers.extend([hline, hfline])
         hflabel = Label(x=time_arr[-1], y=-1 * v3halffacet[0], y_units='data', text='-V3 half-facet value')
         plot.add_layout(hflabel)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -414,8 +450,11 @@ def plt_v3offset_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_lsv2v3offsetsigma(self):
@@ -432,13 +471,16 @@ def plt_lsv2v3offsetsigma(self):
                       x_axis_label='Least Squares Residual V2 Sigma Offset',
                       y_axis_label='Least Squares Residual V3 Sigma Offset')
         plot.circle(x='lsv2sigma', y='lsv3sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.x_range = Range1d(-0.1, 0.1)
         plot.y_range = Range1d(-0.1, 0.1)
+
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -453,8 +495,11 @@ def plt_lsv2v3offsetsigma(self):
                           ('LS V3 offset', '@lsv3offset'),
                           ('LS V3 sigma', '@lsv3sigma'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_res_offsets_corrected(self):
@@ -466,29 +511,30 @@ def plt_res_offsets_corrected(self):
         -------
             plot: bokeh plot object
         """
-        # create a new bokeh plot
         lsv2offset, lsv3offset = self.source.data['lsv2offset'], self.source.data['lsv3offset']
         v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet']
-        # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            v2_half_fac_corr = self.source.data['v2_half_fac_corr']
-            v3_half_fac_corr = self.source.data['v3_half_fac_corr']
-        except KeyError:
+
+        # check if this column exists in the data already, else create it
+        if 'v2_half_fac_corr' not in self.source.data:
             v2_half_fac_corr, v3_half_fac_corr = [], []
             for idx, v2hf in enumerate(v2halffacet):
                 v3hf = v3halffacet[idx]
                 v2_half_fac_corr.append(lsv2offset[idx] + v2hf)
                 v3_half_fac_corr.append(lsv3offset[idx] + v3hf)
+
             # add these to the bokeh data structure
             self.source.data["v2_half_fac_corr"] = v2_half_fac_corr
             self.source.data["v3_half_fac_corr"] = v3_half_fac_corr
+
+        # create a new bokeh plot
         plot = figure(title="MSATA Least Squares Residual V2-V3 Offsets Half-facet corrected",
                       x_axis_label='Least Squares Residual V2 Offset + half-facet',
                       y_axis_label='Least Squares Residual V3 Offset + half-facet')
         plot.circle(x='v2_half_fac_corr', y='v3_half_fac_corr', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
@@ -500,6 +546,8 @@ def plt_res_offsets_corrected(self):
                  angle=90, line_color='purple', line_width=3)
         hflabel = Label(x=xstart / 3.0, y=ystart, y_units='data', text='-V2, -V3 half-facets values')
         plot.add_layout(hflabel)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -514,15 +562,16 @@ def plt_res_offsets_corrected(self):
                           ('V2 half-facet', '@v2halffacet'),
                           ('V3 half-facet', '@v3halffacet'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v2offsigma_time(self):
-        """ Plot the residual Least Squares V2 sigma Offset versus time
-        Parameters
-        ----------
-            None
+        """Plot the residual Least Squares V2 sigma Offset versus time
+
         Returns
         -------
             plot: bokeh plot object
@@ -531,11 +580,14 @@ def plt_v2offsigma_time(self):
         plot = figure(title="MSATA Least Squares V2 Sigma Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V2 Sigma Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv2sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.1, 0.1)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -547,28 +599,34 @@ def plt_v2offsigma_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V2 sigma', '@lsv2sigma'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v3offsigma_time(self):
-        """ Plot the residual Least Squares V3 Offset versus time
-        Parameters
-        ----------
-            None
+        """Plot the residual Least Squares V3 Offset versus time
+
         Returns
         -------
             p: bokeh plot object
         """
         # create a new bokeh plot
-        plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time", x_axis_label='Time',
-                      y_axis_label='Least Squares Residual V3 Sigma Offset', x_axis_type='datetime')
+        plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time",
+                      x_axis_label='Time',
+                      y_axis_label='Least Squares Residual V3 Sigma Offset',
+                      x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv3sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.1, 0.1)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -581,8 +639,11 @@ def plt_v3offsigma_time(self):
                           ('LS V3 offset', '@lsv3offset'),
                           ('LS V3 sigma', '@lsv3sigma'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_roll_offset(self):
@@ -598,10 +659,12 @@ def plt_roll_offset(self):
         plot = figure(title="MSATA Least Squares Roll Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual Roll Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsrolloffset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-600.0, 600.0)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
+
         # Maximum accepted roll line and label
         time_arr = self.source.data['time_arr']
         arlinepos = Span(location=120, dimension='width', line_color='green', line_width=3)
@@ -609,6 +672,8 @@ def plt_roll_offset(self):
         arlabel = Label(x=time_arr[-1], y=125, y_units='data', text='Max accepted roll')
         plot.add_layout(arlabel)
         plot.renderers.extend([hline, arlinepos, arlineneg])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -621,8 +686,11 @@ def plt_roll_offset(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_lsoffsetmag(self):
@@ -638,11 +706,14 @@ def plt_lsoffsetmag(self):
         plot = figure(title="MSATA Least Squares Total Magnitude of the Linear V2, V3 Offset Slew vs Time", x_axis_label='Time',
                       y_axis_label='sqrt((V2_off)**2 + (V3_off)**2)', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsoffsetmag', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -656,8 +727,11 @@ def plt_lsoffsetmag(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_tot_number_of_stars(self):
@@ -672,34 +746,34 @@ def plt_tot_number_of_stars(self):
         # get the number of stars per array
         visit_id = self.source.data['visit_id']
         reference_star_number = self.source.data['reference_star_number']
-        stars_in_fit = self.source.data['stars_in_fit']
-        date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr']
-        # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            tot_number_of_stars = self.source.data['tot_number_of_stars']
-            colors_list = self.source.data['colors_list']
-        except KeyError:
+
+        # check if this column exists in the data already, else create it
+        if 'tot_number_of_stars' not in self.source.data:
             # create the list of color per visit and tot_number_of_stars
             colors_list, tot_number_of_stars = [], []
             color_dict = {}
-            for i, _ in enumerate(visit_id):
+            for i, vid in enumerate(visit_id):
                 tot_stars = len(reference_star_number[i])
                 tot_number_of_stars.append(tot_stars)
                 ci = '#%06X' % randint(0, 0xFFFFFF)
-                if visit_id[i] not in color_dict:
-                    color_dict[visit_id[i]] = ci
-                colors_list.append(color_dict[visit_id[i]])
+                if vid not in color_dict:
+                    color_dict[vid] = ci
+                colors_list.append(color_dict[vid])
+
             # add these to the bokeh data structure
             self.source.data["tot_number_of_stars"] = tot_number_of_stars
             self.source.data["colors_list"] = colors_list
+
         # create a new bokeh plot
         plot = figure(title="Total Number of Measurements vs Time", x_axis_label='Time',
                       y_axis_label='Total number of measurements', x_axis_type='datetime')
         plot.circle(x='time_arr', y='tot_number_of_stars', source=self.source,
-                    color='colors_list', size=7, fill_alpha=0.3)
+                    color='colors_list', size=7, fill_alpha=0.3, view=self.date_view)
         plot.triangle(x='time_arr', y='stars_in_fit', source=self.source,
-                      color='black', size=7, fill_alpha=0.3)
+                      color='black', size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(0.0, 40.0)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -714,8 +788,8 @@ def plt_tot_number_of_stars(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
         return plot
 
     def plt_mags_time(self):
@@ -735,22 +809,24 @@ def plt_mags_time(self):
         planned_v2 = self.source.data['planned_v2']
         planned_v3 = self.source.data['planned_v3']
         reference_star_number = self.source.data['reference_star_number']
-        visits_stars_mags = self.source.data['reference_star_mag']
         box_peak_value = self.source.data['box_peak_value']
         date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr']
         colors_list = self.source.data['colors_list']
         detector_list = self.source.data['detector']
+        filename = self.source.data['filename']
+
         # create the structure matching the number of visits and reference stars
         new_colors_list, vid, dobs, tarr, star_no, status = [], [], [], [], [], []
-        peaks, stars_v2, stars_v3, det = [], [], [], []
+        peaks, stars_v2, stars_v3, det, fnames = [], [], [], [], []
         for i, _ in enumerate(visit_id):
-            v, d, t, c, s, x, y, dt = [], [], [], [], [], [], [], []
+            v, d, t, c, s, x, y, dt, fn = [], [], [], [], [], [], [], [], []
             for j in range(len(reference_star_number[i])):
                 v.append(visit_id[i])
                 d.append(date_obs[i])
                 t.append(time_arr[i])
                 c.append(colors_list[i])
                 dt.append(detector_list[i])
+                fn.append(filename[i])
                 if 'not_removed' in lsf_removed_status[i][j]:
                     s.append('SUCCESS')
                     x.append(planned_v2[i][j])
@@ -769,17 +845,29 @@ def plt_mags_time(self):
             stars_v3.extend(y)
             peaks.extend(box_peak_value[i])
             det.extend(dt)
+            fnames.extend(fn)
+
         # now create the mini ColumnDataSource for this particular plot
         mini_source = {'vid': vid, 'star_no': star_no, 'status': status,
-                       'dobs': dobs, 'tarr': tarr, 'det': det,
+                       'dobs': dobs, 'time_arr': tarr, 'det': det, 'fname': fnames,
                        'peaks': peaks, 'colors_list': new_colors_list,
                        'stars_v2': stars_v2, 'stars_v3': stars_v3}
         mini_source = ColumnDataSource(data=mini_source)
-        # create a the bokeh plot
-        plot = figure(title="MSATA Counts vs Time", x_axis_label='Time', y_axis_label='box_peak [Counts]',
+
+        # hook up the date range slider to this source as well
+        callback = CustomJS(args=dict(s=mini_source), code="""
+            s.change.emit();
+        """)
+        self.date_range.js_on_change('value', callback)
+        mini_view = CDSView(source=mini_source, filters=[self.date_filter])
+
+        # create the bokeh plot
+        plot = figure(title="MSATA Counts vs Time", x_axis_label='Time',
+                      y_axis_label='box_peak [Counts]',
                       x_axis_type='datetime')
-        plot.circle(x='tarr', y='peaks', source=mini_source,
-                    color='colors_list', size=7, fill_alpha=0.3)
+        plot.circle(x='time_arr', y='peaks', source=mini_source,
+                    color='colors_list', size=7, fill_alpha=0.3, view=mini_view)
+
         # add count saturation warning lines
         loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0
         hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3)
@@ -793,9 +881,10 @@ def plt_mags_time(self):
         plot.add_layout(label2)
         plot.add_layout(label3)
         plot.y_range = Range1d(-1000.0, 62000.0)
-        # add hover
+
+        # add tooltips
         hover = HoverTool()
-        hover.tooltips = [('File name', '@filename'),
+        hover.tooltips = [('File name', '@fname'),
                           ('Visit ID', '@vid'),
                           ('Detector', '@det'),
                           ('Star No.', '@star_no'),
@@ -805,18 +894,69 @@ def plt_mags_time(self):
                           ('Measured V2', '@stars_v2'),
                           ('Measured V3', '@stars_v3'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
         return plot
 
+    def setup_date_range(self):
+        """Set up a date range filter, defaulting to the last week of data."""
+        end_date = datetime.now(tz=timezone.utc)
+        one_week_ago = end_date.date() - timedelta(days=7)
+        first_data_point = np.min(self.source.data['time_arr']).date()
+        last_data_point = np.max(self.source.data['time_arr']).date()
+        if last_data_point < one_week_ago:
+            # keep at least one point in the plot if there was
+            # no TA data this week
+            start_date = last_data_point
+        else:
+            start_date = one_week_ago
+
+        # allowed range is from the first ever data point to today
+        self.date_range = DateRangeSlider(
+            title="Date range displayed", start=first_data_point,
+            end=end_date, value=(start_date, end_date), step=1)
+
+        callback = CustomJS(args=dict(s=self.source), code="""
+            s.change.emit();
+        """)
+        self.date_range.js_on_change('value', callback)
+
+        self.date_filter = CustomJSFilter(args=dict(slider=self.date_range), code="""
+                var indices = [];
+                var start = slider.value[0];
+                var end = slider.value[1];
+
+                for (var i=0; i < source.get_length(); i++) {
+                    if (source.data['time_arr'][i] >= start 
+                            && source.data['time_arr'][i] <= end) {
+                        indices.push(true);
+                    } else {
+                        indices.push(false);
+                    }
+                }
+                return indices;
+                """)
+        self.date_view = CDSView(source=self.source, filters=[self.date_filter])
+
     def mk_plt_layout(self):
         """Create the bokeh plot layout"""
         self.source = ColumnDataSource(data=self.msata_data)
+
         # make sure all arrays are lists in order to later be able to read the data
         # from the html file
         for item in self.source.data:
             if not isinstance(self.source.data[item], (str, float, int, list)):
                 self.source.data[item] = self.source.data[item].tolist()
+
+        # add a time array to the data source
+        self.add_time_column()
+
+        # set up selection tools to share
+        self.share_tools = [BoxSelectTool()]
+
+        # set up a date range filter widget
+        self.setup_date_range()
+
         # set the output html file name and create the plot grid
         output_file(self.output_file_name)
         p1 = self.plt_status()
@@ -831,13 +971,15 @@ def mk_plt_layout(self):
         p10 = self.plt_lsoffsetmag()
         p12 = self.plt_tot_number_of_stars()
         p11 = self.plt_mags_time()
+
         # make grid
         grid = gridplot([p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12],
                         ncols=2, merge_tools=False)
-        # show(grid)
-        save(grid)
-        # return the needed components for embeding the results in the MSATA html template
-        script, div = components(grid)
+        box_layout = layout(children=[self.date_range, grid])
+        save(box_layout)
+
+        # return the needed components for embedding the results in the MSATA html template
+        script, div = components(box_layout)
         return script, div
 
     def identify_tables(self):
@@ -884,7 +1026,7 @@ def get_data_from_html(self, html_file):
             File created by the monitor script
         Returns
         -------
-        prev_data_dict: dictionary
+        prev_data_dict: dict
             Dictionary containing all data used in the plots
         """
 
@@ -961,7 +1103,7 @@ def prev_data2expected_format(self, prev_data_dict):
             Date of the latest observation in the previously plotted data
         """
         # remember that the time array created is in milliseconds, removing to get time object
-        time_in_millis = max(prev_data_dict['tarr'])
+        time_in_millis = max(prev_data_dict['time_arr'])
         latest_prev_obs = Time(time_in_millis / 1000., format='unix')
         latest_prev_obs = latest_prev_obs.mjd
         prev_data_expected_cols = {}
@@ -1051,13 +1193,13 @@ def update_ta_success_txtfile(self):
         output_success_ta_txtfile = os.path.join(self.output_dir, "msata_success.txt")
         # check if previous file exsists and read the data from it
         if os.path.isfile(output_success_ta_txtfile):
-            # now rename the the previous file, for backup
+            # now rename the previous file, for backup
             os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_msata_success.txt"))
         # get the new data
         ta_success, ta_inprogress, ta_failure = [], [], []
         filenames, ta_status = self.msata_data.loc[:,'filename'], self.msata_data.loc[:,'ta_status']
         for fname, ta_stat in zip(filenames, ta_status):
-            # select the appriopriate list to append to
+            # select the appropriate list to append to
             if ta_stat == 'SUCCESSFUL':
                 ta_success.append(fname)
             elif ta_stat == 'IN_PROGRESS':
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index 01e52a44c..f248a1d09 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -10,7 +10,7 @@
 
 
 """
-This module contains the code for NIRSpec the Wide Aperture Target
+This module contains the code for the NIRSpec Wide Aperture Target
 Acquisition (WATA) monitor, which monitors the TA offsets.
 
 This monitor displays the comparison of desired versus measured TA.
@@ -20,6 +20,7 @@
 Author
 ______
     - Maria Pena-Guerrero
+    - Melanie Clarke
 
 Use
 ---
@@ -34,21 +35,22 @@
 import os
 import logging
 import shutil
+from datetime import datetime, timezone, timedelta
 
 import numpy as np
 import pandas as pd
-from bs4 import BeautifulSoup
-from datetime import datetime
 from astropy.time import Time
 from astropy.io import fits
-from sqlalchemy.sql.expression import and_
-from bokeh.io import output_file
-from bokeh.plotting import figure, show, save
-from bokeh.models import ColumnDataSource, Range1d
-from bokeh.models.tools import HoverTool
-from bokeh.layouts import gridplot
-from bokeh.models import Span, Label
 from bokeh.embed import components
+from bokeh.io import output_file
+from bokeh.layouts import gridplot, layout
+from bokeh.models import (
+    ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView,
+    Span, Label, DateRangeSlider)
+from bokeh.models.tools import HoverTool, BoxSelectTool
+from bokeh.plotting import figure, save
+from bs4 import BeautifulSoup
+from sqlalchemy.sql.expression import and_
 
 # jwql imports
 from jwql.utils.logging_functions import log_info, log_fail
@@ -60,7 +62,7 @@
 
 
 class WATA():
-    """ Class for executint the NIRSpec WATA monitor.
+    """ Class for executing the NIRSpec WATA monitor.
 
     This class will search for new WATA current files in the file systems
     for NIRSpec and will run the monitor on these files. The monitor will
@@ -130,6 +132,12 @@ def __init__(self):
                                'SAM_X': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_x'},
                                'SAM_Y': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_y'}}
 
+        # initialize attributes to be set later
+        self.source = None
+        self.share_tools = []
+        self.date_range = None
+        self.date_view = None
+
     def get_tainfo_from_fits(self, fits_file):
         """ Get the TA information from the fits file
         Parameters
@@ -200,6 +208,19 @@ def get_wata_data(self, new_filenames):
         wata_df = pd.DataFrame(wata_dict)
         return wata_df, no_ta_ext_msgs
 
+    def add_time_column(self):
+        """Add time column to data source, to be used by all plots."""
+        date_obs = self.source.data['date_obs']
+        if 'time_arr' not in self.source.data:
+            time_arr = []
+            for do_str in date_obs:
+                # convert time string into an array of time (this is in UT)
+                t = datetime.fromisoformat(do_str)
+                time_arr.append(t)
+
+            # add to the bokeh data structure
+            self.source.data["time_arr"] = time_arr
+
     def plt_status(self):
         """ Plot the WATA status (passed = 0 or failed = 1).
         Parameters
@@ -209,36 +230,32 @@ def plt_status(self):
         -------
             plot: bokeh plot object
         """
-        ta_status, date_obs = self.source.data['ta_status'], self.source.data['date_obs']
-        # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            time_arr = self.source.data['time_arr']
-            bool_status = self.source.data['bool_status']
-            status_colors = self.source.data['status_colors']
-        except KeyError:
-            # bokeh does not like to plot strings, turn  into binary type
-            bool_status, time_arr, status_colors = [], [], []
-            for tas, do_str in zip(ta_status, date_obs):
+        ta_status = self.source.data['ta_status']
+
+        # check if this column exists in the data already, else create it
+        if 'bool_status' not in self.source.data:
+            # bokeh does not like to plot strings, turn into binary type
+            bool_status, status_colors = [], []
+            for tas in ta_status:
                 if 'unsuccessful' not in tas.lower():
                     bool_status.append(1)
                     status_colors.append('blue')
                 else:
                     bool_status.append(0)
                     status_colors.append('red')
-                # convert time string into an array of time (this is in UT)
-                t = datetime.fromisoformat(do_str)
-                time_arr.append(t)
+
             # add these to the bokeh data structure
-            self.source.data["time_arr"] = time_arr
             self.source.data["ta_status_bool"] = bool_status
             self.source.data["status_colors"] = status_colors
+
         # create a new bokeh plot
-        plot = figure(title="WATA Status [Succes=1, Fail=0]", x_axis_label='Time',
+        plot = figure(title="WATA Status [Success=1, Fail=0]", x_axis_label='Time',
                       y_axis_label='WATA Status', x_axis_type='datetime',)
         plot.y_range = Range1d(-0.5, 1.5)
         plot.circle(x='time_arr', y='ta_status_bool', source=self.source,
-                    color='status_colors', size=7, fill_alpha=0.3)
-        # output_file("wata_status.html")
+                    color='status_colors', size=7, fill_alpha=0.3, view=self.date_view)
+
+        # make tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -250,6 +267,10 @@ def plt_status(self):
                           ('--------', '----------------')]
 
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_residual_offsets(self):
@@ -265,13 +286,16 @@ def plt_residual_offsets(self):
         plot = figure(title="WATA Residual V2-V3 Offsets", x_axis_label='Residual V2 Offset',
                       y_axis_label='Residual V3 Offset')
         plot.circle(x='v2_offset', y='v3_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -281,8 +305,11 @@ def plt_residual_offsets(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v2offset_time(self):
@@ -298,11 +325,14 @@ def plt_v2offset_time(self):
         plot = figure(title="WATA V2 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Residual V2 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='v2_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -312,8 +342,11 @@ def plt_v2offset_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_v3offset_time(self):
@@ -329,11 +362,14 @@ def plt_v3offset_time(self):
         plot = figure(title="WATA V3 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Residual V3 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='v3_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3)
+                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
         plot.y_range = Range1d(-0.5, 0.5)
+
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -343,8 +379,11 @@ def plt_v3offset_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
     def plt_mag_time(self):
@@ -358,15 +397,9 @@ def plt_mag_time(self):
         """
         # calculate the pseudo magnitudes
         max_val_box, time_arr = self.source.data['max_val_box'], self.source.data['time_arr']
-        # check if this column exists in the data already (the other 2 will exist too), else create it
-        try:
-            nrsrapid_f140x = self.source.data["nrsrapid_f140x"]
-            nrsrapid_f110w = self.source.data["nrsrapid_f110w"]
-            nrsrapid_clear = self.source.data["nrsrapid_clear"]
-            nrsrapidd6_f140x = self.source.data["nrsrapidd6_f140x"]
-            nrsrapidd6_f110w = self.source.data["nrsrapidd6_f110w"]
-            nrsrapidd6_clear = self.source.data["nrsrapidd6_clear"]
-        except KeyError:
+
+        # check if this column exists in the data already, else create it
+        if "nrsrapid_f140x" not in self.source.data:
             # create the arrays per filter and readout pattern
             nrsrapid_f140x, nrsrapid_f110w, nrsrapid_clear = [], [], []
             nrsrapidd6_f140x, nrsrapidd6_f110w, nrsrapidd6_clear = [], [], []
@@ -417,6 +450,7 @@ def plt_mag_time(self):
                         nrsrapidd6_f140x.append(np.NaN)
                         nrsrapidd6_f110w.append(np.NaN)
                         nrsrapidd6_clear.append(val)
+
             # add to the bokeh data structure
             self.source.data["nrsrapid_f140x"] = nrsrapid_f140x
             self.source.data["nrsrapid_f110w"] = nrsrapid_f110w
@@ -424,27 +458,30 @@ def plt_mag_time(self):
             self.source.data["nrsrapidd6_f140x"] = nrsrapidd6_f140x
             self.source.data["nrsrapidd6_f110w"] = nrsrapidd6_f110w
             self.source.data["nrsrapidd6_clear"] = nrsrapidd6_clear
+
         # create a new bokeh plot
         plot = figure(title="WATA Counts vs Time", x_axis_label='Time',
                       y_axis_label='box_peak [Counts]', x_axis_type='datetime')
         plot.circle(x='time_arr', y='nrsrapid_f140x', source=self.source,
-                    color="purple", size=7, fill_alpha=0.4)
+                    color="purple", size=7, fill_alpha=0.4, view=self.date_view)
         plot.circle(x='time_arr', y='nrsrapidd6_f140x', source=self.source,
-                    color="purple", size=12, fill_alpha=0.4)
+                    color="purple", size=12, fill_alpha=0.4, view=self.date_view)
         plot.triangle(x='time_arr', y='nrsrapid_f110w', source=self.source,
-                      color="orange", size=8, fill_alpha=0.4)
+                      color="orange", size=8, fill_alpha=0.4, view=self.date_view)
         plot.triangle(x='time_arr', y='nrsrapidd6_f110w', source=self.source,
-                      color="orange", size=13, fill_alpha=0.4)
+                      color="orange", size=13, fill_alpha=0.4, view=self.date_view)
         plot.square(x='time_arr', y='nrsrapid_clear', source=self.source,
-                    color="gray", size=7, fill_alpha=0.4)
+                    color="gray", size=7, fill_alpha=0.4, view=self.date_view)
         plot.square(x='time_arr', y='nrsrapidd6_clear', source=self.source,
-                    color="gray", size=12, fill_alpha=0.4)
+                    color="gray", size=12, fill_alpha=0.4, view=self.date_view)
+
         # add count saturation warning lines
         loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0
         hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3)
         hline2 = Span(location=loc2, dimension='width', line_color='yellow', line_width=3)
         hline3 = Span(location=loc3, dimension='width', line_color='red', line_width=3)
         plot.renderers.extend([hline1, hline2, hline3])
+
         label1 = Label(x=time_arr[-1], y=loc1, y_units='data', text='45000 counts')
         label2 = Label(x=time_arr[-1], y=loc2, y_units='data', text='50000 counts')
         label3 = Label(x=time_arr[-1], y=loc3, y_units='data', text='60000 counts')
@@ -452,7 +489,8 @@ def plt_mag_time(self):
         plot.add_layout(label2)
         plot.add_layout(label3)
         plot.y_range = Range1d(-1000.0, 62000.0)
-        # add hover
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -462,12 +500,15 @@ def plt_mag_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Box peak', '@max_val_box'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
-    def get_unsucessful_ta(self, arr_name):
-        """ Find unsucessful TAs in this set (to be plotted in red)
+    def get_unsuccessful_ta(self, arr_name):
+        """ Find unsuccessful TAs in this set (to be plotted in red)
         Parameters
         ----------
             arr_name: str, name of the array of interest
@@ -497,16 +538,16 @@ def plt_centroid(self):
             plot: bokeh plot object
         """
         # get the failed TAs to plot in red
-        try:
-            corr_col_failed = self.source.data["corr_col_failed"]
-        except KeyError:
-            corr_col_failed, corr_col_not_failed = self.get_unsucessful_ta('corr_col')
-            corr_row_failed, corr_row_not_failed = self.get_unsucessful_ta('corr_row')
+        if "corr_col_failed" not in self.source.data:
+            corr_col_failed, corr_col_not_failed = self.get_unsuccessful_ta('corr_col')
+            corr_row_failed, corr_row_not_failed = self.get_unsuccessful_ta('corr_row')
+
             # add these to the bokeh data structure
             self.source.data["corr_col_failed"] = corr_col_failed
             self.source.data["corr_col_not_failed"] = corr_col_not_failed
             self.source.data["corr_row_failed"] = corr_row_failed
             self.source.data["corr_row_not_failed"] = corr_row_not_failed
+
         # create a new bokeh plot
         plot = figure(title="WATA Centroid", x_axis_label='Column',
                       y_axis_label='Row')
@@ -514,11 +555,13 @@ def plt_centroid(self):
         plot.x_range = Range1d(limits[0], limits[1])
         plot.y_range = Range1d(limits[0], limits[1])
         plot.circle(x='corr_col_not_failed', y='corr_row_not_failed', source=self.source,
-                    color="blue", size=7, fill_alpha=0.5)
+                    color="blue", size=7, fill_alpha=0.5, view=self.date_view)
         plot.circle(x='corr_col_failed', y='corr_row_failed', source=self.source,
-                    color="red", size=7, fill_alpha=0.5)
+                    color="red", size=7, fill_alpha=0.5, view=self.date_view)
         plot.x_range = Range1d(0.0, 32.0)
         plot.y_range = Range1d(0.0, 32.0)
+
+        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -532,18 +575,72 @@ def plt_centroid(self):
                           ('Det Centr Col', '@detector_final_col'),
                           ('Det Centr Row', '@detector_final_row'),
                           ('--------', '----------------')]
-
         plot.add_tools(hover)
+
+        # add shared selection tools
+        for tool in self.share_tools:
+            plot.add_tools(tool)
         return plot
 
+    def setup_date_range(self):
+        """Set up a date range filter, defaulting to the last week of data."""
+        end_date = datetime.now(tz=timezone.utc)
+        one_week_ago = end_date.date() - timedelta(days=7)
+        first_data_point = np.min(self.source.data['time_arr']).date()
+        last_data_point = np.max(self.source.data['time_arr']).date()
+        if last_data_point < one_week_ago:
+            # keep at least one point in the plot if there was
+            # no TA data this week
+            start_date = last_data_point
+        else:
+            start_date = one_week_ago
+
+        # allowed range is from the first ever data point to today
+        self.date_range = DateRangeSlider(
+            title="Date range displayed", start=first_data_point,
+            end=end_date, value=(start_date, end_date), step=1)
+
+        callback = CustomJS(args=dict(s=self.source), code="""
+            s.change.emit();
+        """)
+        self.date_range.js_on_change('value', callback)
+
+        filt = CustomJSFilter(args=dict(slider=self.date_range), code="""
+                var indices = [];
+                var start = slider.value[0];
+                var end = slider.value[1];
+
+                for (var i=0; i < source.get_length(); i++) {
+                    if (source.data['time_arr'][i] >= start 
+                            && source.data['time_arr'][i] <= end) {
+                        indices.push(true);
+                    } else {
+                        indices.push(false);
+                    }
+                }
+                return indices;
+                """)
+        self.date_view = CDSView(source=self.source, filters=[filt])
+
     def mk_plt_layout(self):
         """Create the bokeh plot layout"""
         self.source = ColumnDataSource(data=self.wata_data)
+
         # make sure all arrays are lists in order to later be able to read the data
         # from the html file
         for item in self.source.data:
             if not isinstance(self.source.data[item], (str, float, int, list)):
                 self.source.data[item] = self.source.data[item].tolist()
+
+        # add a time array to the data source
+        self.add_time_column()
+
+        # set up selection tools to share
+        self.share_tools = [BoxSelectTool()]
+
+        # set up a date range filter widget
+        self.setup_date_range()
+
         # set the output html file name and create the plot grid
         output_file(self.output_file_name)
         p1 = self.plt_status()
@@ -552,11 +649,14 @@ def mk_plt_layout(self):
         p4 = self.plt_v3offset_time()
         p5 = self.plt_centroid()
         p6 = self.plt_mag_time()
+
         # make grid
         grid = gridplot([p1, p2, p3, p4, p5, p6], ncols=2, merge_tools=False)
-        save(grid)
+        box_layout = layout(children=[self.date_range, grid])
+        save(box_layout)
+
         # return the needed components for embeding the results in the WATA html template
-        script, div = components(grid)
+        script, div = components(box_layout)
         return script, div
 
     def identify_tables(self):
@@ -729,15 +829,15 @@ def update_ta_success_txtfile(self):
             Nothing
         """
         output_success_ta_txtfile = os.path.join(self.output_dir, "wata_success.txt")
-        # check if previous file exsists and read the data from it
+        # check if previous file exists and read the data from it
         if os.path.isfile(output_success_ta_txtfile):
-            # now rename the the previous file, for backup
+            # now rename the previous file, for backup
             os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_wata_success.txt"))
         # get the new data
         ta_success, ta_failure = [], []
         filenames, ta_status = self.wata_data.loc[:,'filename'], self.wata_data.loc[:,'ta_status']
         for fname, ta_stat in zip(filenames, ta_status):
-            # select the appriopriate list to append to
+            # select the appropriate list to append to
             if ta_stat == 'SUCCESSFUL':
                 ta_success.append(fname)
             else:
diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index 6ea0c1e9e..d0f1c8cea 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -20,18 +20,20 @@
 import os
 
 from astropy.io import fits
-from astropy.visualization import ZScaleInterval
+from astropy.visualization import ZScaleInterval, MinMaxInterval, PercentileInterval
 import numpy as np
 from bokeh.embed import components
+from bokeh.layouts import layout
 from bokeh.models import (
-    BasicTicker, BoxZoomTool, ColorBar, HoverTool, LinearColorMapper, LogColorMapper,
-    LogTicker, WheelZoomTool)
+    BasicTicker, BoxZoomTool, Button, ColorBar, CustomJS, Div, HoverTool,
+    LinearColorMapper, LogColorMapper, LogTicker, RadioGroup, Row, Select, Spacer,
+    Spinner, WheelZoomTool)
 from bokeh.plotting import figure, output_file, show, save
 
 from jwst.datamodels import dqflags
 
 
-class InteractivePreviewImg():
+class InteractivePreviewImg:
     """Class to create the interactive Bokeh figure.
     """
 
@@ -113,16 +115,20 @@ def create_bokeh_image(self):
             limits = (self.low_lim, limits[1])
         if self.high_lim is not None:
             limits = (limits[0], self.high_lim)
-        if self.scaling == 'log':
-            if limits[0] <= 0:
-                limits = (1e-4, limits[1])
-            color_mapper = LogColorMapper(
-                palette="Viridis256", low=limits[0], high=limits[1])
-            ticker = LogTicker()
-        elif self.scaling == 'lin':
-            color_mapper = LinearColorMapper(
-                palette="Viridis256", low=limits[0], high=limits[1])
-            ticker = BasicTicker()
+
+        # handle log or linear scaling
+        if limits[0] <= 0:
+            log_limits = (1e-4, limits[1])
+        else:
+            log_limits = limits
+        log_color_mapper = LogColorMapper(
+            palette="Viridis256", low=log_limits[0], high=log_limits[1])
+        log_ticker = LogTicker()
+        lin_color_mapper = LinearColorMapper(
+            palette="Viridis256", low=limits[0], high=limits[1])
+        lin_ticker = BasicTicker()
+        active = int(self.scaling == 'log')
+
         yd, xd = self.data.shape
         info = dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd])
         if 'DQ' in self.extname:
@@ -132,6 +138,8 @@ def create_bokeh_image(self):
                         title=os.path.basename(self.filename))
 
         # fix figure aspect from data aspect
+        # bokeh throws errors if plot is too small, so make sure
+        # the smaller dimension has reasonable size
         if xd > yd:
             plot_width = 800
             plot_height = int(plot_width * yd / xd)
@@ -148,13 +156,24 @@ def create_bokeh_image(self):
         fig.add_tools(BoxZoomTool(match_aspect=True))
         fig.add_tools(WheelZoomTool(zoom_on_axis=False))
 
-        img = fig.image(source=info, image='image',
-                        level="image", color_mapper=color_mapper)
-        color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker,
-                             title=self.signal_units, bar_line_color='black',
-                             minor_tick_line_color='black', major_tick_line_color='black')
-        fig.add_layout(color_bar, 'below')
+        # make both linear and log scale images to allow toggling between them
+        images = []
+        color_bars = []
+        scales = ((lin_color_mapper, lin_ticker), (log_color_mapper, log_ticker))
+        for i, config in enumerate(scales):
+            color_mapper, ticker = config
+            visible = (i == active)
+            img = fig.image(source=info, image='image',
+                            level="image", color_mapper=color_mapper, visible=visible)
+            color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker,
+                                 title=self.signal_units, bar_line_color='black',
+                                 minor_tick_line_color='black', major_tick_line_color='black',
+                                 visible=visible)
+            fig.add_layout(color_bar, 'below')
+            images.append(img)
+            color_bars.append(color_bar)
 
+        # limit whitespace around image as much as possible
         fig.x_range.range_padding = fig.y_range.range_padding = 0
         if xd >= yd:
             fig.x_range.start = 0
@@ -168,23 +187,130 @@ def create_bokeh_image(self):
         if 'DQ' not in self.extname:
             hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"),
                                              ('Value', '@image{0.4f}')
-                                             ], mode='mouse', renderers=[img])
+                                             ], mode='mouse', renderers=images)
         else:
             hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"),
                                              ('Value', '@dq')
-                                             ], mode='mouse', renderers=[img])
+                                             ], mode='mouse', renderers=images)
         self.create_figure_title()
         fig.title.text = self.title
         fig.xaxis.axis_label = 'Pixel'
         fig.yaxis.axis_label = 'Pixel'
         fig.tools.append(hover_tool)
+
+        # add interactive widgets
+        widgets = self.add_interactive_controls(images, color_bars)
+        box_layout = layout(children=[fig, *widgets])
+
         # Show figure on screen if requested
         if self.show:
-            show(fig)
+            show(box_layout)
         elif self.save_html is not None:
-            save(fig)
+            save(box_layout)
         else:
-            return components(fig)
+            return components(box_layout)
+
+    def add_interactive_controls(self, images, color_bars):
+        """
+        Add client-side controls for images.
+
+        Currently includes image scaling and limit setting controls.
+
+        Parameters
+        ----------
+        images : list of bokeh.models.Image
+            2-element list of images. The first is linear scale, second is log scale.
+            Only one should be visible at any time.
+        color_bars : list of bokeh.models.ColorBar
+            2-element list of color bars, matching the images.
+
+        Returns
+        -------
+        widgets: list of bokeh.Widget
+            Widgets to add to the page layout.
+        """
+        # active scaling (0=linear, 1=log)
+        active = int(self.scaling == 'log')
+
+        tools_label = Div(text="<h4>Image Settings</h4>")
+
+        scale_label = Div(text="Scaling:")
+        scale_group = RadioGroup(labels=["linear", "log"],
+                                 inline=True, active=active)
+        scale_set = Row(scale_label, scale_group,
+                        css_classes=['mb-4'])
+
+        current_low = images[active].glyph.color_mapper.low
+        current_high = images[active].glyph.color_mapper.high
+        preset_limits = {'ZScale': (current_low, current_high),
+                         'Min/Max': MinMaxInterval().get_limits(self.data),
+                         '99.5%': PercentileInterval(99.5).get_limits(self.data),
+                         '99%': PercentileInterval(99).get_limits(self.data),
+                         '95%': PercentileInterval(95).get_limits(self.data),
+                         '90%': PercentileInterval(90).get_limits(self.data)}
+        options = [*preset_limits.keys(), 'Custom']
+        preset_label = Div(text="Percentile presets:")
+        preset_select = Select(value='ZScale', options=options, width=120)
+        preset_set = Row(preset_label, preset_select)
+
+        limit_label = Div(text="Limits:")
+        limit_low = Spinner(title="Low", value=current_low)
+        limit_high = Spinner(title="High", value=current_high)
+        reset = Button(label='Reset', button_type='primary')
+        limit_set = Row(limit_label, limit_low, limit_high,
+                        css_classes=['mb-4'])
+
+        # JS callbacks for client side controls
+
+        # set alternate image visibility when scale selection changes
+        scale_group.js_on_click(CustomJS(args={'i1': images[0], 'c1': color_bars[0],
+                                               'i2': images[1], 'c2': color_bars[1]},
+                                         code="""
+            if (i1.visible == true) {
+                i1.visible = false;
+                c1.visible = false;
+                i2.visible = true;
+                c2.visible = true;
+            } else {
+                i1.visible = true;
+                c1.visible = true;
+                i2.visible = false;
+                c2.visible = false;
+            }
+        """))
+
+        # set scaling limits from select box on change
+        limit_reset = CustomJS(
+            args={'setting': preset_select, 'limits': preset_limits, 'low': limit_low,
+                  'high': limit_high, 'scale': scale_group},
+            code="""
+                if (setting.value != "Custom") {
+                    if (scale.active == 1 && limits[setting.value][0] <= 0) {
+                        low.value = 0.0001;
+                    } else {
+                        low.value = limits[setting.value][0];
+                    }
+                    high.value = limits[setting.value][1];
+                }
+                """)
+        preset_select.js_on_change('value', limit_reset)
+
+        # set scaling limits from text boxes on change
+        for i in range(len(images)):
+            limit_low.js_link('value', images[i].glyph.color_mapper, 'low')
+            limit_low.js_link('value', color_bars[i].color_mapper, 'low')
+            limit_high.js_link('value', images[i].glyph.color_mapper, 'high')
+            limit_high.js_link('value', color_bars[i].color_mapper, 'high')
+
+        # reset boxes to preset range on button click
+        reset.js_on_click(limit_reset)
+
+        # also reset when swapping limit style
+        scale_group.js_on_click(limit_reset)
+
+        # return widgets
+        spacer = Spacer(height=20)
+        return [tools_label, scale_set, preset_set, limit_set, reset, spacer]
 
     def create_figure_title(self):
         """Create title for the image"""
diff --git a/jwql/website/apps/jwql/static/css/jwql.css b/jwql/website/apps/jwql/static/css/jwql.css
index 8431c82f3..eb567b25f 100644
--- a/jwql/website/apps/jwql/static/css/jwql.css
+++ b/jwql/website/apps/jwql/static/css/jwql.css
@@ -23,24 +23,31 @@
     width: 100%;
     margin: auto;
   }
-  
+
+  /* Make sure bokeh widgets match default theme */
+  .bk-root, .bk-btn {
+      font-family: 'Overpass', sans-serif !important;
+      font-size: 1rem !important;
+      flex-grow: 0 !important;
+  }
+
   /*Make normal buttons and highlighted outline buttons orange*/
   .btn-primary, .btn-primary.disabled, .btn-outline-primary:hover,
-  .btn-outline-primary.active {
+  .btn-outline-primary.active, .bk-btn-primary {
       background-color: #c85108 !important;
       border-color: #c85108 !important;
       color: white !important;
-      border-radius: 0px;
+      border-radius: 0px !important;
       text-decoration: none;
   }
   
   /*Make outline buttons and highlighted normal buttons white*/
   .btn-primary:hover, .btn-primary.active, .btn-outline-primary,
-  .show > .btn-primary.dropdown-toggle {
+  .show > .btn-primary.dropdown-toggle, .bk-btn-primary:hover {
       background-color: white !important;
       border-color: #c85108 !important ;
       color: #c85108 !important;
-      border-radius: 0px;
+      border-radius: 0px !important;
       text-decoration: none;
   }
   
@@ -544,7 +551,7 @@
   }
 
    /*Format thumbnail groups when active*/
-   .thumbnail-group {
+  .thumbnail-group {
       display: inline;
       font-size: 0.75rem;
   }
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index e86dfa9c6..705931280 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1075,9 +1075,6 @@ function update_wata_page(base_url) {
         var calc_difference = document.getElementById("calcDifference").checked;
 
         // Get the arguments to update
-        var scaling = get_radio_button_value("scaling");
-        var low_lim = get_number_or_none("low_lim");
-        var high_lim = get_number_or_none("high_lim");
         var ext_name = get_radio_button_value("extension");
         var int1_nr = get_number_or_none("integration1");
         var grp1_nr = get_number_or_none("group1");
@@ -1090,7 +1087,7 @@ function update_wata_page(base_url) {
             int2_nr="None";
             grp2_nr="None";
         }
-        optional_params = optional_params + "/scaling_" + scaling + "/low_" + low_lim + "/high_" + high_lim + "/ext_" + ext_name + "/int1_" + int1_nr + "/grp1_" + grp1_nr + "/int2_" + int2_nr + "/grp2_" + grp2_nr;
+        optional_params = optional_params + "/ext_" + ext_name + "/int1_" + int1_nr + "/grp1_" + grp1_nr + "/int2_" + int2_nr + "/grp2_" + grp2_nr;
     }
 
     $.ajax({
diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html
index cced80f37..f72a92081 100644
--- a/jwql/website/apps/jwql/templates/explore_image.html
+++ b/jwql/website/apps/jwql/templates/explore_image.html
@@ -17,59 +17,12 @@ <h3>{{ file_root }}_{{ filetype }}.fits</h3>
             <a class="btn btn-primary my-2 mx-2" role="button" href={{ url('jwql:archive_thumb_per_obs', args=[inst, file_root[2:7], file_root[7:10]]) }}> View Proposal</a>
 	    </p>
 
-        <!-- Allow the user to change the settings in the image to explore -->
-        <h4>Image Settings</h4>
-        <div class="mt-4">
-            Scaling:&nbsp;&nbsp;
-            <input type="radio" name="scaling" value="log" id="log"> log &nbsp;&nbsp;&nbsp;&nbsp;
-            <input type="radio" name="scaling" value="lin" id="lin" checked="checked"> linear &nbsp;&nbsp;
-            <br>
-            Scale Limits:&nbsp;&nbsp;
-            <input type="number" id="low_lim" size="15" step="any">&nbsp;Low &nbsp;&nbsp;&nbsp;&nbsp;
-            <input type="number" id="high_lim" size="15" step="any">&nbsp;High &nbsp;&nbsp;&nbsp;&nbsp;
-            <br>
-            <br>
-            Extension:&nbsp;&nbsp;
-            {% for extension in extensions %}
-                {% if extension == 'SCI' %}
-                    <input type="radio" name="extension" value="{{ extension }}" id="{{ extension }}" checked="checked" onclick='explore_image_update_enable_options("{{ extension_ints }}", "{{ extension_groups }}");'> {{ extension }} &nbsp;&nbsp;&nbsp;&nbsp;
-                {% else %}
-                    <input type="radio" name="extension" value="{{ extension }}" id="{{ extension }}" onclick='explore_image_update_enable_options("{{ extension_ints }}", "{{ extension_groups }}");'> {{ extension }} &nbsp;&nbsp;&nbsp;&nbsp;
-                {% endif %}
-            {% endfor %}
-            <div class="row">
-                <div class="explorer_options" id="integrationInput1">
-                    <input type="number" id="integration1" min="0" size="3" step="1">
-                    <label id="integration1-name" for="integration1">Integration</label>
-                </div>
-                <div class="explorer_options" id="groupInput1">
-                    <input type="number" id="group1" min="0" size="3" step="1">
-                    <label id="group1-name" for="group1">Group</label>
-                </div>
-            </div>
-            <div class="form-check" id="calcDifferenceForm">
-                <input class="form-check-input" type="checkbox" id="calcDifference" onclick='explore_image_update_enable_options("{{ extension_ints }}", "{{ extension_groups }}");'>
-                <label class="form-check-label" for="calcDifference">Calculate Difference</label>
-            </div>
-            <div class="row">
-                <div class="explorer_options" id="integrationInput2">
-                    <input type="number" id="integration2" min="0" size="3" step="1">
-                    <label id="integration2-name" for="integration2">Integration</label>
-                </div>
-                <div class="explorer_options" id="groupInput2">
-                    <input type="number" id="group2" min="0" size="3" step="1">
-                    <label id="group2-name" for="group2">Group</label>
-                </div>
-            </div>
-        </div>
-        <p>
-            <a class="btn btn-primary my-2 mx-2" role="button" onclick="update_explore_image_page('{{inst}}', '{{file_root}}', '{{filetype}}', '{{base_url}}', true)"> Apply Settings</a>
-	    </p>
+
 		<div class="row">
             <!-- Display the image -->
-    	    <div class="col-xl-9 text-center">
+    	    <div class="col-xl-9">
     		    <!-- Loading animation -->
-                <div id="loading">
+                <div id="loading" class="text-center">
                     <div class="lds-css ng-scope">
                         <div style="width:100%;height:100%" class="lds-magnify">
                             <div>
@@ -88,6 +41,46 @@ <h4>Image Settings</h4>
                 <!-- Display the data -->
                 <div id='explore_image'></div>
                 <a id="explore_image_fail" style='display: none'>Image explorer failed to load</a>
+
+                <!-- Allow the user to change the settings in the image to explore -->
+                <h4>Data Settings</h4>
+                <div class="mt-4">
+                    Extension:&nbsp;&nbsp;
+                    {% for extension in extensions %}
+                        {% if extension == 'SCI' %}
+                            <input type="radio" name="extension" value="{{ extension }}" id="{{ extension }}" checked="checked" onclick='explore_image_update_enable_options("{{ extension_ints }}", "{{ extension_groups }}");'> {{ extension }} &nbsp;&nbsp;&nbsp;&nbsp;
+                        {% else %}
+                            <input type="radio" name="extension" value="{{ extension }}" id="{{ extension }}" onclick='explore_image_update_enable_options("{{ extension_ints }}", "{{ extension_groups }}");'> {{ extension }} &nbsp;&nbsp;&nbsp;&nbsp;
+                        {% endif %}
+                    {% endfor %}
+                    <div class="row">
+                        <div class="explorer_options" id="integrationInput1">
+                            <input type="number" id="integration1" min="0" size="3" step="1">
+                            <label id="integration1-name" for="integration1">Integration</label>
+                        </div>
+                        <div class="explorer_options" id="groupInput1">
+                            <input type="number" id="group1" min="0" size="3" step="1">
+                            <label id="group1-name" for="group1">Group</label>
+                        </div>
+                    </div>
+                    <div class="form-check" id="calcDifferenceForm">
+                        <input class="form-check-input" type="checkbox" id="calcDifference" onclick='explore_image_update_enable_options("{{ extension_ints }}", "{{ extension_groups }}");'>
+                        <label class="form-check-label" for="calcDifference">Calculate Difference</label>
+                    </div>
+                    <div class="row">
+                        <div class="explorer_options" id="integrationInput2">
+                            <input type="number" id="integration2" min="0" size="3" step="1">
+                            <label id="integration2-name" for="integration2">Integration</label>
+                        </div>
+                        <div class="explorer_options" id="groupInput2">
+                            <input type="number" id="group2" min="0" size="3" step="1">
+                            <label id="group2-name" for="group2">Group</label>
+                        </div>
+                    </div>
+                </div>
+                <p>
+                    <a class="btn btn-primary my-2 mx-2" role="button" onclick="update_explore_image_page('{{inst}}', '{{file_root}}', '{{filetype}}', '{{base_url}}', true)"> Apply Settings</a>
+                </p>
             </div>
 
             <!-- Display the anomaly form -->
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index 4a6227047..c0535b378 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -96,7 +96,7 @@
     re_path('ajax/query_submit/', views.archive_thumbnails_query_ajax, name='archive_thumb_query_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/archive/$'.format(instruments), views.archived_proposals_ajax, name='archive_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'),
-    re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/scaling_(?P<scaling>.+)/low_(?P<low_lim>.+)/high_(?P<high_lim>.+)/ext_(?P<ext_name>.+)/int1_(?P<int1_nr>.+)/grp1_(?P<grp1_nr>.+)/int2_(?P<int2_nr>.+)/grp2_(?P<grp2_nr>.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'),
+    re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/ext_(?P<ext_name>.+)/int1_(?P<int1_nr>.+)/grp1_(?P<grp1_nr>.+)/int2_(?P<int2_nr>.+)/grp2_(?P<grp2_nr>.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/archive/(?P<proposal>[\d]{{1,5}})/obs(?P<observation>[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_ajax, name='archive_thumb_ajax'),
     re_path(r'^ajax/viewed/(?P<file_root>.+)/$', views.toggle_viewed_ajax, name='toggle_viewed_ajax'),
     re_path(r'^ajax/viewed_group/(?P<group_root>.+)/(?P<status>(viewed|new|Viewed|New))/$',

From 6815685f57cc6be99871bcd530ca91a469464502 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Thu, 8 Jun 2023 16:27:33 -0400
Subject: [PATCH 144/256] Fix accidental commit for TA monitors

---
 .../ta_monitors/msata_monitor.py              | 322 +++++-------------
 .../ta_monitors/wata_monitor.py               | 232 ++++---------
 2 files changed, 156 insertions(+), 398 deletions(-)

diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
index 55abcf95e..bc21a4643 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/msata_monitor.py
@@ -24,7 +24,6 @@
 Author
 ______
     - Maria Pena-Guerrero
-    - Melanie Clarke
 
 Use
 ---
@@ -39,7 +38,6 @@
 import os
 import logging
 import shutil
-from datetime import datetime, timezone, timedelta
 from random import randint
 
 import numpy as np
@@ -47,13 +45,12 @@
 from astropy.time import Time
 from astropy.io import fits
 from bokeh.embed import components
-from bokeh.layouts import gridplot, layout
-from bokeh.models import (
-    ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView,
-    Span, Label, DateRangeSlider)
-from bokeh.models.tools import HoverTool, BoxSelectTool
-from bokeh.plotting import figure, save, output_file
+from bokeh.plotting import figure, output_file, save
+from bokeh.layouts import gridplot
+from bokeh.models import ColumnDataSource, Range1d, Span, Label
+from bokeh.models.tools import HoverTool
 from bs4 import BeautifulSoup
+from datetime import datetime
 from sqlalchemy.sql.expression import and_
 
 # jwql imports
@@ -62,11 +59,11 @@
 from jwql.utils import monitor_utils
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.logging_functions import log_info, log_fail
-from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, filename_parser
 
 
 class MSATA():
-    """ Class for executing the NIRSpec MSATA monitor.
+    """ Class for executint the NIRSpec MSATA monitor.
 
     This class will search for new MSATA current files in the file systems
     for NIRSpec and will run the monitor on these files. The monitor will
@@ -139,13 +136,6 @@ def __init__(self):
                                'planned_v3': {'loc': 'ta_table', 'alt_key': None, 'name': 'planned_v3', 'type': float},
                                'FITTARGS': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'stars_in_fit', 'type': int}}
 
-        # initialize attributes to be set later
-        self.source = None
-        self.share_tools = []
-        self.date_range = None
-        self.date_filter = None
-        self.date_view = None
-
     def get_tainfo_from_fits(self, fits_file):
         """ Get the TA information from the fits file
         Parameters
@@ -250,19 +240,6 @@ def get_msata_data(self, new_filenames):
         msata_df = pd.DataFrame(msata_dict)
         return msata_df, no_ta_ext_msgs
 
-    def add_time_column(self):
-        """Add time column to data source, to be used by all plots."""
-        date_obs = self.source.data['date_obs']
-        if 'time_arr' not in self.source.data:
-            time_arr = []
-            for do_str in date_obs:
-                # convert time string into an array of time (this is in UT)
-                t = datetime.fromisoformat(do_str)
-                time_arr.append(t)
-
-            # add to the bokeh data structure
-            self.source.data["time_arr"] = time_arr
-
     def plt_status(self):
         """ Plot the MSATA status versus time.
         Parameters
@@ -272,12 +249,18 @@ def plt_status(self):
         -------
             plot: bokeh plot object
         """
+        # ta_status, date_obs = source.data['ta_status'], source.data['date_obs']
+        date_obs = self.source.data['date_obs']
         ta_status = self.source.data['ta_status']
         # check if this column exists in the data already (the other 2 will exist too), else create it
-        if 'bool_status' not in self.source.data:
-            # bokeh does not like to plot strings, turn into numbers
-            number_status, status_colors = [], []
-            for tas in ta_status:
+        try:
+            time_arr = self.source.data['time_arr']
+            bool_status = self.source.data['bool_status']
+            status_colors = self.source.data['status_colors']
+        except KeyError:
+            # bokeh does not like to plot strings, turn  into numbers
+            number_status, time_arr, status_colors = [], [], []
+            for tas, do_str in zip(ta_status, date_obs):
                 if tas.lower() == 'unsuccessful':
                     number_status.append(0.0)
                     status_colors.append('red')
@@ -287,19 +270,19 @@ def plt_status(self):
                 else:
                     number_status.append(1.0)
                     status_colors.append('blue')
-
+                # convert time string into an array of time (this is in UT with 0.0 milliseconds)
+                t = datetime.fromisoformat(do_str)
+                time_arr.append(t)
             # add these to the bokeh data structure
+            self.source.data["time_arr"] = time_arr
             self.source.data["number_status"] = number_status
             self.source.data["status_colors"] = status_colors
-
         # create a new bokeh plot
         plot = figure(title="MSATA Status [Success=1, In Progress=0.5, Fail=0]", x_axis_label='Time',
                       y_axis_label='MSATA Status', x_axis_type='datetime',)
         plot.y_range = Range1d(-0.5, 1.5)
         plot.circle(x='time_arr', y='number_status', source=self.source,
-                    color='status_colors', size=7, fill_alpha=0.3, view=self.date_view)
-
-        # add tooltips
+                    color='status_colors', size=7, fill_alpha=0.3)
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -310,11 +293,8 @@ def plt_status(self):
                           ('Date-Obs', '@date_obs'),
                           ('Subarray', '@subarray'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_residual_offsets(self):
@@ -331,8 +311,7 @@ def plt_residual_offsets(self):
                       x_axis_label='Least Squares Residual V2 Offset',
                       y_axis_label='Least Squares Residual V3 Offset')
         plot.circle(x='lsv2offset', y='lsv3offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
-
+                    color="blue", size=7, fill_alpha=0.3)
         v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet']
         xstart, ystart, ray_length = -1 * v2halffacet[0], -1 * v3halffacet[0], 0.05
         plot.ray(x=xstart - ray_length / 2.0, y=ystart, length=ray_length, angle_units="deg",
@@ -343,13 +322,10 @@ def plt_residual_offsets(self):
         plot.add_layout(hflabel)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
-
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -362,11 +338,8 @@ def plt_residual_offsets(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_v2offset_time(self):
@@ -382,9 +355,8 @@ def plt_v2offset_time(self):
         plot = figure(title="MSATA Least Squares V2 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V2 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv2offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.y_range = Range1d(-0.5, 0.5)
-
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         time_arr, v2halffacet = self.source.data['time_arr'], self.source.data['v2halffacet']
@@ -392,8 +364,6 @@ def plt_v2offset_time(self):
         plot.renderers.extend([hline, hfline])
         hflabel = Label(x=time_arr[-1], y=-1 * v2halffacet[0], y_units='data', text='-V2 half-facet value')
         plot.add_layout(hflabel)
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -406,11 +376,8 @@ def plt_v2offset_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_v3offset_time(self):
@@ -426,9 +393,8 @@ def plt_v3offset_time(self):
         plot = figure(title="MSATA Least Squares V3 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V3 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv3offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.y_range = Range1d(-0.5, 0.5)
-
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         time_arr, v3halffacet = self.source.data['time_arr'], self.source.data['v3halffacet']
@@ -436,8 +402,6 @@ def plt_v3offset_time(self):
         plot.renderers.extend([hline, hfline])
         hflabel = Label(x=time_arr[-1], y=-1 * v3halffacet[0], y_units='data', text='-V3 half-facet value')
         plot.add_layout(hflabel)
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -450,11 +414,8 @@ def plt_v3offset_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_lsv2v3offsetsigma(self):
@@ -471,16 +432,13 @@ def plt_lsv2v3offsetsigma(self):
                       x_axis_label='Least Squares Residual V2 Sigma Offset',
                       y_axis_label='Least Squares Residual V3 Sigma Offset')
         plot.circle(x='lsv2sigma', y='lsv3sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.x_range = Range1d(-0.1, 0.1)
         plot.y_range = Range1d(-0.1, 0.1)
-
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -495,11 +453,8 @@ def plt_lsv2v3offsetsigma(self):
                           ('LS V3 offset', '@lsv3offset'),
                           ('LS V3 sigma', '@lsv3sigma'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_res_offsets_corrected(self):
@@ -511,30 +466,29 @@ def plt_res_offsets_corrected(self):
         -------
             plot: bokeh plot object
         """
+        # create a new bokeh plot
         lsv2offset, lsv3offset = self.source.data['lsv2offset'], self.source.data['lsv3offset']
         v2halffacet, v3halffacet = self.source.data['v2halffacet'], self.source.data['v3halffacet']
-
-        # check if this column exists in the data already, else create it
-        if 'v2_half_fac_corr' not in self.source.data:
+        # check if this column exists in the data already (the other 2 will exist too), else create it
+        try:
+            v2_half_fac_corr = self.source.data['v2_half_fac_corr']
+            v3_half_fac_corr = self.source.data['v3_half_fac_corr']
+        except KeyError:
             v2_half_fac_corr, v3_half_fac_corr = [], []
             for idx, v2hf in enumerate(v2halffacet):
                 v3hf = v3halffacet[idx]
                 v2_half_fac_corr.append(lsv2offset[idx] + v2hf)
                 v3_half_fac_corr.append(lsv3offset[idx] + v3hf)
-
             # add these to the bokeh data structure
             self.source.data["v2_half_fac_corr"] = v2_half_fac_corr
             self.source.data["v3_half_fac_corr"] = v3_half_fac_corr
-
-        # create a new bokeh plot
         plot = figure(title="MSATA Least Squares Residual V2-V3 Offsets Half-facet corrected",
                       x_axis_label='Least Squares Residual V2 Offset + half-facet',
                       y_axis_label='Least Squares Residual V3 Offset + half-facet')
         plot.circle(x='v2_half_fac_corr', y='v3_half_fac_corr', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
-
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
@@ -546,8 +500,6 @@ def plt_res_offsets_corrected(self):
                  angle=90, line_color='purple', line_width=3)
         hflabel = Label(x=xstart / 3.0, y=ystart, y_units='data', text='-V2, -V3 half-facets values')
         plot.add_layout(hflabel)
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -562,16 +514,15 @@ def plt_res_offsets_corrected(self):
                           ('V2 half-facet', '@v2halffacet'),
                           ('V3 half-facet', '@v3halffacet'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_v2offsigma_time(self):
-        """Plot the residual Least Squares V2 sigma Offset versus time
-
+        """ Plot the residual Least Squares V2 sigma Offset versus time
+        Parameters
+        ----------
+            None
         Returns
         -------
             plot: bokeh plot object
@@ -580,14 +531,11 @@ def plt_v2offsigma_time(self):
         plot = figure(title="MSATA Least Squares V2 Sigma Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual V2 Sigma Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv2sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.y_range = Range1d(-0.1, 0.1)
-
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -599,34 +547,28 @@ def plt_v2offsigma_time(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V2 sigma', '@lsv2sigma'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_v3offsigma_time(self):
-        """Plot the residual Least Squares V3 Offset versus time
-
+        """ Plot the residual Least Squares V3 Offset versus time
+        Parameters
+        ----------
+            None
         Returns
         -------
             p: bokeh plot object
         """
         # create a new bokeh plot
-        plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time",
-                      x_axis_label='Time',
-                      y_axis_label='Least Squares Residual V3 Sigma Offset',
-                      x_axis_type='datetime')
+        plot = figure(title="MSATA Least Squares V3 Sigma Offset vs Time", x_axis_label='Time',
+                      y_axis_label='Least Squares Residual V3 Sigma Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsv3sigma', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.y_range = Range1d(-0.1, 0.1)
-
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -639,11 +581,8 @@ def plt_v3offsigma_time(self):
                           ('LS V3 offset', '@lsv3offset'),
                           ('LS V3 sigma', '@lsv3sigma'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_roll_offset(self):
@@ -659,12 +598,10 @@ def plt_roll_offset(self):
         plot = figure(title="MSATA Least Squares Roll Offset vs Time", x_axis_label='Time',
                       y_axis_label='Least Squares Residual Roll Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsrolloffset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.y_range = Range1d(-600.0, 600.0)
-
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
-
         # Maximum accepted roll line and label
         time_arr = self.source.data['time_arr']
         arlinepos = Span(location=120, dimension='width', line_color='green', line_width=3)
@@ -672,8 +609,6 @@ def plt_roll_offset(self):
         arlabel = Label(x=time_arr[-1], y=125, y_units='data', text='Max accepted roll')
         plot.add_layout(arlabel)
         plot.renderers.extend([hline, arlinepos, arlineneg])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -686,11 +621,8 @@ def plt_roll_offset(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_lsoffsetmag(self):
@@ -706,14 +638,11 @@ def plt_lsoffsetmag(self):
         plot = figure(title="MSATA Least Squares Total Magnitude of the Linear V2, V3 Offset Slew vs Time", x_axis_label='Time',
                       y_axis_label='sqrt((V2_off)**2 + (V3_off)**2)', x_axis_type='datetime')
         plot.circle(x='time_arr', y='lsoffsetmag', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.y_range = Range1d(-0.5, 0.5)
-
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -727,11 +656,8 @@ def plt_lsoffsetmag(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_tot_number_of_stars(self):
@@ -746,34 +672,34 @@ def plt_tot_number_of_stars(self):
         # get the number of stars per array
         visit_id = self.source.data['visit_id']
         reference_star_number = self.source.data['reference_star_number']
-
-        # check if this column exists in the data already, else create it
-        if 'tot_number_of_stars' not in self.source.data:
+        stars_in_fit = self.source.data['stars_in_fit']
+        date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr']
+        # check if this column exists in the data already (the other 2 will exist too), else create it
+        try:
+            tot_number_of_stars = self.source.data['tot_number_of_stars']
+            colors_list = self.source.data['colors_list']
+        except KeyError:
             # create the list of color per visit and tot_number_of_stars
             colors_list, tot_number_of_stars = [], []
             color_dict = {}
-            for i, vid in enumerate(visit_id):
+            for i, _ in enumerate(visit_id):
                 tot_stars = len(reference_star_number[i])
                 tot_number_of_stars.append(tot_stars)
                 ci = '#%06X' % randint(0, 0xFFFFFF)
-                if vid not in color_dict:
-                    color_dict[vid] = ci
-                colors_list.append(color_dict[vid])
-
+                if visit_id[i] not in color_dict:
+                    color_dict[visit_id[i]] = ci
+                colors_list.append(color_dict[visit_id[i]])
             # add these to the bokeh data structure
             self.source.data["tot_number_of_stars"] = tot_number_of_stars
             self.source.data["colors_list"] = colors_list
-
         # create a new bokeh plot
         plot = figure(title="Total Number of Measurements vs Time", x_axis_label='Time',
                       y_axis_label='Total number of measurements', x_axis_type='datetime')
         plot.circle(x='time_arr', y='tot_number_of_stars', source=self.source,
-                    color='colors_list', size=7, fill_alpha=0.3, view=self.date_view)
+                    color='colors_list', size=7, fill_alpha=0.3)
         plot.triangle(x='time_arr', y='stars_in_fit', source=self.source,
-                      color='black', size=7, fill_alpha=0.3, view=self.date_view)
+                      color='black', size=7, fill_alpha=0.3)
         plot.y_range = Range1d(0.0, 40.0)
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -788,8 +714,8 @@ def plt_tot_number_of_stars(self):
                           ('LS V2 offset', '@lsv2offset'),
                           ('LS V3 offset', '@lsv3offset'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
+        plot.add_tools(hover)
         return plot
 
     def plt_mags_time(self):
@@ -809,24 +735,22 @@ def plt_mags_time(self):
         planned_v2 = self.source.data['planned_v2']
         planned_v3 = self.source.data['planned_v3']
         reference_star_number = self.source.data['reference_star_number']
+        visits_stars_mags = self.source.data['reference_star_mag']
         box_peak_value = self.source.data['box_peak_value']
         date_obs, time_arr = self.source.data['date_obs'], self.source.data['time_arr']
         colors_list = self.source.data['colors_list']
         detector_list = self.source.data['detector']
-        filename = self.source.data['filename']
-
         # create the structure matching the number of visits and reference stars
         new_colors_list, vid, dobs, tarr, star_no, status = [], [], [], [], [], []
-        peaks, stars_v2, stars_v3, det, fnames = [], [], [], [], []
+        peaks, stars_v2, stars_v3, det = [], [], [], []
         for i, _ in enumerate(visit_id):
-            v, d, t, c, s, x, y, dt, fn = [], [], [], [], [], [], [], [], []
+            v, d, t, c, s, x, y, dt = [], [], [], [], [], [], [], []
             for j in range(len(reference_star_number[i])):
                 v.append(visit_id[i])
                 d.append(date_obs[i])
                 t.append(time_arr[i])
                 c.append(colors_list[i])
                 dt.append(detector_list[i])
-                fn.append(filename[i])
                 if 'not_removed' in lsf_removed_status[i][j]:
                     s.append('SUCCESS')
                     x.append(planned_v2[i][j])
@@ -845,29 +769,17 @@ def plt_mags_time(self):
             stars_v3.extend(y)
             peaks.extend(box_peak_value[i])
             det.extend(dt)
-            fnames.extend(fn)
-
         # now create the mini ColumnDataSource for this particular plot
         mini_source = {'vid': vid, 'star_no': star_no, 'status': status,
-                       'dobs': dobs, 'time_arr': tarr, 'det': det, 'fname': fnames,
+                       'dobs': dobs, 'tarr': tarr, 'det': det,
                        'peaks': peaks, 'colors_list': new_colors_list,
                        'stars_v2': stars_v2, 'stars_v3': stars_v3}
         mini_source = ColumnDataSource(data=mini_source)
-
-        # hook up the date range slider to this source as well
-        callback = CustomJS(args=dict(s=mini_source), code="""
-            s.change.emit();
-        """)
-        self.date_range.js_on_change('value', callback)
-        mini_view = CDSView(source=mini_source, filters=[self.date_filter])
-
-        # create the bokeh plot
-        plot = figure(title="MSATA Counts vs Time", x_axis_label='Time',
-                      y_axis_label='box_peak [Counts]',
+        # create a the bokeh plot
+        plot = figure(title="MSATA Counts vs Time", x_axis_label='Time', y_axis_label='box_peak [Counts]',
                       x_axis_type='datetime')
-        plot.circle(x='time_arr', y='peaks', source=mini_source,
-                    color='colors_list', size=7, fill_alpha=0.3, view=mini_view)
-
+        plot.circle(x='tarr', y='peaks', source=mini_source,
+                    color='colors_list', size=7, fill_alpha=0.3)
         # add count saturation warning lines
         loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0
         hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3)
@@ -881,10 +793,9 @@ def plt_mags_time(self):
         plot.add_layout(label2)
         plot.add_layout(label3)
         plot.y_range = Range1d(-1000.0, 62000.0)
-
-        # add tooltips
+        # add hover
         hover = HoverTool()
-        hover.tooltips = [('File name', '@fname'),
+        hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@vid'),
                           ('Detector', '@det'),
                           ('Star No.', '@star_no'),
@@ -894,69 +805,18 @@ def plt_mags_time(self):
                           ('Measured V2', '@stars_v2'),
                           ('Measured V3', '@stars_v3'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
+        plot.add_tools(hover)
         return plot
 
-    def setup_date_range(self):
-        """Set up a date range filter, defaulting to the last week of data."""
-        end_date = datetime.now(tz=timezone.utc)
-        one_week_ago = end_date.date() - timedelta(days=7)
-        first_data_point = np.min(self.source.data['time_arr']).date()
-        last_data_point = np.max(self.source.data['time_arr']).date()
-        if last_data_point < one_week_ago:
-            # keep at least one point in the plot if there was
-            # no TA data this week
-            start_date = last_data_point
-        else:
-            start_date = one_week_ago
-
-        # allowed range is from the first ever data point to today
-        self.date_range = DateRangeSlider(
-            title="Date range displayed", start=first_data_point,
-            end=end_date, value=(start_date, end_date), step=1)
-
-        callback = CustomJS(args=dict(s=self.source), code="""
-            s.change.emit();
-        """)
-        self.date_range.js_on_change('value', callback)
-
-        self.date_filter = CustomJSFilter(args=dict(slider=self.date_range), code="""
-                var indices = [];
-                var start = slider.value[0];
-                var end = slider.value[1];
-
-                for (var i=0; i < source.get_length(); i++) {
-                    if (source.data['time_arr'][i] >= start 
-                            && source.data['time_arr'][i] <= end) {
-                        indices.push(true);
-                    } else {
-                        indices.push(false);
-                    }
-                }
-                return indices;
-                """)
-        self.date_view = CDSView(source=self.source, filters=[self.date_filter])
-
     def mk_plt_layout(self):
         """Create the bokeh plot layout"""
         self.source = ColumnDataSource(data=self.msata_data)
-
         # make sure all arrays are lists in order to later be able to read the data
         # from the html file
         for item in self.source.data:
             if not isinstance(self.source.data[item], (str, float, int, list)):
                 self.source.data[item] = self.source.data[item].tolist()
-
-        # add a time array to the data source
-        self.add_time_column()
-
-        # set up selection tools to share
-        self.share_tools = [BoxSelectTool()]
-
-        # set up a date range filter widget
-        self.setup_date_range()
-
         # set the output html file name and create the plot grid
         output_file(self.output_file_name)
         p1 = self.plt_status()
@@ -971,15 +831,13 @@ def mk_plt_layout(self):
         p10 = self.plt_lsoffsetmag()
         p12 = self.plt_tot_number_of_stars()
         p11 = self.plt_mags_time()
-
         # make grid
         grid = gridplot([p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12],
                         ncols=2, merge_tools=False)
-        box_layout = layout(children=[self.date_range, grid])
-        save(box_layout)
-
-        # return the needed components for embedding the results in the MSATA html template
-        script, div = components(box_layout)
+        # show(grid)
+        save(grid)
+        # return the needed components for embeding the results in the MSATA html template
+        script, div = components(grid)
         return script, div
 
     def identify_tables(self):
@@ -1026,7 +884,7 @@ def get_data_from_html(self, html_file):
             File created by the monitor script
         Returns
         -------
-        prev_data_dict: dict
+        prev_data_dict: dictionary
             Dictionary containing all data used in the plots
         """
 
@@ -1103,7 +961,7 @@ def prev_data2expected_format(self, prev_data_dict):
             Date of the latest observation in the previously plotted data
         """
         # remember that the time array created is in milliseconds, removing to get time object
-        time_in_millis = max(prev_data_dict['time_arr'])
+        time_in_millis = max(prev_data_dict['tarr'])
         latest_prev_obs = Time(time_in_millis / 1000., format='unix')
         latest_prev_obs = latest_prev_obs.mjd
         prev_data_expected_cols = {}
@@ -1193,13 +1051,13 @@ def update_ta_success_txtfile(self):
         output_success_ta_txtfile = os.path.join(self.output_dir, "msata_success.txt")
         # check if previous file exsists and read the data from it
         if os.path.isfile(output_success_ta_txtfile):
-            # now rename the previous file, for backup
+            # now rename the the previous file, for backup
             os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_msata_success.txt"))
         # get the new data
         ta_success, ta_inprogress, ta_failure = [], [], []
         filenames, ta_status = self.msata_data.loc[:,'filename'], self.msata_data.loc[:,'ta_status']
         for fname, ta_stat in zip(filenames, ta_status):
-            # select the appropriate list to append to
+            # select the appriopriate list to append to
             if ta_stat == 'SUCCESSFUL':
                 ta_success.append(fname)
             elif ta_stat == 'IN_PROGRESS':
diff --git a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
index f248a1d09..01e52a44c 100755
--- a/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
+++ b/jwql/instrument_monitors/nirspec_monitors/ta_monitors/wata_monitor.py
@@ -10,7 +10,7 @@
 
 
 """
-This module contains the code for the NIRSpec Wide Aperture Target
+This module contains the code for NIRSpec the Wide Aperture Target
 Acquisition (WATA) monitor, which monitors the TA offsets.
 
 This monitor displays the comparison of desired versus measured TA.
@@ -20,7 +20,6 @@
 Author
 ______
     - Maria Pena-Guerrero
-    - Melanie Clarke
 
 Use
 ---
@@ -35,22 +34,21 @@
 import os
 import logging
 import shutil
-from datetime import datetime, timezone, timedelta
 
 import numpy as np
 import pandas as pd
+from bs4 import BeautifulSoup
+from datetime import datetime
 from astropy.time import Time
 from astropy.io import fits
-from bokeh.embed import components
-from bokeh.io import output_file
-from bokeh.layouts import gridplot, layout
-from bokeh.models import (
-    ColumnDataSource, Range1d, CustomJS, CustomJSFilter, CDSView,
-    Span, Label, DateRangeSlider)
-from bokeh.models.tools import HoverTool, BoxSelectTool
-from bokeh.plotting import figure, save
-from bs4 import BeautifulSoup
 from sqlalchemy.sql.expression import and_
+from bokeh.io import output_file
+from bokeh.plotting import figure, show, save
+from bokeh.models import ColumnDataSource, Range1d
+from bokeh.models.tools import HoverTool
+from bokeh.layouts import gridplot
+from bokeh.models import Span, Label
+from bokeh.embed import components
 
 # jwql imports
 from jwql.utils.logging_functions import log_info, log_fail
@@ -62,7 +60,7 @@
 
 
 class WATA():
-    """ Class for executing the NIRSpec WATA monitor.
+    """ Class for executint the NIRSpec WATA monitor.
 
     This class will search for new WATA current files in the file systems
     for NIRSpec and will run the monitor on these files. The monitor will
@@ -132,12 +130,6 @@ def __init__(self):
                                'SAM_X': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_x'},
                                'SAM_Y': {'loc': 'ta_hdr', 'alt_key': None, 'name': 'sam_y'}}
 
-        # initialize attributes to be set later
-        self.source = None
-        self.share_tools = []
-        self.date_range = None
-        self.date_view = None
-
     def get_tainfo_from_fits(self, fits_file):
         """ Get the TA information from the fits file
         Parameters
@@ -208,19 +200,6 @@ def get_wata_data(self, new_filenames):
         wata_df = pd.DataFrame(wata_dict)
         return wata_df, no_ta_ext_msgs
 
-    def add_time_column(self):
-        """Add time column to data source, to be used by all plots."""
-        date_obs = self.source.data['date_obs']
-        if 'time_arr' not in self.source.data:
-            time_arr = []
-            for do_str in date_obs:
-                # convert time string into an array of time (this is in UT)
-                t = datetime.fromisoformat(do_str)
-                time_arr.append(t)
-
-            # add to the bokeh data structure
-            self.source.data["time_arr"] = time_arr
-
     def plt_status(self):
         """ Plot the WATA status (passed = 0 or failed = 1).
         Parameters
@@ -230,32 +209,36 @@ def plt_status(self):
         -------
             plot: bokeh plot object
         """
-        ta_status = self.source.data['ta_status']
-
-        # check if this column exists in the data already, else create it
-        if 'bool_status' not in self.source.data:
-            # bokeh does not like to plot strings, turn into binary type
-            bool_status, status_colors = [], []
-            for tas in ta_status:
+        ta_status, date_obs = self.source.data['ta_status'], self.source.data['date_obs']
+        # check if this column exists in the data already (the other 2 will exist too), else create it
+        try:
+            time_arr = self.source.data['time_arr']
+            bool_status = self.source.data['bool_status']
+            status_colors = self.source.data['status_colors']
+        except KeyError:
+            # bokeh does not like to plot strings, turn  into binary type
+            bool_status, time_arr, status_colors = [], [], []
+            for tas, do_str in zip(ta_status, date_obs):
                 if 'unsuccessful' not in tas.lower():
                     bool_status.append(1)
                     status_colors.append('blue')
                 else:
                     bool_status.append(0)
                     status_colors.append('red')
-
+                # convert time string into an array of time (this is in UT)
+                t = datetime.fromisoformat(do_str)
+                time_arr.append(t)
             # add these to the bokeh data structure
+            self.source.data["time_arr"] = time_arr
             self.source.data["ta_status_bool"] = bool_status
             self.source.data["status_colors"] = status_colors
-
         # create a new bokeh plot
-        plot = figure(title="WATA Status [Success=1, Fail=0]", x_axis_label='Time',
+        plot = figure(title="WATA Status [Succes=1, Fail=0]", x_axis_label='Time',
                       y_axis_label='WATA Status', x_axis_type='datetime',)
         plot.y_range = Range1d(-0.5, 1.5)
         plot.circle(x='time_arr', y='ta_status_bool', source=self.source,
-                    color='status_colors', size=7, fill_alpha=0.3, view=self.date_view)
-
-        # make tooltips
+                    color='status_colors', size=7, fill_alpha=0.3)
+        # output_file("wata_status.html")
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -267,10 +250,6 @@ def plt_status(self):
                           ('--------', '----------------')]
 
         plot.add_tools(hover)
-
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
         return plot
 
     def plt_residual_offsets(self):
@@ -286,16 +265,13 @@ def plt_residual_offsets(self):
         plot = figure(title="WATA Residual V2-V3 Offsets", x_axis_label='Residual V2 Offset',
                       y_axis_label='Residual V3 Offset')
         plot.circle(x='v2_offset', y='v3_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.x_range = Range1d(-0.5, 0.5)
         plot.y_range = Range1d(-0.5, 0.5)
-
         # mark origin lines
         vline = Span(location=0, dimension='height', line_color='black', line_width=0.7)
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([vline, hline])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -305,11 +281,8 @@ def plt_residual_offsets(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_v2offset_time(self):
@@ -325,14 +298,11 @@ def plt_v2offset_time(self):
         plot = figure(title="WATA V2 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Residual V2 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='v2_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.y_range = Range1d(-0.5, 0.5)
-
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -342,11 +312,8 @@ def plt_v2offset_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_v3offset_time(self):
@@ -362,14 +329,11 @@ def plt_v3offset_time(self):
         plot = figure(title="WATA V3 Offset vs Time", x_axis_label='Time',
                       y_axis_label='Residual V3 Offset', x_axis_type='datetime')
         plot.circle(x='time_arr', y='v3_offset', source=self.source,
-                    color="blue", size=7, fill_alpha=0.3, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.3)
         plot.y_range = Range1d(-0.5, 0.5)
-
         # mark origin line
         hline = Span(location=0, dimension='width', line_color='black', line_width=0.7)
         plot.renderers.extend([hline])
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -379,11 +343,8 @@ def plt_v3offset_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Magnitude', '@star_mag'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
     def plt_mag_time(self):
@@ -397,9 +358,15 @@ def plt_mag_time(self):
         """
         # calculate the pseudo magnitudes
         max_val_box, time_arr = self.source.data['max_val_box'], self.source.data['time_arr']
-
-        # check if this column exists in the data already, else create it
-        if "nrsrapid_f140x" not in self.source.data:
+        # check if this column exists in the data already (the other 2 will exist too), else create it
+        try:
+            nrsrapid_f140x = self.source.data["nrsrapid_f140x"]
+            nrsrapid_f110w = self.source.data["nrsrapid_f110w"]
+            nrsrapid_clear = self.source.data["nrsrapid_clear"]
+            nrsrapidd6_f140x = self.source.data["nrsrapidd6_f140x"]
+            nrsrapidd6_f110w = self.source.data["nrsrapidd6_f110w"]
+            nrsrapidd6_clear = self.source.data["nrsrapidd6_clear"]
+        except KeyError:
             # create the arrays per filter and readout pattern
             nrsrapid_f140x, nrsrapid_f110w, nrsrapid_clear = [], [], []
             nrsrapidd6_f140x, nrsrapidd6_f110w, nrsrapidd6_clear = [], [], []
@@ -450,7 +417,6 @@ def plt_mag_time(self):
                         nrsrapidd6_f140x.append(np.NaN)
                         nrsrapidd6_f110w.append(np.NaN)
                         nrsrapidd6_clear.append(val)
-
             # add to the bokeh data structure
             self.source.data["nrsrapid_f140x"] = nrsrapid_f140x
             self.source.data["nrsrapid_f110w"] = nrsrapid_f110w
@@ -458,30 +424,27 @@ def plt_mag_time(self):
             self.source.data["nrsrapidd6_f140x"] = nrsrapidd6_f140x
             self.source.data["nrsrapidd6_f110w"] = nrsrapidd6_f110w
             self.source.data["nrsrapidd6_clear"] = nrsrapidd6_clear
-
         # create a new bokeh plot
         plot = figure(title="WATA Counts vs Time", x_axis_label='Time',
                       y_axis_label='box_peak [Counts]', x_axis_type='datetime')
         plot.circle(x='time_arr', y='nrsrapid_f140x', source=self.source,
-                    color="purple", size=7, fill_alpha=0.4, view=self.date_view)
+                    color="purple", size=7, fill_alpha=0.4)
         plot.circle(x='time_arr', y='nrsrapidd6_f140x', source=self.source,
-                    color="purple", size=12, fill_alpha=0.4, view=self.date_view)
+                    color="purple", size=12, fill_alpha=0.4)
         plot.triangle(x='time_arr', y='nrsrapid_f110w', source=self.source,
-                      color="orange", size=8, fill_alpha=0.4, view=self.date_view)
+                      color="orange", size=8, fill_alpha=0.4)
         plot.triangle(x='time_arr', y='nrsrapidd6_f110w', source=self.source,
-                      color="orange", size=13, fill_alpha=0.4, view=self.date_view)
+                      color="orange", size=13, fill_alpha=0.4)
         plot.square(x='time_arr', y='nrsrapid_clear', source=self.source,
-                    color="gray", size=7, fill_alpha=0.4, view=self.date_view)
+                    color="gray", size=7, fill_alpha=0.4)
         plot.square(x='time_arr', y='nrsrapidd6_clear', source=self.source,
-                    color="gray", size=12, fill_alpha=0.4, view=self.date_view)
-
+                    color="gray", size=12, fill_alpha=0.4)
         # add count saturation warning lines
         loc1, loc2, loc3 = 45000.0, 50000.0, 60000.0
         hline1 = Span(location=loc1, dimension='width', line_color='green', line_width=3)
         hline2 = Span(location=loc2, dimension='width', line_color='yellow', line_width=3)
         hline3 = Span(location=loc3, dimension='width', line_color='red', line_width=3)
         plot.renderers.extend([hline1, hline2, hline3])
-
         label1 = Label(x=time_arr[-1], y=loc1, y_units='data', text='45000 counts')
         label2 = Label(x=time_arr[-1], y=loc2, y_units='data', text='50000 counts')
         label3 = Label(x=time_arr[-1], y=loc3, y_units='data', text='60000 counts')
@@ -489,8 +452,7 @@ def plt_mag_time(self):
         plot.add_layout(label2)
         plot.add_layout(label3)
         plot.y_range = Range1d(-1000.0, 62000.0)
-
-        # add tooltips
+        # add hover
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -500,15 +462,12 @@ def plt_mag_time(self):
                           ('Date-Obs', '@date_obs'),
                           ('Box peak', '@max_val_box'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
-    def get_unsuccessful_ta(self, arr_name):
-        """ Find unsuccessful TAs in this set (to be plotted in red)
+    def get_unsucessful_ta(self, arr_name):
+        """ Find unsucessful TAs in this set (to be plotted in red)
         Parameters
         ----------
             arr_name: str, name of the array of interest
@@ -538,16 +497,16 @@ def plt_centroid(self):
             plot: bokeh plot object
         """
         # get the failed TAs to plot in red
-        if "corr_col_failed" not in self.source.data:
-            corr_col_failed, corr_col_not_failed = self.get_unsuccessful_ta('corr_col')
-            corr_row_failed, corr_row_not_failed = self.get_unsuccessful_ta('corr_row')
-
+        try:
+            corr_col_failed = self.source.data["corr_col_failed"]
+        except KeyError:
+            corr_col_failed, corr_col_not_failed = self.get_unsucessful_ta('corr_col')
+            corr_row_failed, corr_row_not_failed = self.get_unsucessful_ta('corr_row')
             # add these to the bokeh data structure
             self.source.data["corr_col_failed"] = corr_col_failed
             self.source.data["corr_col_not_failed"] = corr_col_not_failed
             self.source.data["corr_row_failed"] = corr_row_failed
             self.source.data["corr_row_not_failed"] = corr_row_not_failed
-
         # create a new bokeh plot
         plot = figure(title="WATA Centroid", x_axis_label='Column',
                       y_axis_label='Row')
@@ -555,13 +514,11 @@ def plt_centroid(self):
         plot.x_range = Range1d(limits[0], limits[1])
         plot.y_range = Range1d(limits[0], limits[1])
         plot.circle(x='corr_col_not_failed', y='corr_row_not_failed', source=self.source,
-                    color="blue", size=7, fill_alpha=0.5, view=self.date_view)
+                    color="blue", size=7, fill_alpha=0.5)
         plot.circle(x='corr_col_failed', y='corr_row_failed', source=self.source,
-                    color="red", size=7, fill_alpha=0.5, view=self.date_view)
+                    color="red", size=7, fill_alpha=0.5)
         plot.x_range = Range1d(0.0, 32.0)
         plot.y_range = Range1d(0.0, 32.0)
-
-        # add tooltips
         hover = HoverTool()
         hover.tooltips = [('File name', '@filename'),
                           ('Visit ID', '@visit_id'),
@@ -575,72 +532,18 @@ def plt_centroid(self):
                           ('Det Centr Col', '@detector_final_col'),
                           ('Det Centr Row', '@detector_final_row'),
                           ('--------', '----------------')]
-        plot.add_tools(hover)
 
-        # add shared selection tools
-        for tool in self.share_tools:
-            plot.add_tools(tool)
+        plot.add_tools(hover)
         return plot
 
-    def setup_date_range(self):
-        """Set up a date range filter, defaulting to the last week of data."""
-        end_date = datetime.now(tz=timezone.utc)
-        one_week_ago = end_date.date() - timedelta(days=7)
-        first_data_point = np.min(self.source.data['time_arr']).date()
-        last_data_point = np.max(self.source.data['time_arr']).date()
-        if last_data_point < one_week_ago:
-            # keep at least one point in the plot if there was
-            # no TA data this week
-            start_date = last_data_point
-        else:
-            start_date = one_week_ago
-
-        # allowed range is from the first ever data point to today
-        self.date_range = DateRangeSlider(
-            title="Date range displayed", start=first_data_point,
-            end=end_date, value=(start_date, end_date), step=1)
-
-        callback = CustomJS(args=dict(s=self.source), code="""
-            s.change.emit();
-        """)
-        self.date_range.js_on_change('value', callback)
-
-        filt = CustomJSFilter(args=dict(slider=self.date_range), code="""
-                var indices = [];
-                var start = slider.value[0];
-                var end = slider.value[1];
-
-                for (var i=0; i < source.get_length(); i++) {
-                    if (source.data['time_arr'][i] >= start 
-                            && source.data['time_arr'][i] <= end) {
-                        indices.push(true);
-                    } else {
-                        indices.push(false);
-                    }
-                }
-                return indices;
-                """)
-        self.date_view = CDSView(source=self.source, filters=[filt])
-
     def mk_plt_layout(self):
         """Create the bokeh plot layout"""
         self.source = ColumnDataSource(data=self.wata_data)
-
         # make sure all arrays are lists in order to later be able to read the data
         # from the html file
         for item in self.source.data:
             if not isinstance(self.source.data[item], (str, float, int, list)):
                 self.source.data[item] = self.source.data[item].tolist()
-
-        # add a time array to the data source
-        self.add_time_column()
-
-        # set up selection tools to share
-        self.share_tools = [BoxSelectTool()]
-
-        # set up a date range filter widget
-        self.setup_date_range()
-
         # set the output html file name and create the plot grid
         output_file(self.output_file_name)
         p1 = self.plt_status()
@@ -649,14 +552,11 @@ def mk_plt_layout(self):
         p4 = self.plt_v3offset_time()
         p5 = self.plt_centroid()
         p6 = self.plt_mag_time()
-
         # make grid
         grid = gridplot([p1, p2, p3, p4, p5, p6], ncols=2, merge_tools=False)
-        box_layout = layout(children=[self.date_range, grid])
-        save(box_layout)
-
+        save(grid)
         # return the needed components for embeding the results in the WATA html template
-        script, div = components(box_layout)
+        script, div = components(grid)
         return script, div
 
     def identify_tables(self):
@@ -829,15 +729,15 @@ def update_ta_success_txtfile(self):
             Nothing
         """
         output_success_ta_txtfile = os.path.join(self.output_dir, "wata_success.txt")
-        # check if previous file exists and read the data from it
+        # check if previous file exsists and read the data from it
         if os.path.isfile(output_success_ta_txtfile):
-            # now rename the previous file, for backup
+            # now rename the the previous file, for backup
             os.rename(output_success_ta_txtfile, os.path.join(self.output_dir, "prev_wata_success.txt"))
         # get the new data
         ta_success, ta_failure = [], []
         filenames, ta_status = self.wata_data.loc[:,'filename'], self.wata_data.loc[:,'ta_status']
         for fname, ta_stat in zip(filenames, ta_status):
-            # select the appropriate list to append to
+            # select the appriopriate list to append to
             if ta_stat == 'SUCCESSFUL':
                 ta_success.append(fname)
             else:

From 7efae1e0981760a25f5de7fd9a396076ee8e617a Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 9 Jun 2023 17:38:52 -0400
Subject: [PATCH 145/256] Add row and column plots

---
 jwql/utils/interactive_preview_image.py       | 142 ++++++++++++++++--
 jwql/website/apps/jwql/static/js/jwql.js      |   7 +-
 .../apps/jwql/templates/explore_image.html    |  19 ++-
 jwql/website/apps/jwql/urls.py                |   2 +-
 jwql/website/apps/jwql/views.py               |  13 +-
 5 files changed, 156 insertions(+), 27 deletions(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index d0f1c8cea..0c4480882 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -16,6 +16,7 @@
         Required Arguments:
         ''filename'' - Name of a fits file containing a JWST observation
     """
+import datetime
 from copy import deepcopy
 import os
 
@@ -23,12 +24,13 @@
 from astropy.visualization import ZScaleInterval, MinMaxInterval, PercentileInterval
 import numpy as np
 from bokeh.embed import components
-from bokeh.layouts import layout
+from bokeh.layouts import gridplot, layout
 from bokeh.models import (
-    BasicTicker, BoxZoomTool, Button, ColorBar, CustomJS, Div, HoverTool,
-    LinearColorMapper, LogColorMapper, LogTicker, RadioGroup, Row, Select, Spacer,
-    Spinner, WheelZoomTool)
+    BasicTicker, BoxZoomTool, Button, ColorBar, ColumnDataSource,
+    CustomJS, CustomJSTransform, DataRange1d, Div, HoverTool, LinearColorMapper, LogColorMapper,
+    LogTicker, RadioGroup, Range1d, Row, Select, Spacer, Spinner, WheelZoomTool)
 from bokeh.plotting import figure, output_file, show, save
+from bokeh.transform import transform
 
 from jwst.datamodels import dqflags
 
@@ -37,8 +39,8 @@ class InteractivePreviewImg:
     """Class to create the interactive Bokeh figure.
     """
 
-    def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contrast=None, extname='SCI',
-                 group=None, integ=None, mask=None, save_html=None, show=False):
+    def __init__(self, filename, low_lim=None, high_lim=None, scaling='lin', contrast=None, extname='SCI',
+                 group=None, integ=None, mask=None, line_plots=False, save_html=None, show=False):
         """Populate attributes, read in data, and create the Bokeh figure
         Parameters
         ----------
@@ -68,6 +70,9 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contras
         mask : numpy.ndarray
             Mask to use in order to avoid some pixels when auto-scaling. Pixels with a value other than 0 will
             be ignored when auto-scaling.
+        line_plots : bool
+            If set, column and row plots are added to the layout, to be updated on click in the main figure.
+            These take some time to create, so are off by default.
         save_html : str
             Name of html file to save the figure to. If None, the components are returned instead.
         show : bool
@@ -80,6 +85,7 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contras
         self.contrast = contrast
         self.extname = extname.upper()
         self.mask = mask
+        self.show_line_plots = line_plots
         self.show = show
         self.save_html = save_html
 
@@ -105,6 +111,8 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='log', contras
         self.get_data()
         if 'DQ' in self.extname:
             self.get_bits()
+            # col/row plots not available for dq values
+            self.show_line_plots = False
         self.script, self.div = self.create_bokeh_image()
 
     def create_bokeh_image(self):
@@ -140,16 +148,17 @@ def create_bokeh_image(self):
         # fix figure aspect from data aspect
         # bokeh throws errors if plot is too small, so make sure
         # the smaller dimension has reasonable size
+        max_dim, min_dim = 700, 300
         if xd > yd:
-            plot_width = 800
+            plot_width = max_dim
             plot_height = int(plot_width * yd / xd)
-            if plot_height < 400:
-                plot_height = 400
+            if plot_height < min_dim:
+                plot_height = min_dim
         else:
-            plot_height = 800
+            plot_height = max_dim
             plot_width = int(plot_height * xd / yd)
-            if plot_width < 400:
-                plot_width = 400
+            if plot_width < min_dim:
+                plot_width = min_dim
 
         fig = figure(tools='pan,reset,save', match_aspect=True,
                      plot_width=plot_width, plot_height=plot_height)
@@ -169,7 +178,10 @@ def create_bokeh_image(self):
                                  title=self.signal_units, bar_line_color='black',
                                  minor_tick_line_color='black', major_tick_line_color='black',
                                  visible=visible)
-            fig.add_layout(color_bar, 'below')
+            if self.show_line_plots:
+                fig.add_layout(color_bar, 'above')
+            else:
+                fig.add_layout(color_bar, 'below')
             images.append(img)
             color_bars.append(color_bar)
 
@@ -196,11 +208,19 @@ def create_bokeh_image(self):
         fig.title.text = self.title
         fig.xaxis.axis_label = 'Pixel'
         fig.yaxis.axis_label = 'Pixel'
-        fig.tools.append(hover_tool)
+        fig.add_tools(hover_tool)
 
         # add interactive widgets
         widgets = self.add_interactive_controls(images, color_bars)
-        box_layout = layout(children=[fig, *widgets])
+        if self.show_line_plots:
+            # add row and column plots
+            col_plot, row_plot = self.line_plots(fig)
+            grid = gridplot([fig, col_plot, row_plot],
+                            ncols=2, merge_tools=False)
+        else:
+            grid = fig
+
+        box_layout = layout(children=[grid, *widgets])
 
         # Show figure on screen if requested
         if self.show:
@@ -210,6 +230,98 @@ def create_bokeh_image(self):
         else:
             return components(box_layout)
 
+    def line_plots(self, main_figure):
+        new_plots = []
+        new_lines = []
+        ny, nx = self.data.shape
+        col_idx, row_idx = np.indices((ny, nx))
+
+        for direction in ['y', 'x']:
+            if direction == 'y':
+                # column plots
+                fig = figure(plot_width=200, plot_height=main_figure.height,
+                             tools='xwheel_zoom, xpan, reset',
+                             y_axis_location='right')
+
+                fig.x_range = DataRange1d(only_visible=True)
+                fig.y_range = Range1d()
+                match_range = fig.y_range
+                main_range = main_figure.y_range
+
+                x_plot = 'data'
+                y_plot = 'index'
+                n_plot = nx
+                data_source = {'data': self.data, 'index': col_idx}
+                source = ColumnDataSource(data_source)
+            else:
+                # row plots
+                fig = figure(plot_height=200, plot_width=main_figure.width,
+                             tools='ywheel_zoom, ypan, reset')
+
+                fig.y_range = DataRange1d(only_visible=True)
+                fig.x_range = Range1d()
+                match_range = fig.x_range
+                main_range = main_figure.x_range
+
+                x_plot = 'index'
+                y_plot = 'data'
+                n_plot = ny
+                # indexing is off by 1 for row plots for some reason
+                data_source = {'data': self.data.T, 'index': row_idx.T + 1}
+                source = ColumnDataSource(data_source)
+
+            # match one of the axes to the main figure
+            if main_range.start is not None:
+                match_range.start = main_range.start
+            if main_range.end is not None:
+                match_range.end = main_range.end
+            main_range.js_link('start', match_range, 'start')
+            main_range.js_link('start', match_range, 'reset_start')
+            main_range.js_link('end', match_range, 'end')
+            main_range.js_link('end', match_range, 'reset_end')
+
+            lines = []
+            initial_visible = n_plot // 2
+            for i in range(n_plot):
+                idx_transform = CustomJSTransform(
+                    args={'idx': i},
+                    v_func="""
+                        const x_val = new Float64Array(xs.length);
+                        for (let i = 0; i < xs.length; i++) {
+                            x_val[i] = xs[i][idx];
+                        }
+                        return x_val;
+                    """)
+                line = fig.step(x=transform(x_plot, idx_transform),
+                                y=transform(y_plot, idx_transform),
+                                mode='before', source=source,
+                                visible=(i == initial_visible))
+                lines.append(line)
+            new_lines.append(lines)
+            new_plots.append(fig)
+
+        update_plot = CustomJS(args={'lines': new_lines}, code="""
+            var x = Math.floor(cb_obj.x);
+            var y = Math.floor(cb_obj.y);
+            for (let i=0; i < lines[0].length; i++) {
+                if (i == x) {
+                    lines[0][i].visible = true;
+                } else {
+                    lines[0][i].visible = false;
+                }
+            }
+            for (let j=0; j < lines[1].length; j++) {
+                if (j == y) {
+                    lines[1][j].visible = true;
+                } else {
+                    lines[1][j].visible = false;
+                }
+            }
+        """)
+        main_figure.js_on_event('tap', update_plot)
+
+        return new_plots
+
     def add_interactive_controls(self, images, color_bars):
         """
         Add client-side controls for images.
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 10cbe7f23..273057412 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -423,7 +423,7 @@ function get_radio_button_value(element_name) {
 }
 
 /**
- * get_scaling_value
+ * Get value from a numerical text field
  * @param {String} element_id - The element id
  * @returns value - value of element id or "None" if empty or not a number
 */
@@ -1058,7 +1058,7 @@ function update_wata_page(base_url) {
 /**
  * Updates various components on the thumbnails page
  * @param {String} inst - The instrument of interest (e.g. "FGS")
- * @param {String} file_root - The rootname of the file forresponding tot he instrument (e.g. "JW01473015001_04101_00001_MIRIMAGE")
+ * @param {String} file_root - The rootname of the file corresponding to the instrument (e.g. "JW01473015001_04101_00001_MIRIMAGE")
  * @param {String} filetype - The type to be viewed (e.g. "cal" or "rate").
  * @param {String} base_url - The base URL for gathering data from the AJAX view.
  * @param {Boolean} do_opt_args - Flag to calculate and send optional arguments in URL
@@ -1073,6 +1073,7 @@ function update_wata_page(base_url) {
         document.getElementById("explore_image").style.display = "none";
         document.getElementById("explore_image_fail").style.display = "none";
         var calc_difference = document.getElementById("calcDifference").checked;
+        var show_line_plots = document.getElementById("show_line_plots").checked;
 
         // Get the arguments to update
         var ext_name = get_radio_button_value("extension");
@@ -1087,7 +1088,7 @@ function update_wata_page(base_url) {
             int2_nr="None";
             grp2_nr="None";
         }
-        optional_params = optional_params + "/ext_" + ext_name + "/int1_" + int1_nr + "/grp1_" + grp1_nr + "/int2_" + int2_nr + "/grp2_" + grp2_nr;
+        optional_params = optional_params + "/plot_" + show_line_plots + "/ext_" + ext_name + "/int1_" + int1_nr + "/grp1_" + grp1_nr + "/int2_" + int2_nr + "/grp2_" + grp2_nr;
     }
 
     $.ajax({
diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html
index f72a92081..af750a8c8 100644
--- a/jwql/website/apps/jwql/templates/explore_image.html
+++ b/jwql/website/apps/jwql/templates/explore_image.html
@@ -44,7 +44,13 @@ <h3>{{ file_root }}_{{ filetype }}.fits</h3>
 
                 <!-- Allow the user to change the settings in the image to explore -->
                 <h4>Data Settings</h4>
-                <div class="mt-4">
+                <div class="col">
+                    <div class="row">
+                        <input class="mr-2" type="checkbox" id="show_line_plots">
+                        <label class="form-check-label" for="show_line_plots">Show column/row plots</label>
+                    </div>
+
+                    <div class="row">
                     Extension:&nbsp;&nbsp;
                     {% for extension in extensions %}
                         {% if extension == 'SCI' %}
@@ -53,6 +59,8 @@ <h4>Data Settings</h4>
                             <input type="radio" name="extension" value="{{ extension }}" id="{{ extension }}" onclick='explore_image_update_enable_options("{{ extension_ints }}", "{{ extension_groups }}");'> {{ extension }} &nbsp;&nbsp;&nbsp;&nbsp;
                         {% endif %}
                     {% endfor %}
+                    </div>
+
                     <div class="row">
                         <div class="explorer_options" id="integrationInput1">
                             <input type="number" id="integration1" min="0" size="3" step="1">
@@ -83,8 +91,11 @@ <h4>Data Settings</h4>
                 </p>
             </div>
 
-            <!-- Display the anomaly form -->
-            <div class="col-xl-3 text-left">
+
+        </div>
+
+        <!-- Display the anomaly form -->
+        <div class="col-xl-3 text-left">
                 <!--Load the file search form from the view-->
                 <div class="anomaly_form">
                     <h5>Submit Anomaly</h5>
@@ -115,8 +126,6 @@ <h5>Submit Anomaly</h5>
                     </form>
                 </div>
             </div>
-            <!-- Display the anomaly form above -->
-        </div>
 
     </main>
     <script>explore_image_update_enable_options('{{ extension_ints }}', '{{ extension_groups }}');</script>
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index c0535b378..0b4420080 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -96,7 +96,7 @@
     re_path('ajax/query_submit/', views.archive_thumbnails_query_ajax, name='archive_thumb_query_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/archive/$'.format(instruments), views.archived_proposals_ajax, name='archive_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'),
-    re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/ext_(?P<ext_name>.+)/int1_(?P<int1_nr>.+)/grp1_(?P<grp1_nr>.+)/int2_(?P<int2_nr>.+)/grp2_(?P<grp2_nr>.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'),
+    re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/plot_(?P<line_plots>(true|false))/ext_(?P<ext_name>.+)/int1_(?P<int1_nr>.+)/grp1_(?P<grp1_nr>.+)/int2_(?P<int2_nr>.+)/grp2_(?P<grp2_nr>.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/archive/(?P<proposal>[\d]{{1,5}})/obs(?P<observation>[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_ajax, name='archive_thumb_ajax'),
     re_path(r'^ajax/viewed/(?P<file_root>.+)/$', views.toggle_viewed_ajax, name='toggle_viewed_ajax'),
     re_path(r'^ajax/viewed_group/(?P<group_root>.+)/(?P<status>(viewed|new|Viewed|New))/$',
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index b0240735f..af4beb1b0 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -994,7 +994,7 @@ def explore_image(request, inst, file_root, filetype):
     return render(request, template, context)
 
 
-def explore_image_ajax(request, inst, file_root, filetype, scaling="lin", low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None):
+def explore_image_ajax(request, inst, file_root, filetype, line_plots='false', low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None):
     """Generate the page listing all archived images in the database
     for a certain proposal
 
@@ -1008,7 +1008,7 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="lin", low_li
         FITS file_root of selected image in filesystem
     filetype : str
         Type of file (e.g. ``uncal``)
-    scaling : str
+    line_plots : str
         Scaling to implement in interactive preview image ("log" or "lin")
     low_lim : str
         Signal value to use as the lower limit of the displayed image. If "None", it will be calculated using the ZScale function
@@ -1067,7 +1067,14 @@ def explore_image_ajax(request, inst, file_root, filetype, scaling="lin", low_li
         else:
             integ = int(int1_nr)
 
-    int_preview_image = InteractivePreviewImg(full_fits_file, low_lim, high_lim, scaling, None, ext_name, group, integ)
+    if str(line_plots).strip().lower() == 'true':
+        line_plots = True
+    else:
+        line_plots = False
+
+    int_preview_image = InteractivePreviewImg(
+        full_fits_file, low_lim=low_lim, high_lim=high_lim, extname=ext_name,
+        group=group, integ=integ, line_plots=line_plots)
 
     context = {'inst': "inst",
                'script': int_preview_image.script,

From 9938a5460253e3971da4806d6719f9069c5e4290 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Mon, 12 Jun 2023 16:41:59 -0400
Subject: [PATCH 146/256] Readability improvements

---
 jwql/utils/interactive_preview_image.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index 0c4480882..262fa2c07 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -148,7 +148,7 @@ def create_bokeh_image(self):
         # fix figure aspect from data aspect
         # bokeh throws errors if plot is too small, so make sure
         # the smaller dimension has reasonable size
-        max_dim, min_dim = 700, 300
+        max_dim, min_dim = 700, 400
         if xd > yd:
             plot_width = max_dim
             plot_height = int(plot_width * yd / xd)
@@ -231,6 +231,19 @@ def create_bokeh_image(self):
             return components(box_layout)
 
     def line_plots(self, main_figure):
+        """
+        Pre-compute column and row plots for each pixel.
+
+        Parameters
+        ----------
+        main_figure : figure
+            Main figure containing image.
+
+        Returns
+        -------
+        list of figure
+            New figures to add to the page layout.
+        """
         new_plots = []
         new_lines = []
         ny, nx = self.data.shape
@@ -242,12 +255,15 @@ def line_plots(self, main_figure):
                 fig = figure(plot_width=200, plot_height=main_figure.height,
                              tools='xwheel_zoom, xpan, reset',
                              y_axis_location='right')
-
                 fig.x_range = DataRange1d(only_visible=True)
                 fig.y_range = Range1d()
                 match_range = fig.y_range
                 main_range = main_figure.y_range
 
+                fig.xaxis.axis_label = self.signal_units
+                fig.yaxis.axis_label = 'Row pixel (y)'
+                fig.xaxis.major_label_orientation = np.radians(-45)
+
                 x_plot = 'data'
                 y_plot = 'index'
                 n_plot = nx
@@ -263,6 +279,9 @@ def line_plots(self, main_figure):
                 match_range = fig.x_range
                 main_range = main_figure.x_range
 
+                fig.xaxis.axis_label = 'Column pixel (x)'
+                fig.yaxis.axis_label = self.signal_units
+
                 x_plot = 'index'
                 y_plot = 'data'
                 n_plot = ny

From 6e370d49378cbdda45a748080320debc99768468 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 13 Jun 2023 13:03:53 -0400
Subject: [PATCH 147/256] Move hover tool to separate div

---
 jwql/utils/interactive_preview_image.py | 116 +++++++++++++++++-------
 1 file changed, 82 insertions(+), 34 deletions(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index 262fa2c07..a414d7e63 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -27,8 +27,9 @@
 from bokeh.layouts import gridplot, layout
 from bokeh.models import (
     BasicTicker, BoxZoomTool, Button, ColorBar, ColumnDataSource,
-    CustomJS, CustomJSTransform, DataRange1d, Div, HoverTool, LinearColorMapper, LogColorMapper,
-    LogTicker, RadioGroup, Range1d, Row, Select, Spacer, Spinner, WheelZoomTool)
+    CustomJS, CustomJSTransform, DataRange1d, Div, HoverTool,
+    LinearColorMapper, LogColorMapper, LogTicker, RadioGroup, Range1d, Row,
+    Select, Spacer, Spinner, WheelZoomTool)
 from bokeh.plotting import figure, output_file, show, save
 from bokeh.transform import transform
 
@@ -138,7 +139,7 @@ def create_bokeh_image(self):
         active = int(self.scaling == 'log')
 
         yd, xd = self.data.shape
-        info = dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd])
+        info = ColumnDataSource(dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd]))
         if 'DQ' in self.extname:
             info["dq"] = [self.bit_list]
         if not self.show and self.save_html is not None:
@@ -196,14 +197,8 @@ def create_bokeh_image(self):
             fig.y_range.end = yd
             fig.y_range.bounds = (0, yd)
 
-        if 'DQ' not in self.extname:
-            hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"),
-                                             ('Value', '@image{0.4f}')
-                                             ], mode='mouse', renderers=images)
-        else:
-            hover_tool = HoverTool(tooltips=[("(x,y)", "($x{0.2f}, $y{0.2f})"),
-                                             ('Value', '@dq')
-                                             ], mode='mouse', renderers=images)
+        hover_div, hover_tool = self.add_hover_tool(info, images)
+
         self.create_figure_title()
         fig.title.text = self.title
         fig.xaxis.axis_label = 'Pixel'
@@ -215,10 +210,10 @@ def create_bokeh_image(self):
         if self.show_line_plots:
             # add row and column plots
             col_plot, row_plot = self.line_plots(fig)
-            grid = gridplot([fig, col_plot, row_plot],
+            grid = gridplot([fig, col_plot, row_plot, hover_div],
                             ncols=2, merge_tools=False)
         else:
-            grid = fig
+            grid = gridplot([fig, hover_div], ncols=2, merge_tools=False)
 
         box_layout = layout(children=[grid, *widgets])
 
@@ -249,12 +244,12 @@ def line_plots(self, main_figure):
         ny, nx = self.data.shape
         col_idx, row_idx = np.indices((ny, nx))
 
-        for direction in ['y', 'x']:
-            if direction == 'y':
+        for index_direction in ['x', 'y']:
+            if index_direction == 'x':
                 # column plots
                 fig = figure(plot_width=200, plot_height=main_figure.height,
                              tools='xwheel_zoom, xpan, reset',
-                             y_axis_location='right')
+                             y_axis_location='right', margin=(0, 0, 0, 30))
                 fig.x_range = DataRange1d(only_visible=True)
                 fig.y_range = Range1d()
                 match_range = fig.y_range
@@ -314,33 +309,86 @@ def line_plots(self, main_figure):
                 line = fig.step(x=transform(x_plot, idx_transform),
                                 y=transform(y_plot, idx_transform),
                                 mode='before', source=source,
-                                visible=(i == initial_visible))
+                                visible=(i == initial_visible),
+                                name=f'Data at {index_direction}={i}')
                 lines.append(line)
+            fig.title = lines[initial_visible].name
             new_lines.append(lines)
             new_plots.append(fig)
 
-        update_plot = CustomJS(args={'lines': new_lines}, code="""
-            var x = Math.floor(cb_obj.x);
-            var y = Math.floor(cb_obj.y);
-            for (let i=0; i < lines[0].length; i++) {
-                if (i == x) {
-                    lines[0][i].visible = true;
-                } else {
-                    lines[0][i].visible = false;
-                }
-            }
-            for (let j=0; j < lines[1].length; j++) {
-                if (j == y) {
-                    lines[1][j].visible = true;
-                } else {
-                    lines[1][j].visible = false;
+        update_plot = CustomJS(
+            args={'lines': new_lines, 'figures': new_plots},
+            code="""
+                var x = Math.floor(cb_obj.x);
+                var y = Math.floor(cb_obj.y);
+                figures[0].title.text = "";
+                for (let i=0; i < lines[0].length; i++) {
+                    if (i == x) {
+                        lines[0][i].visible = true;
+                        figures[0].title.text = lines[0][i].name;
+                    } else {
+                        lines[0][i].visible = false;
+                    }
                 }
-            }
-        """)
+                figures[1].title.text = "";
+                for (let j=0; j < lines[1].length; j++) {
+                    if (j == y) {
+                        lines[1][j].visible = true;
+                        figures[1].title.text = lines[1][j].name;
+                    } else {
+                        lines[1][j].visible = false;
+                    }
+                }""")
         main_figure.js_on_event('tap', update_plot)
 
         return new_plots
 
+    def add_hover_tool(self, info, images):
+        """
+        Make a hover tool with a div to display text.
+
+        Parameters
+        ----------
+        info : bokeh.models.ColumnDataSource
+            Data source for the figure.
+        images : list of bokeh.models.GlyphRenderer
+            Images to use as renderers for the hover tool.
+
+        Returns
+        -------
+        hover_div : bokeh.models.Div
+            Div element that will contain text from hover tool.
+        hover_tool : bokeh.models.
+        """
+        hover_div = Div(height=200, width=200)
+        hover_callback = CustomJS(args={'s': info, 'd': hover_div}, code="""
+            const idx = cb_data.index.image_indices;
+            if (idx.length > 0) { 
+                var x = idx[0].dim1;
+                var y = idx[0].dim2;
+                var val = s.data['image'][0][y][x];
+                val = val.toPrecision(5);
+                d.text = "<div style='margin:20px'><h5>Pixel Value</h5>" + 
+                         "<div style='display:table; border-spacing: 2px'>" + 
+                         "<div style='display:table-row'>" + 
+                         "<div style='display:table-cell; text-align:right'>(x, y) =</div>" +
+                         "<div style='display:table-cell'>(" + x + ", " + y + ")</div>" +
+                         "</div>" +
+                         "<div style='display:table-row'>" + 
+                         "<div style='display:table-cell; text-align:right'>Value =</div>" +
+                         "<div style='display:table-cell'>" + val + "</div></div></div></div>";
+            } else {
+                d.text = "";
+            }
+        """)
+        if 'DQ' not in self.extname:
+            hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images,
+                                   callback=hover_callback)
+        else:
+            hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images,
+                                   callback=hover_callback)
+        return hover_div, hover_tool
+
     def add_interactive_controls(self, images, color_bars):
         """
         Add client-side controls for images.

From 4445dde8f786979d4b7eb6e172a81ace8a2aaf3d Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 13 Jun 2023 14:31:40 -0400
Subject: [PATCH 148/256] Add RA/Dec to hover tool if available

---
 jwql/utils/interactive_preview_image.py       | 63 ++++++++++++++-----
 .../apps/jwql/templates/explore_image.html    |  2 +-
 2 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index a414d7e63..615176cd2 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -16,13 +16,13 @@
         Required Arguments:
         ''filename'' - Name of a fits file containing a JWST observation
     """
-import datetime
 from copy import deepcopy
 import os
 
+import numpy as np
 from astropy.io import fits
 from astropy.visualization import ZScaleInterval, MinMaxInterval, PercentileInterval
-import numpy as np
+from astropy.wcs import WCS
 from bokeh.embed import components
 from bokeh.layouts import gridplot, layout
 from bokeh.models import (
@@ -32,7 +32,6 @@
     Select, Spacer, Spinner, WheelZoomTool)
 from bokeh.plotting import figure, output_file, show, save
 from bokeh.transform import transform
-
 from jwst.datamodels import dqflags
 
 
@@ -109,6 +108,10 @@ def __init__(self, filename, low_lim=None, high_lim=None, scaling='lin', contras
                 raise ValueError(
                     'integ must be an integer or 2-element list')
         self.integ = integ
+
+        self.data = None
+        self.signal_units = None
+        self.wcs_coord = None
         self.get_data()
         if 'DQ' in self.extname:
             self.get_bits()
@@ -139,9 +142,14 @@ def create_bokeh_image(self):
         active = int(self.scaling == 'log')
 
         yd, xd = self.data.shape
-        info = ColumnDataSource(dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd]))
+        info = dict(image=[self.data], x=[0], y=[0], dw=[xd], dh=[yd])
         if 'DQ' in self.extname:
             info["dq"] = [self.bit_list]
+        if self.wcs_coord is not None and len(self.wcs_coord) == 2:
+            info["ra"] = [self.wcs_coord[0]]
+            info["dec"] = [self.wcs_coord[1]]
+        source = ColumnDataSource(info)
+
         if not self.show and self.save_html is not None:
             output_file(filename=self.save_html,
                         title=os.path.basename(self.filename))
@@ -173,7 +181,7 @@ def create_bokeh_image(self):
         for i, config in enumerate(scales):
             color_mapper, ticker = config
             visible = (i == active)
-            img = fig.image(source=info, image='image',
+            img = fig.image(source=source, image='image',
                             level="image", color_mapper=color_mapper, visible=visible)
             color_bar = ColorBar(color_mapper=color_mapper, label_standoff=12, ticker=ticker,
                                  title=self.signal_units, bar_line_color='black',
@@ -197,7 +205,7 @@ def create_bokeh_image(self):
             fig.y_range.end = yd
             fig.y_range.bounds = (0, yd)
 
-        hover_div, hover_tool = self.add_hover_tool(info, images)
+        hover_div, hover_tool = self.add_hover_tool(source, images)
 
         self.create_figure_title()
         fig.title.text = self.title
@@ -343,13 +351,13 @@ def line_plots(self, main_figure):
 
         return new_plots
 
-    def add_hover_tool(self, info, images):
+    def add_hover_tool(self, source, images):
         """
         Make a hover tool with a div to display text.
 
         Parameters
         ----------
-        info : bokeh.models.ColumnDataSource
+        source : bokeh.models.ColumnDataSource
             Data source for the figure.
         images : list of bokeh.models.GlyphRenderer
             Images to use as renderers for the hover tool.
@@ -360,23 +368,35 @@ def add_hover_tool(self, info, images):
             Div element that will contain text from hover tool.
         hover_tool : bokeh.models.
         """
-        hover_div = Div(height=200, width=200)
-        hover_callback = CustomJS(args={'s': info, 'd': hover_div}, code="""
+        hover_div = Div(height=300, width=300)
+        hover_callback = CustomJS(args={'s': source, 'd': hover_div, 'u': self.signal_units}, code="""
             const idx = cb_data.index.image_indices;
             if (idx.length > 0) { 
                 var x = idx[0].dim1;
                 var y = idx[0].dim2;
-                var val = s.data['image'][0][y][x];
-                val = val.toPrecision(5);
+                var flat = idx[0].flat_index;
+                var val = s.data['image'][0][y][x].toPrecision(5);
                 d.text = "<div style='margin:20px'><h5>Pixel Value</h5>" + 
                          "<div style='display:table; border-spacing: 2px'>" + 
                          "<div style='display:table-row'>" + 
                          "<div style='display:table-cell; text-align:right'>(x, y) =</div>" +
                          "<div style='display:table-cell'>(" + x + ", " + y + ")</div>" +
-                         "</div>" +
-                         "<div style='display:table-row'>" + 
-                         "<div style='display:table-cell; text-align:right'>Value =</div>" +
-                         "<div style='display:table-cell'>" + val + "</div></div></div></div>";
+                         "</div>"
+                if ('ra' in s.data && 'dec' in s.data) {
+                    var ra = s.data['ra'][0][flat].toPrecision(8);
+                    var dec = s.data['dec'][0][flat].toPrecision(8);
+                    d.text += "<div style='display:table-row'>" + 
+                              "<div style='display:table-cell; text-align:right'>RA (deg) =</div>" +
+                              "<div style='display:table-cell'>" + ra + "</div>" +
+                              "</div>" +
+                              "<div style='display:table-row'>" + 
+                              "<div style='display:table-cell; text-align:right'>Dec (deg)=</div>" +
+                              "<div style='display:table-cell'>" + dec + "</div>" +
+                              "</div>"
+                }
+                d.text += "<div style='display:table-row'>" + 
+                          "<div style='display:table-cell; text-align:right'>Value (" + u + ") =</div>" +
+                          "<div style='display:table-cell'>" + val + "</div></div></div></div>";
             } else {
                 d.text = "";
             }
@@ -548,6 +568,17 @@ def get_data(self):
         except KeyError:
             self.signal_units = ''
 
+        ny, nx = self.data.shape
+        col_idx, row_idx = np.indices((ny, nx))
+        try:
+            wcs = WCS(header)
+            if wcs.has_celestial:
+                self.wcs_coord = wcs.pixel_to_world_values(col_idx, row_idx)
+            else:
+                self.wcs_coord = None
+        except (ValueError, TypeError):
+            self.wcs_coord = None
+
     def get_scale(self):
         """Calculate the limits for the display, following the ZScale function
         originally created or IRAF.
diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html
index af750a8c8..1115c0dc5 100644
--- a/jwql/website/apps/jwql/templates/explore_image.html
+++ b/jwql/website/apps/jwql/templates/explore_image.html
@@ -10,7 +10,7 @@
 
     <main role="main" class="container">
     	<!-- Show fits file name -->
-        <h3>Explore Mode </h3>
+        <h2>Explore Mode </h2>
     	<h3>{{ file_root }}_{{ filetype }}.fits</h3>
     	<p>
             <a class="btn btn-primary my-2 mx-2" role="button" href={{ url('jwql:view_image', args=[inst, file_root]) }}>View Image</a>

From fd8f1251c0e1eaef6aa091d7ef305818f7477cd7 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Tue, 13 Jun 2023 18:22:47 -0400
Subject: [PATCH 149/256] Autoscale plots to windowed data instead of zoom
 tools

---
 jwql/utils/interactive_preview_image.py | 127 +++++++++++++++++-------
 1 file changed, 92 insertions(+), 35 deletions(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index 615176cd2..a2e81ea73 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -27,11 +27,9 @@
 from bokeh.layouts import gridplot, layout
 from bokeh.models import (
     BasicTicker, BoxZoomTool, Button, ColorBar, ColumnDataSource,
-    CustomJS, CustomJSTransform, DataRange1d, Div, HoverTool,
-    LinearColorMapper, LogColorMapper, LogTicker, RadioGroup, Range1d, Row,
-    Select, Spacer, Spinner, WheelZoomTool)
+    CustomJS, Div, HoverTool, LinearColorMapper, LogColorMapper, LogTicker,
+    RadioGroup, Range1d, Row, Select, Spacer, Spinner, WheelZoomTool)
 from bokeh.plotting import figure, output_file, show, save
-from bokeh.transform import transform
 from jwst.datamodels import dqflags
 
 
@@ -249,48 +247,59 @@ def line_plots(self, main_figure):
         """
         new_plots = []
         new_lines = []
+        match_ranges = []
+        value_ranges = []
+
         ny, nx = self.data.shape
         col_idx, row_idx = np.indices((ny, nx))
-
-        for index_direction in ['x', 'y']:
+        directions = ['x', 'y']
+        for index_direction in directions:
             if index_direction == 'x':
                 # column plots
-                fig = figure(plot_width=200, plot_height=main_figure.height,
-                             tools='xwheel_zoom, xpan, reset',
+                fig = figure(plot_width=200, plot_height=main_figure.height, tools='',
                              y_axis_location='right', margin=(0, 0, 0, 30))
-                fig.x_range = DataRange1d(only_visible=True)
+                fig.toolbar.logo = None
+
+                fig.x_range = Range1d()
                 fig.y_range = Range1d()
                 match_range = fig.y_range
                 main_range = main_figure.y_range
+                value_range = fig.x_range
 
                 fig.xaxis.axis_label = self.signal_units
                 fig.yaxis.axis_label = 'Row pixel (y)'
                 fig.xaxis.major_label_orientation = np.radians(-45)
 
-                x_plot = 'data'
-                y_plot = 'index'
                 n_plot = nx
-                data_source = {'data': self.data, 'index': col_idx}
-                source = ColumnDataSource(data_source)
+                initial_visible = n_plot // 2
+
+                x = self.data.T
+                y = col_idx.T
+                min_val = np.nanmin(x[initial_visible])
+                max_val = np.nanmax(x[initial_visible])
+
             else:
                 # row plots
-                fig = figure(plot_height=200, plot_width=main_figure.width,
-                             tools='ywheel_zoom, ypan, reset')
+                fig = figure(plot_height=200, plot_width=main_figure.width, tools='')
+                fig.toolbar.logo = None
 
-                fig.y_range = DataRange1d(only_visible=True)
+                fig.y_range = Range1d()
                 fig.x_range = Range1d()
                 match_range = fig.x_range
                 main_range = main_figure.x_range
+                value_range = fig.y_range
 
                 fig.xaxis.axis_label = 'Column pixel (x)'
                 fig.yaxis.axis_label = self.signal_units
 
-                x_plot = 'index'
-                y_plot = 'data'
-                n_plot = ny
                 # indexing is off by 1 for row plots for some reason
-                data_source = {'data': self.data.T, 'index': row_idx.T + 1}
-                source = ColumnDataSource(data_source)
+                n_plot = ny
+                initial_visible = n_plot // 2
+
+                x = row_idx + 1
+                y = self.data
+                min_val = np.nanmin(y[initial_visible])
+                max_val = np.nanmax(y[initial_visible])
 
             # match one of the axes to the main figure
             if main_range.start is not None:
@@ -302,28 +311,29 @@ def line_plots(self, main_figure):
             main_range.js_link('end', match_range, 'end')
             main_range.js_link('end', match_range, 'reset_end')
 
+            # initialize the other to the data
+            pad = 0.1 * (max_val - min_val)
+            value_range.start = min_val - pad
+            value_range.end = max_val + pad
+
+            # plot a step line for each column and plot
+            # all but one are hidden to start
             lines = []
-            initial_visible = n_plot // 2
             for i in range(n_plot):
-                idx_transform = CustomJSTransform(
-                    args={'idx': i},
-                    v_func="""
-                        const x_val = new Float64Array(xs.length);
-                        for (let i = 0; i < xs.length; i++) {
-                            x_val[i] = xs[i][idx];
-                        }
-                        return x_val;
-                    """)
-                line = fig.step(x=transform(x_plot, idx_transform),
-                                y=transform(y_plot, idx_transform),
-                                mode='before', source=source,
+                line = fig.step(x=x[i], y=y[i],
+                                mode='before',
                                 visible=(i == initial_visible),
                                 name=f'Data at {index_direction}={i}')
                 lines.append(line)
             fig.title = lines[initial_visible].name
+
             new_lines.append(lines)
             new_plots.append(fig)
+            match_ranges.append(match_range)
+            value_ranges.append(value_range)
 
+        # watch for tap on plot - makes a new line visible,
+        # matching the selected point
         update_plot = CustomJS(
             args={'lines': new_lines, 'figures': new_plots},
             code="""
@@ -349,6 +359,48 @@ def line_plots(self, main_figure):
                 }""")
         main_figure.js_on_event('tap', update_plot)
 
+        # watch for changes to matched axis to reset data range on value axis
+        for i in range(len(directions)):
+            limit_reset = CustomJS(
+                args={'line': new_lines[i],
+                      'direction': directions[i],
+                      'value_range': value_ranges[i],
+                      'match_range': match_ranges[i]},
+                code="""
+                    var min_val = Infinity;
+                    var max_val = -Infinity;
+                    for (let i=0; i < line.length; i++) {
+                        if (line[i].visible == true) {
+                            var data, idx;
+                            if (direction == 'x') {
+                                data = line[i].data_source.data['x'];
+                                idx = line[i].data_source.data['y'];
+                            } else {
+                                data = line[i].data_source.data['y'];
+                                idx = line[i].data_source.data['x'];
+                            }
+                            for (let j=0; j < data.length; j++) {
+                                if (idx[j] >= match_range.start 
+                                        && idx[j] <= match_range.end) {
+                                    min_val = Math.min(data[j], min_val);
+                                    max_val = Math.max(data[j], max_val);
+                                }
+                            }
+                            break;
+                        }
+                    }
+                    if (min_val < Infinity && min_val != max_val) {
+                        var pad = 0.1 * (max_val - min_val);
+                        value_range.start = min_val - pad;
+                        value_range.end = max_val + pad;
+                    }
+                    """)
+            match_ranges[i].js_on_change('start', limit_reset)
+            match_ranges[i].js_on_change('end', limit_reset)
+
+            # also reset the limits when the plot is tapped for a new column/row
+            main_figure.js_on_event('tap', limit_reset)
+
         return new_plots
 
     def add_hover_tool(self, source, images):
@@ -375,7 +427,12 @@ def add_hover_tool(self, source, images):
                 var x = idx[0].dim1;
                 var y = idx[0].dim2;
                 var flat = idx[0].flat_index;
-                var val = s.data['image'][0][y][x].toPrecision(5);
+                var val = s.data['image'][0][y][x];
+                if (!Number.isFinite(val)) {
+                    val = 'NaN';
+                } else {
+                    val = val.toPrecision(5);
+                }
                 d.text = "<div style='margin:20px'><h5>Pixel Value</h5>" + 
                          "<div style='display:table; border-spacing: 2px'>" + 
                          "<div style='display:table-row'>" + 

From 7b34d6007d1919d3aa0025ca7265e20c063673a2 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Wed, 14 Jun 2023 15:34:08 -0400
Subject: [PATCH 150/256] Fix timing and NaN issues with plot limit setting

---
 jwql/utils/interactive_preview_image.py | 28 ++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index a2e81ea73..07545fd6b 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -367,6 +367,13 @@ def line_plots(self, main_figure):
                       'value_range': value_ranges[i],
                       'match_range': match_ranges[i]},
                 code="""
+                    var timeout;
+                    if (direction == 'x') {
+                        timeout = window._autoscale_timeout_x;
+                    } else {
+                        timeout = window._autoscale_timeout_y;
+                    }
+                    clearTimeout(timeout);
                     var min_val = Infinity;
                     var max_val = -Infinity;
                     for (let i=0; i < line.length; i++) {
@@ -382,17 +389,28 @@ def line_plots(self, main_figure):
                             for (let j=0; j < data.length; j++) {
                                 if (idx[j] >= match_range.start 
                                         && idx[j] <= match_range.end) {
-                                    min_val = Math.min(data[j], min_val);
-                                    max_val = Math.max(data[j], max_val);
+                                    if (Number.isFinite(data[j])) {
+                                        min_val = Math.min(data[j], min_val);
+                                        max_val = Math.max(data[j], max_val);
+                                    }
                                 }
                             }
                             break;
                         }
                     }
-                    if (min_val < Infinity && min_val != max_val) {
+                    if (Number.isFinite(min_val) && Number.isFinite(max_val) && min_val != max_val) {
                         var pad = 0.1 * (max_val - min_val);
-                        value_range.start = min_val - pad;
-                        value_range.end = max_val + pad;
+                        if (direction == 'x') {
+                            window._autoscale_timeout_x = setTimeout(function() {
+                                value_range.start = min_val - pad;
+                                value_range.end = max_val + pad;
+                            });
+                        } else {
+                            window._autoscale_timeout_y = setTimeout(function() {
+                                value_range.start = min_val - pad;
+                                value_range.end = max_val + pad;
+                            });
+                        }
                     }
                     """)
             match_ranges[i].js_on_change('start', limit_reset)

From 82d0360bf449323a24f2b4a831332f64e404bb2d Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 23 Jun 2023 15:29:01 -0400
Subject: [PATCH 151/256] Add download links to image pages

---
 jwql/website/apps/jwql/static/js/jwql.js            | 5 +++++
 jwql/website/apps/jwql/templates/explore_image.html | 5 +++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 73f33eae5..f13e82a9a 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -125,6 +125,11 @@
     document.getElementById("visit_id").innerHTML = parsed_name.visit_id;
     document.getElementById("detector").innerHTML = file_root.split('_')[3];
 
+    // Add a link to download the file from MAST
+     document.getElementById("fits_filename").setAttribute('href',
+         'https://mast.stsci.edu/api/v0.1/Download/file?uri=mast%3AJWST%2Fproduct%2F' +
+         fits_filename + '.fits');
+
     // Show the appropriate image
     var img = document.getElementById("image_viewer");
     var jpg_filepath = '/static/preview_images/' + parsed_name.program + '/' + file_root + '_' + type + '_integ0.jpg';
diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html
index 205bffc2b..d674ebfb7 100644
--- a/jwql/website/apps/jwql/templates/explore_image.html
+++ b/jwql/website/apps/jwql/templates/explore_image.html
@@ -12,9 +12,10 @@
     	<!-- Show fits file name -->
         <h3>Explore Mode </h3>
     	<h3>{{ file_root }}_{{ filetype }}.fits</h3>
-    	<p>
+        <p>
             <a class="btn btn-primary my-2 mx-2" role="button" href={{ url('jwql:view_image', args=[inst, file_root]) }}>View Image</a>
             <a class="btn btn-primary my-2 mx-2" role="button" href={{ url('jwql:archive_thumb_per_obs', args=[inst, file_root[2:7], file_root[7:10]]) }}> View Proposal</a>
+            <a class="btn btn-primary my-2 mx-2" role="button" href="https://mast.stsci.edu/api/v0.1/Download/file?uri=mast%3AJWST%2Fproduct%2F{{ file_root }}_{{ filetype }}.fits">Download File</a>
 	    </p>
 
         <!-- Allow the user to change the settings in the image to explore -->
@@ -129,4 +130,4 @@ <h5>Submit Anomaly</h5>
     <script>explore_image_update_enable_options('{{ extension_ints }}', '{{ extension_groups }}');</script>
     <script>update_explore_image_page('{{inst}}', '{{file_root}}', '{{filetype}}', '{{base_url}}');</script>
 
-{% endblock %}
\ No newline at end of file
+{% endblock %}

From 153727854c50bb1e05753198d38df543d88f982c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 3 Jul 2023 17:22:38 -0400
Subject: [PATCH 152/256] Tweaks from testing on server

---
 jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +-
 jwql/shared_tasks/run_pipeline.py                        | 2 ++
 jwql/shared_tasks/shared_tasks.py                        | 7 ++++---
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index b8399b1d9..929d9d099 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -712,7 +712,7 @@ def process(self, file_list):
                 pipeline_files.append(filename)
 
         # Specify that we want to skip the dark current correction step
-        step_args = {'dark_current': {'skip: True'}}
+        step_args = {'dark_current': {'skip': True}}
 
         # Call the pipeline
         outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args)
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 2e34f58e3..2ab107318 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -334,6 +334,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     working_path = args.working_path
     pipe_type = args.pipe
     outputs = args.outputs
+    step_args = args.step_args
 
     status_file = os.path.join(working_path, short_name+"_status.txt")
     with open(status_file, 'w') as out_file:
@@ -344,6 +345,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         out_file.write("\tinstrument is {} ({})\n".format(instrument, type(instrument)))
         out_file.write("\tinput_file is {} ({})\n".format(input_file, type(input_file)))
         out_file.write("\tshort_name is {} ({})\n".format(short_name, type(short_name)))
+        out_file.write("\tstep_args is {} ({})\n".format(step_args, type(step_args)))
 
     if not os.path.isfile(args.input_file):
         raise FileNotFoundError("No input file {}".format(args.input_file))
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 047084d62..42ee9021b 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -239,7 +239,7 @@ def convert_step_args_to_string(args_dict):
     args_str : str
         String representation of ``args_dict``
     """
-    args_str='{'
+    args_str="'{"
 
     for i, step in enumerate(args_dict):
         args_str += f'"{step}":'
@@ -248,9 +248,10 @@ def convert_step_args_to_string(args_dict):
             args_str += f'"{param}":"{val}"'
             if j < len(args_dict[step])-1:
                 args_str += ', '
-        args_str += '}'
+        args_str += "}"
         if i < len(args_dict)-1:
             args_str += ','
+    args_str += "}'"
     return args_str
 
 
@@ -262,7 +263,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
     logging.info("Running {}".format(command))
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
-    with process.stderr:
+    with process.stderr:<log_subprocess
         log_subprocess_output(process.stderr)
     result = process.wait()
     logging.info("Subprocess result was {}".format(result))

From 5dba6d062f411580a40b44deed742a18417a29b8 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 7 Jul 2023 12:03:37 -0400
Subject: [PATCH 153/256] Add help message for column/row plots

---
 jwql/website/apps/jwql/static/js/jwql.js            | 7 +++++++
 jwql/website/apps/jwql/templates/explore_image.html | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index dfc3baa48..4bca2470e 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1109,6 +1109,13 @@ function update_wata_page(base_url) {
             */
             $('#explore_image').html(content);
 
+            // Add a help message for plots
+            if (show_line_plots === true) {
+                $('#help').html('<span class="help-tip mx-1">i</span>Click on the image to update the column/row plots.');
+            } else {
+                $('#help').html('');
+            }
+
             // Replace loading screen
             document.getElementById("loading").style.display = "none";
             document.getElementById("explore_image").style.display = "inline-block";
diff --git a/jwql/website/apps/jwql/templates/explore_image.html b/jwql/website/apps/jwql/templates/explore_image.html
index c1629fa9b..62ac88c9b 100644
--- a/jwql/website/apps/jwql/templates/explore_image.html
+++ b/jwql/website/apps/jwql/templates/explore_image.html
@@ -17,7 +17,7 @@ <h3>{{ file_root }}_{{ filetype }}.fits</h3>
             <a class="btn btn-primary my-2 mx-2" role="button" href={{ url('jwql:archive_thumb_per_obs', args=[inst, file_root[2:7], file_root[7:10]]) }}> View Proposal</a>
             <a class="btn btn-primary my-2 mx-2" role="button" href="https://mast.stsci.edu/api/v0.1/Download/file?uri=mast%3AJWST%2Fproduct%2F{{ file_root }}_{{ filetype }}.fits">Download File</a>
 	    </p>
-
+        <p id="help"></p>
 
 		<div class="row">
             <!-- Display the image -->

From 75b2edb74cc17a8755fdf5cae88781d5707be533 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 7 Jul 2023 16:45:50 -0400
Subject: [PATCH 154/256] Update docstring

---
 jwql/website/apps/jwql/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index af4beb1b0..f1916dce1 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -1009,7 +1009,7 @@ def explore_image_ajax(request, inst, file_root, filetype, line_plots='false', l
     filetype : str
         Type of file (e.g. ``uncal``)
     line_plots : str
-        Scaling to implement in interactive preview image ("log" or "lin")
+        If 'true', column and row plots will be computed and shown with the image.
     low_lim : str
         Signal value to use as the lower limit of the displayed image. If "None", it will be calculated using the ZScale function
     high_lim : str

From f8b84a338df7235b76fdf8d2d1df78dd872d0cdc Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 7 Jul 2023 16:46:55 -0400
Subject: [PATCH 155/256] Fix hover tool for DQ image

---
 jwql/utils/interactive_preview_image.py | 34 +++++++++++++++----------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index 07545fd6b..d3b93bf73 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -439,17 +439,28 @@ def add_hover_tool(self, source, images):
         hover_tool : bokeh.models.
         """
         hover_div = Div(height=300, width=300)
-        hover_callback = CustomJS(args={'s': source, 'd': hover_div, 'u': self.signal_units}, code="""
+
+        is_dq = ('DQ' in self.extname)
+        hover_callback = CustomJS(args={'s': source, 'd': hover_div,
+                                        'u': self.signal_units, 'dq': is_dq}, code="""
             const idx = cb_data.index.image_indices;
             if (idx.length > 0) { 
                 var x = idx[0].dim1;
                 var y = idx[0].dim2;
                 var flat = idx[0].flat_index;
-                var val = s.data['image'][0][y][x];
-                if (!Number.isFinite(val)) {
-                    val = 'NaN';
+                var val;
+                var label;
+                if (dq === true) {
+                    val = s.data['dq'][0][y][x].join(', ');
+                    label = "Value";
                 } else {
-                    val = val.toPrecision(5);
+                    var val = s.data['image'][0][y][x];
+                    if (!Number.isFinite(val)) {
+                        val = 'NaN';
+                    } else {
+                        val = val.toPrecision(5);
+                    }
+                    label = "Value (" + u + ")";
                 }
                 d.text = "<div style='margin:20px'><h5>Pixel Value</h5>" + 
                          "<div style='display:table; border-spacing: 2px'>" + 
@@ -461,7 +472,7 @@ def add_hover_tool(self, source, images):
                     var ra = s.data['ra'][0][flat].toPrecision(8);
                     var dec = s.data['dec'][0][flat].toPrecision(8);
                     d.text += "<div style='display:table-row'>" + 
-                              "<div style='display:table-cell; text-align:right'>RA (deg) =</div>" +
+                              "<div style='display:table-cell; text-align:right'>RA (deg)=</div>" +
                               "<div style='display:table-cell'>" + ra + "</div>" +
                               "</div>" +
                               "<div style='display:table-row'>" + 
@@ -470,18 +481,15 @@ def add_hover_tool(self, source, images):
                               "</div>"
                 }
                 d.text += "<div style='display:table-row'>" + 
-                          "<div style='display:table-cell; text-align:right'>Value (" + u + ") =</div>" +
+                          "<div style='display:table-cell; text-align:right'>" + label + "=</div>" +
                           "<div style='display:table-cell'>" + val + "</div></div></div></div>";
             } else {
                 d.text = "";
             }
         """)
-        if 'DQ' not in self.extname:
-            hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images,
-                                   callback=hover_callback)
-        else:
-            hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images,
-                                   callback=hover_callback)
+        hover_tool = HoverTool(tooltips=None, mode='mouse', renderers=images,
+                               callback=hover_callback)
+
         return hover_div, hover_tool
 
     def add_interactive_controls(self, images, color_bars):

From 51bcd01fe1e79bfab70680767ebe09547dbed1b8 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 7 Jul 2023 17:32:06 -0400
Subject: [PATCH 156/256] Fix hover tool for uncal images

---
 jwql/utils/interactive_preview_image.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index d3b93bf73..211a03340 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -454,7 +454,13 @@ def add_hover_tool(self, source, images):
                     val = s.data['dq'][0][y][x].join(', ');
                     label = "Value";
                 } else {
-                    var val = s.data['image'][0][y][x];
+                    // get the data from the array of arrays
+                    val = s.data['image'][0][y][x];
+                    if (val === undefined) {
+                        // uncal images have to be addressed with the flat index
+                        val = s.data['image'][0][flat];
+                    }
+                    // report any non-number as NaN
                     if (!Number.isFinite(val)) {
                         val = 'NaN';
                     } else {

From eb45ae1b48502c0af68dc9f6094776a84bc360c4 Mon Sep 17 00:00:00 2001
From: Melanie Clarke <mclarke@stsci.edu>
Date: Fri, 7 Jul 2023 17:47:58 -0400
Subject: [PATCH 157/256] Handle single-valued DQ pixels

---
 jwql/utils/interactive_preview_image.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/jwql/utils/interactive_preview_image.py b/jwql/utils/interactive_preview_image.py
index 211a03340..da9563770 100644
--- a/jwql/utils/interactive_preview_image.py
+++ b/jwql/utils/interactive_preview_image.py
@@ -451,7 +451,10 @@ def add_hover_tool(self, source, images):
                 var val;
                 var label;
                 if (dq === true) {
-                    val = s.data['dq'][0][y][x].join(', ');
+                    val = s.data['dq'][0][y][x];
+                    if (Array.isArray(val)) {
+                        val = val.join(', ');
+                    }
                     label = "Value";
                 } else {
                     // get the data from the array of arrays

From 13dcb142b0c71d0136fe03b27f315b0057b9b4dc Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 12 Jul 2023 10:05:27 -0400
Subject: [PATCH 158/256] prepare models for external postgres db

---
 jwql/example_config.json           | 8 ++++++++
 jwql/website/jwql_proj/settings.py | 5 +----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/jwql/example_config.json b/jwql/example_config.json
index 2f9c3ad8c..112b13755 100644
--- a/jwql/example_config.json
+++ b/jwql/example_config.json
@@ -10,6 +10,14 @@
         "host" : "",
         "port" : ""
     },
+    "django_database" : {
+        "engine" : "",
+        "name" : "",
+        "user" : "",
+        "password" : "",
+        "host" : "",
+        "port" : ""
+    },
     "jwql_dir" : "",
     "jwql_version": "",
     "server_type": "",
diff --git a/jwql/website/jwql_proj/settings.py b/jwql/website/jwql_proj/settings.py
index 676d164d9..6dc070ca7 100644
--- a/jwql/website/jwql_proj/settings.py
+++ b/jwql/website/jwql_proj/settings.py
@@ -108,10 +108,7 @@
 # Database
 # https://docs.djangoproject.com/en/2.0/ref/settings/#databases
 DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
-    },
+    'default': get_config()['django_database']
 }
 
 # Password validation

From 06458e6968c30a8be05b098724f00d6b95f87f89 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 12 Jul 2023 11:56:52 -0400
Subject: [PATCH 159/256] db keys must be all caps

---
 jwql/example_config.json | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/jwql/example_config.json b/jwql/example_config.json
index 112b13755..937a1d63b 100644
--- a/jwql/example_config.json
+++ b/jwql/example_config.json
@@ -11,12 +11,12 @@
         "port" : ""
     },
     "django_database" : {
-        "engine" : "",
-        "name" : "",
-        "user" : "",
-        "password" : "",
-        "host" : "",
-        "port" : ""
+        "ENGINE" : "",
+        "NAME" : "",
+        "USER" : "",
+        "PASSWORD" : "",
+        "HOST" : "",
+        "PORT" : ""
     },
     "jwql_dir" : "",
     "jwql_version": "",

From 9f97ca9a799b6db2fe4f8532b0d56c176596d87b Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Thu, 13 Jul 2023 17:39:46 -0400
Subject: [PATCH 160/256] update tab_service search for all intents

---
 jwql/website/apps/jwql/data_containers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index c3828b11a..1b6cdcae7 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1248,9 +1248,9 @@ def get_instrument_proposals(instrument):
         List of proposals for the given instrument
     """
     tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx")
-    tap_results = tap_service.search(f"select distinct proposal_id from dbo.ObsPointing where obs_collection='JWST' and calib_level>0 and instrument_name like '{instrument.lower()}%'")
+    tap_results = tap_service.search(f"select distinct prpID from CaomObservation where collection='JWST' and maxLevel>0 and insName like '{instrument.lower()}%'")
     prop_table = tap_results.to_table()
-    proposals = prop_table['proposal_id'].data
+    proposals = prop_table['prpID'].data
     inst_proposals = sorted(proposals.compressed(), reverse=True)
     return inst_proposals
 

From 02a6b63ef686357b7920ad8603ddd15b828080a4 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Jul 2023 12:47:08 -0400
Subject: [PATCH 161/256] rateints case working

---
 jwql/shared_tasks/run_pipeline.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 2ab107318..7e743bbf8 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -50,6 +50,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         status_f.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename)))
         status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir)))
         status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file)))
+        status_f.write(f"\t outputs is {outputs}\n")
 
     try:
         copy_files([input_file], work_directory)
@@ -133,8 +134,12 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
                             pass
                         model[0].save(output_file)
                         if 'rateints' in outputs:
-                            model[1].save(output_file.replace('rate', 'rateints'))
-
+                            outbase = os.path.basename(output_file)
+                            outbase = outbase.replace('rate', 'rateints')
+                            output_file = os.path.join(work_directory, outbase)
+                            model[1].save(output_file)
+                            with open(status_file, 'a+') as status_f:
+                                status_f.write(f"Saved rateints model to {output_file}\n")
                     done = True
                     for output in outputs:
                         output_name = "{}_{}.fits".format(short_name, output)

From 9f5f05d1fc8271faaf36554d21aff59dbbcacabd Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Jul 2023 12:54:01 -0400
Subject: [PATCH 162/256] Uncomment the get() lines

---
 jwql/shared_tasks/shared_tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 42ee9021b..244d1a3e8 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -746,7 +746,7 @@ def run_pipeline(input_file, in_ext, ext_or_exts, instrument, jump_pipe=False):
         uncal_name = os.path.basename(uncal_file)
         result = start_pipeline(uncal_name, short_name, ext_or_exts, instrument, jump_pipe=jump_pipe)
         logging.info("\t\tStarting with ID {}".format(result.id))
-        #processed_path = result.get()
+        processed_path = result.get()
         logging.info("\t\tPipeline Complete")
         output = retrieve_files(short_name, ext_or_exts, retrieve_dir)
     except Exception as e:
@@ -838,7 +838,7 @@ def run_parallel_pipeline(input_files, in_ext, ext_or_exts, instrument, jump_pip
         for short_name in results:
             try:
                 logging.info("\tWaiting for {} ({})".format(short_name, results[short_name].id))
-                #processed_path = results[short_name].get()
+                processed_path = results[short_name].get()
                 logging.info("\t{} retrieved".format(short_name))
                 outputs[input_file_paths[short_name]] = retrieve_files(short_name, ext_or_exts, output_dirs[short_name])
                 logging.info("\tFiles copied for {}".format(short_name))

From 21e0d3eb5ad810fccf8a930a4ee5c67f239dedf3 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Jul 2023 15:07:18 -0400
Subject: [PATCH 163/256] Remove bad cut and paste text

---
 jwql/shared_tasks/shared_tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index e83dd66ce..b35410291 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -263,7 +263,7 @@ def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_fi
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
     logging.info("Running {}".format(command))
     process = Popen(command, shell=True, executable="/bin/bash", stderr=PIPE)
-    with process.stderr:<log_subprocess
+    with process.stderr:
         log_subprocess_output(process.stderr)
     result = process.wait()
     logging.info("Subprocess result was {}".format(result))

From 6330d2bb6ab845826a6b83bc3dede0115c85d343 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 25 Jul 2023 15:20:48 -0400
Subject: [PATCH 164/256] Return dark monitor to specifying rate outputs.
 rateints will be another PR

---
 jwql/instrument_monitors/common_monitors/dark_monitor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 929d9d099..8ffab9146 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -715,7 +715,7 @@ def process(self, file_list):
         step_args = {'dark_current': {'skip': True}}
 
         # Call the pipeline
-        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate", "rateints"], self.instrument, step_args=step_args)
+        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate"], self.instrument, step_args=step_args)
         for filename in file_list:
             processed_file = filename.replace("_dark", "_rate")
             if processed_file not in slope_files and os.path.isfile(processed_file):

From 95de7d98f945b681d951897e52d74d0298275e1a Mon Sep 17 00:00:00 2001
From: Bradley Sappington
 <101193271+BradleySappington@users.noreply.github.com>
Date: Tue, 25 Jul 2023 16:39:30 -0400
Subject: [PATCH 165/256] Update README.md

---
 README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 5909b8889..70b4e173b 100644
--- a/README.md
+++ b/README.md
@@ -154,10 +154,11 @@ Any questions about the `jwql` project or its software can be directed to `jwql@
 - Mees Fix (Technical Lead, INS) [@mfixstsci](https://github.com/mfixstsci)
 - Misty Cracraft (INS) [@cracraft](https://github.com/cracraft)
 - Mike Engesser (INS) [@mengesser](https://github.com/mengesser)
-- Shannon Osborne (INS) [@shanosborne](https://github.com/shanosborne)
 - Maria Pena-Guerrero [@penaguerrero](https://github.com/penaguerrero)
 - Ben Sunnquist (INS) [@bsunnquist](https://github.com/bsunnquist)
 - Brian York (INS) [@york-stsci](https://github.com/york-stsci)
+- Bradley Sappington (INS) [@bradleysappington](https://github.com/bradleysappington)
+- Melanie Clarke (INS) [@melanieclarke](https://github.com/melanieclarke)
 
 ## Past Development Team Members
 - Matthew Bourque (INS) [@bourque](https://github.com/bourque)
@@ -168,7 +169,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@
 - Sara Ogaz (DMD) [@SaOgaz](https://github.com/SaOgaz)
 - Catherine Martlin (INS) [@catherine-martlin](https://github.com/catherine-martlin)
 - Johannes Sahlmann (INS) [@Johannes-Sahlmann](https://github.com/johannes-sahlmann)
-
+- Shannon Osborne (INS) [@shanosborne](https://github.com/shanosborne)
 
 ## Acknowledgments:
 - Faith Abney (DMD)

From 4adc5853acfc0398b3035f8fbbdcafd048afd480 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 27 Jul 2023 15:28:36 -0400
Subject: [PATCH 166/256] use json loads rather than custom function. resp to
 review comments

---
 jwql/shared_tasks/run_pipeline.py |  3 --
 jwql/shared_tasks/shared_tasks.py | 54 +++++--------------------------
 2 files changed, 8 insertions(+), 49 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 7e743bbf8..d9af19086 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -213,9 +213,6 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         if instrument == 'nircam':
             params['refpix'] = dict(odd_even_rows=False)
 
-        # Default CR rejection threshold is too low
-        #params['jump'] = dict(rejection_threshold=15)
-
         # Set up to save jump step output
         params['jump']['save_results'] = True
         params['jump']['output_dir'] = work_directory
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index b35410291..b7164b042 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -75,6 +75,7 @@ def some_function(some_arguments):
 from copy import deepcopy
 import gc
 from glob import glob
+import json
 import logging
 from logging import FileHandler, StreamHandler
 import os
@@ -224,40 +225,9 @@ def collect_after_task(**kwargs):
     gc.collect()
 
 
-def convert_step_args_to_string(args_dict):
-    """Convert the nested dictionary containing pipeline step parameter keyword/value pairs
-    to a string so that it can be passed via command line
-
-    Parameters
-    ----------
-    args_dict : dict
-        Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing
-        keyword value pairs for that step.
-
-    Returns
-    -------
-    args_str : str
-        String representation of ``args_dict``
-    """
-    args_str="'{"
-
-    for i, step in enumerate(args_dict):
-        args_str += f'"{step}":'
-        args_str += '{'
-        for j, (param, val) in enumerate(args_dict[step].items()):
-            args_str += f'"{param}":"{val}"'
-            if j < len(args_dict[step])-1:
-                args_str += ', '
-        args_str += "}"
-        if i < len(args_dict)-1:
-            args_str += ','
-    args_str += "}'"
-    return args_str
-
-
 def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args):
     # Convert step_args dictionary to a string so that it can be passed via command line
-    step_args_str = convert_step_args_to_string(step_args)
+    step_args_str = json.loads(step_args)
 
     command = "{} {} {} '{}' {} {} {} {} --step_args {}"
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)
@@ -557,7 +527,7 @@ def prep_file(input_file, in_ext):
     short_name : str
         The exposure ID with the calibration tag and the fits extension chopped off.
 
-    uncal_file : str
+    input_name : str
         The raw file to be calibrated
     """
     config = get_config()
@@ -569,16 +539,8 @@ def prep_file(input_file, in_ext):
     input_path, input_name = os.path.split(input_file)
     logging.info("\tPath is {}, file is {}".format(input_path, input_name))
 
-    if "uncal" not in in_ext and "dark" not in in_ext:
-        logging.info("\tSwitching from {} to uncal".format(in_ext))
-        uncal_name = os.path.basename(input_file).replace(in_ext, "uncal")
-        uncal_file = filesystem_path(uncal_name, check_existence=True)
-    else:
-        uncal_file = input_file
-        uncal_name = input_name
-
-    if not os.path.isfile(uncal_file):
-        raise FileNotFoundError("Input File {} does not exist.".format(uncal_file))
+    if not os.path.isfile(input_file):
+        raise FileNotFoundError("Input File {} does not exist.".format(input_file))
 
     output_file_or_files = []
     short_name = input_name.replace("_" + in_ext, "").replace(".fits", "")
@@ -590,9 +552,9 @@ def prep_file(input_file, in_ext):
         logging.critical(msg.format(short_name))
         raise ValueError("Redis lock for {} is in an unknown state".format(short_name))
     logging.info("\t\tAcquired Lock.")
-    logging.info("\t\tCopying {} to {}".format(uncal_file, send_path))
-    copy_files([uncal_file], send_path)
-    return short_name, cal_lock, os.path.join(send_path, uncal_name)
+    logging.info("\t\tCopying {} to {}".format(input_file, send_path))
+    copy_files([input_file], send_path)
+    return short_name, cal_lock, os.path.join(send_path, input_name)
 
 
 def start_pipeline(input_file, short_name, ext_or_exts, instrument, jump_pipe=False, step_args={}):

From 5fdcec1fee1a96732786f3e1aef1fe41b1b7d8ab Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 28 Jul 2023 12:33:20 -0400
Subject: [PATCH 167/256] Returning to custom function for dictioanry to string
 conversion

---
 jwql/shared_tasks/run_pipeline.py |  1 -
 jwql/shared_tasks/shared_tasks.py | 35 +++++++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index d9af19086..74754afd0 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -87,7 +87,6 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # Set any steps the user specifically asks to skip
         for step, step_dict in step_args.items():
             if 'skip' in step_dict:
-                print(f'SKIP the {step}!!')
                 if step_dict['skip']:
                     steps[step] = False
 
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index b7164b042..7a05f6f92 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -224,10 +224,41 @@ def after_setup_celery_logger(logger, **kwargs):
 def collect_after_task(**kwargs):
     gc.collect()
 
+def convert_step_args_to_string(args_dict):
+    """Convert the nested dictionary containing pipeline step parameter keyword/value pairs
+    to a string so that it can be passed via command line
+
+    Parameters
+    ----------
+    args_dict : dict
+        Nested dictionary. Top level keys are pipeline step names. Values are dictionaries containing
+        keyword value pairs for that step.
+
+    Returns
+    -------
+    args_str : str
+        String representation of ``args_dict``
+    """
+    args_str="'{"
+
+    for i, step in enumerate(args_dict):
+        args_str += f'"{step}":'
+        args_str += '{'
+        for j, (param, val) in enumerate(args_dict[step].items()):
+            args_str += f'"{param}":"{val}"'
+            if j < len(args_dict[step])-1:
+                args_str += ', '
+        args_str += "}"
+        if i < len(args_dict)-1:
+            args_str += ','
+    args_str += "}'"
+    return args_str
+
 
 def run_subprocess(name, cmd, outputs, cal_dir, ins, in_file, short_name, res_file, cores, step_args):
-    # Convert step_args dictionary to a string so that it can be passed via command line
-    step_args_str = json.loads(step_args)
+    # Convert step_args dictionary to a string so that it can be passed via command line.
+    # For some reason, json.dumps() doesn't seem to work correctly, so we use a custom function.
+    step_args_str = convert_step_args_to_string(step_args)
 
     command = "{} {} {} '{}' {} {} {} {} --step_args {}"
     command = command.format(name, cmd, outputs, cal_dir, ins, in_file, short_name, cores, step_args_str)

From 7b1fe988bd52cbfbd8ba4052e58b3efd02831da1 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 28 Jul 2023 13:54:06 -0400
Subject: [PATCH 168/256] Return jump threshold statement.

---
 jwql/shared_tasks/run_pipeline.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 74754afd0..a79167e0b 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -212,6 +212,9 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         if instrument == 'nircam':
             params['refpix'] = dict(odd_even_rows=False)
 
+        # Default CR rejection threshold is too low
+        params['jump']['rejection_threshold'] = 15
+
         # Set up to save jump step output
         params['jump']['save_results'] = True
         params['jump']['output_dir'] = work_directory

From dd136258b7436e32685e2c09874ed87453b014af Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 28 Jul 2023 14:48:26 -0400
Subject: [PATCH 169/256] update anomaly search backend

---
 jwql/website/apps/jwql/data_containers.py | 42 +++--------------------
 1 file changed, 5 insertions(+), 37 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 1b6cdcae7..f4ecb6aa7 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1498,8 +1498,7 @@ def get_rootnames_from_query(parameters):
     filtered_rootnames : list
         A list of all root filenames filtered from the given parameters
     """
-    # TODO - This code setup is temporary until the merge to Postgres is complete.
-    # selected_rootfileinfos = RootFileInfo.objects.none()
+
     filtered_rootnames = []
     # Each Query Selection is Instrument specific
     for inst in parameters[QUERY_CONFIG_KEYS.INSTRUMENTS]:
@@ -1544,23 +1543,13 @@ def get_rootnames_from_query(parameters):
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(read_patt__in=inst_read_patt)
         if (inst_subarray != []):
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(subarray__in=inst_subarray)
-
-        # TODO - This uncommented CODE is what we will use while DJANGO is still using the SQLITE3 DB.
-        #        ONCE DB IS MIGRATED TO POSTGRES WE CAN REPLACE THIS CODE WITH THE UNTESTED CODE COMMENTED OUT BELOW
-        #      >>>  START CODE PRE DB MIGRATION HERE  <<<
         if (inst_anomalies != []):
-            # If the rootfile info has any of the marked anomalies we want it
-            # Make union of marked anomalies from our current query set, then save intersection.
-            anomaly_filters = None
+            anomaly_rootfileinfos = RootFileInfo.objects.none()
             for anomaly in inst_anomalies:
+                # If the rootfile info has any of the marked anomalies we want it
                 anomaly_filter = "anomalies__" + str(anomaly).lower()
-                this_anomaly_rootnames = current_ins_rootfileinfos.filter(**{anomaly_filter: True})
-                if anomaly_filters is None:
-                    anomaly_filters = this_anomaly_rootnames
-                else:
-                    anomaly_filters |= this_anomaly_rootnames
-            current_ins_rootfileinfos = anomaly_filters
-        #      >>>  END CODE PRE DB MIGRATION HERE  <<<
+                anomaly_rootfileinfos = anomaly_rootfileinfos.union(current_ins_rootfileinfos.filter(**{anomaly_filter: True}))
+            current_ins_rootfileinfos = current_ins_rootfileinfos.intersection(anomaly_rootfileinfos)
 
         # sort as desired
         if sort_type.upper() == 'ASCENDING':
@@ -1577,27 +1566,6 @@ def get_rootnames_from_query(parameters):
 
     return filtered_rootnames
 
-    # TODO - BELOW IS THE OUTLINE OF CODE WE WANT TO USE, HOWEVER THIS CAN'T BE IMPLEMENTED WITH DJANGO RUNNING SQLITE
-    #             ONCE WE MIGRATE TO POSTGRES WE CAN IMPLEMENT THE BELOW FUNCTIONALITY WHICH SHOULD MAKE THIS CODE MOVE A LITTLE FASTER
-    #             NOTE: THIS CODE OUTLINE IS UNTESTED and is only a rough outline!
-    #      >>>  START CODE AFTER DB MIGRATION HERE  <<<
-    #     if (inst_anomalies != []):
-    #         # If the rootfile info has any of the marked anomalies we want it
-    #         # Make union of marked anomalies from our current query set, then save intersection.
-    #         anomaly_rootfileinfos = RootFileInfo.objects.none()
-
-    #         for anomaly in inst_anomalies:
-    #             anomaly_filter = "anomalies__" + str(anomaly).lower()
-    #             anomaly_rootfileinfos.union(current_ins_rootfileinfos.filter(**{anomaly_filter: True}))
-
-    #         current_ins_rootfileinfos = current_ins_rootfileinfos.intersection(anomaly_rootfileinfos)
-
-    #     # Add this instrument's query set to our return queryset
-    #     selected_rootfileinfos = selected_rootfileinfos.union(current_ins_rootfileinfos)
-
-    # return selected_rootfileinfos  # OR RETURN LIST OF ROOTNAMES
-    #      >>>  END CODE AFTER DB MIGRATION HERE  <<<
-
 
 def get_thumbnails_by_instrument(inst):
     """Return a list of thumbnails available in the filesystem for the

From 7f4334566f787a1f75dc94b8ab85dd76d9437ab4 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 1 Aug 2023 16:37:32 -0400
Subject: [PATCH 170/256] exploring

---
 environment_python_3.10.yml | 64 ++++++++++++++++++-------------------
 environment_python_3.9.yml  | 64 ++++++++++++++++++-------------------
 2 files changed, 64 insertions(+), 64 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 207f9d658..32bef050d 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -15,58 +15,58 @@
 # To remove the environment entirely, run the following command:
 # $ conda env remove -n jwql-3.10
 
-name: jwql-3.10
+name: jwql-3.10-deptest
 
 channels:
   - conda-forge
   - defaults
 
 dependencies:
-  - astropy=5.2.2
+  - astropy=5.3.1
   - bokeh=2.4.3
   - beautifulsoup4=4.12.2
-  - celery=5.2.7
-  - cryptography=40.0.2
-  - django=4.2.1
+  - celery=5.3.1
+  - cryptography=41.0.2
+  - django=4.2.3
   - inflection=0.5.1
-  - ipython=8.13.2
+  - ipython=8.14.0
   - jinja2=3.1.2
   - jsonschema=4.17.3
-  - matplotlib=3.7.0
-  - nodejs=18.15.0
-  - numpy=1.24.3
+  - matplotlib=3.7.2
+  - nodejs=18.16.1
+  - numpy=1.25.1
   - numpydoc=1.5.0
-  - pandas=2.0.1
-  - pip=23.1.2
-  - postgresql=15.2
-  - psycopg2=2.9.3
-  - pytest=7.3.1
-  - pytest-cov=4.0.0
-  - pytest-mock=3.10.0
-  - python=3.10.9
+  - pandas=2.0.3
+  - pip=23.2.1
+  - postgresql=15.3
+  - psycopg2=2.9.6
+  - pytest=7.4.0
+  - pytest-cov=4.1.0
+  - pytest-mock=3.11.1
+  - python=3.10.12
   - pyyaml=6.0
   - redis
-  - ruff=0.0.269
+  - ruff=0.0.280
   - scipy=1.9.3
-  - setuptools=67.7.2
+  - setuptools=68.0.0
   - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.15
+  - sphinx_rtd_theme=1.2.2
+  - sqlalchemy=2.0.19
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
-    - astroquery==0.4.6
-    - bandit==1.7.5
-    - jwst==1.10.2
-    - pysiaf==0.19.1
-    - pysqlite3==0.5.0
-    - pyvo==1.4.1
-    - redis==4.5.5
-    - selenium==4.9.1
-    - stdatamodels==1.3.1
-    - stsci_rtd_theme==1.0.0
-    - vine==5.0.0
+    - astroquery=0.4.6
+    - bandit=1.7.5
+    - jwst=1.11.3
+    - pysiaf=0.20.0
+    - pysqlite3=0.5.1
+    - pyvo=1.4.1
+    - redis=4.6.0
+    - selenium=4.10.0
+    - stdatamodels=1.7.1
+    - stsci_rtd_theme=1.0.0
+    - vine=5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
 
     # Current package
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index cf20b4108..dd677fa50 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -15,58 +15,58 @@
 # To remove the environment entirely, run the following command:
 # $ conda env remove -n jwql-3.9
 
-name: jwql-3.9
+name: jwql-3.9-deptest
 
 channels:
   - conda-forge
   - defaults
 
 dependencies:
-  - astropy=5.2.2
+  - astropy=5.3.1
   - bokeh=2.4.3
   - beautifulsoup4=4.12.2
-  - celery=5.2.7
-  - cryptography=40.0.2
-  - django=4.2.1
+  - celery=5.3.1
+  - cryptography=41.0.2
+  - django=4.2.3
   - inflection=0.5.1
-  - ipython=8.13.2
+  - ipython=8.14.0
   - jinja2=3.1.2
   - jsonschema=4.17.3
-  - matplotlib=3.7.0
-  - nodejs=18.15.0
-  - numpy=1.24.3
+  - matplotlib=3.7.2
+  - nodejs=18.16.1
+  - numpy=1.25.1
   - numpydoc=1.5.0
-  - pandas=2.0.1
-  - pip=23.1.2
-  - postgresql=15.2
-  - psycopg2=2.9.3
-  - pytest=7.3.1
-  - pytest-cov=4.0.0
-  - pytest-mock=3.10.0
-  - python=3.9.16
+  - pandas=2.0.3
+  - pip=23.2.1
+  - postgresql=15.3
+  - psycopg2=2.9.6
+  - pytest=7.4.0
+  - pytest-cov=4.1.0
+  - pytest-mock=3.11.1
+  - python=3.9.17
   - pyyaml=6.0
   - redis
-  - ruff=0.0.269
+  - ruff=0.0.280
   - scipy=1.9.3
-  - setuptools=67.7.2
+  - setuptools=68.0.0
   - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.15
+  - sphinx_rtd_theme=1.2.2
+  - sqlalchemy=2.0.19
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
-    - astroquery==0.4.6
-    - bandit==1.7.5
-    - jwst==1.10.2
-    - pysiaf==0.19.1
-    - pysqlite3==0.5.0
-    - pyvo==1.4.1
-    - redis==4.5.5
-    - selenium==4.9.1
-    - stdatamodels==1.3.1
-    - stsci_rtd_theme==1.0.0
-    - vine==5.0.0
+    - astroquery=0.4.6
+    - bandit=1.7.5
+    - jwst=1.11.3
+    - pysiaf=0.20.0
+    - pysqlite3=0.5.1
+    - pyvo=1.4.1
+    - redis=4.6.0
+    - selenium=4.10.0
+    - stdatamodels=1.7.1
+    - stsci_rtd_theme=1.0.0
+    - vine=5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
 
     # Current package

From 2673b3cee46078019dfac099826482358aba2a8b Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Tue, 1 Aug 2023 16:46:30 -0400
Subject: [PATCH 171/256] Smarter about base filename

---
 jwql/shared_tasks/shared_tasks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 7a05f6f92..be232935d 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -459,7 +459,8 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
         logging.error("File {} not found!".format(input_file))
         raise FileNotFoundError("{} not found".format(input_file))
 
-    short_name = input_file_name[0: input_file_name.rfind('_')]
+    parts = input_file_name.split('_')
+    short_name = f'{parts[0]}_{parts[1]}_{parts[2]}_{parts[3]}'
     ensure_dir_exists(cal_dir)
     output_dir = os.path.join(config["transfer_dir"], "outgoing")
 

From 99bcb7a83bcab8a7c7205ada5b51ab7b1275745c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 12:55:54 -0400
Subject: [PATCH 172/256] Remove unused badpix values from columndatasource

---
 .../common_monitors/dark_monitor.py                 | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 8ffab9146..75cdf1062 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -649,8 +649,7 @@ def overplot_bad_pix(self, pix_type, coords, values):
             values = []
 
         sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0],
-                                                       pixels_y=coords[1],
-                                                       values=values
+                                                       pixels_y=coords[1]
                                                       )
                                              )
 
@@ -658,14 +657,6 @@ def overplot_bad_pix(self, pix_type, coords, values):
         badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y',
                                                  source=sources[pix_type], color=colors[pix_type])
 
-        # Create hover tools for the bad pixel types
-        #hover_tools[pix_type] = HoverTool(tooltips=[(f'{pix_type} (x, y):', '(@pixels_x, @pixels_y)'),
-        #                                            ('value:', f'@values'),
-        #                                            ],
-        #                                    renderers=[badpixplots[pix_type]])
-        # Add tool to plot
-        #self.plot.tools.append(hover_tools[pix_type])
-
         # Add to the legend
         if  numpix > 0:
             if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT:
@@ -814,7 +805,7 @@ def process(self, file_list):
                 logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0])))
                 self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time)
 
-            logging.info("Creating Mean Slope Image {}".format(slope_image))
+            logging.info("Creating Mean Slope Image")
             # Create png file of mean slope image. Add bad pixels only for full frame apertures
             self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix,
                                           noisyxy=new_noisy_pixels, baseline_file=baseline_file)

From 9f45a404b5ea7f0f3e6fb5021e1800848c5fa7b0 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 13:22:53 -0400
Subject: [PATCH 173/256] Set first pipeline step name per instrument

---
 jwql/shared_tasks/run_pipeline.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index a79167e0b..0531f0337 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -61,7 +61,7 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # If the input file is a file other than uncal.fits, then we may only need to run a
         # subset of steps. Check the completed steps in the input file. Find the latest step
         # that has been completed, and skip that plus all prior steps
-        if 'uncal.fits' not in input_file:
+        if 'uncal' not in input_file:
             completed_steps = completed_pipeline_steps(input_file)
 
             # Reverse the boolean value, so that now steps answers the question: "Do we need
@@ -73,7 +73,11 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         # run, and only run subsequent steps. This protects against cases where some early
         # step was not run. In that case, we don't want to go back and run it because running
         # pipeline steps out of order doesn't work.
-        last_run = 'group_scale'  # initialize to the first step
+        if instrument in ['miri', 'nirspec']:
+            last_run = 'group_scale'  # initialize to the first step
+        else:
+            last_run = 'dq_init'
+
         for step in steps:
             if not steps[step]:
                 last_run = deepcopy(step)

From 0b4791ba9024c6a9828c82edadda3f42d453dae8 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 15:02:05 -0400
Subject: [PATCH 174/256] avoid logging large numpy arrays

---
 jwql/instrument_monitors/common_monitors/bias_monitor.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/jwql/instrument_monitors/common_monitors/bias_monitor.py b/jwql/instrument_monitors/common_monitors/bias_monitor.py
index 120cfab0e..9e1f9ad32 100755
--- a/jwql/instrument_monitors/common_monitors/bias_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/bias_monitor.py
@@ -432,7 +432,13 @@ def process(self, file_list):
             # Add this new entry to the bias database table
             with engine.begin() as connection:
                 connection.execute(self.stats_table.__table__.insert(), bias_db_entry)
-            logging.info('\tNew entry added to bias database table: {}'.format(bias_db_entry))
+
+            # Don't print long arrays of numbers to the log file
+            log_dict = {}
+            for key in bias_db_entry:
+                if key not in ['collapsed_rows', 'collapsed_columns', 'counts', 'bin_centers']:
+                    log_dict[key] = bias_db_entry[key]
+            logging.info('\tNew entry added to bias database table: {}'.format(log_dict))
 
             # Remove the raw and calibrated files to save memory space
             os.remove(filename)

From 8737521c9ab8c4ea45389faf47ef4b27ef0d94e6 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 15:36:02 -0400
Subject: [PATCH 175/256] Make plotting functions robust against missing db
 entries

---
 .../jwql/monitor_pages/monitor_bias_bokeh.py  | 54 +++++++++++--------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index 7f14c9fb2..35cd6dbe0 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -466,31 +466,39 @@ def create_plot(self, colname):
             title_text = 'Column'
             axis_text = 'Row Number'
 
-        #datestr = self.data['expstart'][0].strftime("%m/%d/%Y")
-        datestr = self.data['expstart_str'].iloc[0]
-        title_str = f'Calibrated data: Collapsed {title_text}, {datestr}'
-
-        if len(self.data[colname].iloc[0]) > 0:
-            plot = figure(title=title_str, tools='pan,box_zoom,reset,wheel_zoom,save',
-                          background_fill_color="#fafafa")
-
-            # Add a column containing pixel numbers to plot against
-            pix_num = np.arange(len(self.data[colname].iloc[0]))
-            self.data['pixel'] = [pix_num]
-
-            series = self.data.iloc[0]
-            series = series[['pixel', colname]]
-            source = ColumnDataSource(dict(series))
-            plot.scatter(x='pixel', y=colname, fill_color="#C85108", line_color="#C85108",
-                         alpha=0.75, source=source)
-
-            hover_text = axis_text.split(' ')[0]
-            hover_tool = HoverTool(tooltips=f'{hover_text} @pixel: @{colname}')
-            plot.tools.append(hover_tool)
-            plot.xaxis.axis_label = axis_text
-            plot.yaxis.axis_label = 'Median Signal (DN)'
+        # Make sure there is data present
+        if len(self.data) > 0:
+            # Make sure that the colname column is not empty
+            if len(self.data[colname].iloc[0]) > 0:
+                datestr = self.data['expstart_str'].iloc[0]
+                title_str = f'Calibrated data: Collapsed {title_text}, {datestr}'
+
+                plot = figure(title=title_str, tools='pan,box_zoom,reset,wheel_zoom,save',
+                              background_fill_color="#fafafa")
+
+                # Add a column containing pixel numbers to plot against
+                pix_num = np.arange(len(self.data[colname].iloc[0]))
+                self.data['pixel'] = [pix_num]
+
+                series = self.data.iloc[0]
+                series = series[['pixel', colname]]
+                source = ColumnDataSource(dict(series))
+                plot.scatter(x='pixel', y=colname, fill_color="#C85108", line_color="#C85108",
+                             alpha=0.75, source=source)
+
+                hover_text = axis_text.split(' ')[0]
+                hover_tool = HoverTool(tooltips=f'{hover_text} @pixel: @{colname}')
+                plot.tools.append(hover_tool)
+                plot.xaxis.axis_label = axis_text
+                plot.yaxis.axis_label = 'Median Signal (DN)'
+            else:
+                # If there is a latest_data entry, but the collapsed_row or collapsed_col
+                # columns are empty, then make a placeholder plot.
+                title_str = f'Calibrated data: Collapsed {title_text}'
+                plot = PlaceholderPlot(title_str, axis_text, 'Median Signal (DN)').plot
         else:
             # If there are no data, then create an empty placeholder plot
+            title_str = f'Calibrated data: Collapsed {title_text}'
             plot = PlaceholderPlot(title_str, axis_text, 'Median Signal (DN)').plot
 
         return plot

From 8dd5bbd1a57f226f5e12bfcd45431c2b234bfb94 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 15:43:49 -0400
Subject: [PATCH 176/256] Make other plotting functions robust against missing
 db entries

---
 .../jwql/monitor_pages/monitor_bias_bokeh.py  | 53 ++++++++++---------
 1 file changed, 29 insertions(+), 24 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index 35cd6dbe0..47a85b8b8 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -382,31 +382,36 @@ def create_plot(self):
         """
         x_label = 'Signal (DN)'
         y_label = '# Pixels'
-        if len(self.data['counts'].iloc[0]) > 0:
-
-            # In order to use Bokeh's quad, we need left and right bin edges, rather than bin centers
-            bin_centers = np.array(self.data['bin_centers'][0])
-            half_widths = (bin_centers[1:] - bin_centers[0:-1]) / 2
-            half_widths = np.insert(half_widths, 0, half_widths[0])
-            self.data['bin_left'] = [bin_centers - half_widths]
-            self.data['bin_right'] = [bin_centers + half_widths]
-
-            datestr = self.data['expstart_str'].iloc[0]
-            self.plot = figure(title=f'Calibrated data: Histogram, {datestr}', tools='pan,box_zoom,reset,wheel_zoom,save',
-                               background_fill_color="#fafafa")
-
-            # Keep only the columns where the data are a list
-            series = self.data.iloc[0]
-            series = series[['counts', 'bin_left', 'bin_right', 'bin_centers']]
-            source = ColumnDataSource(dict(series))
-            self.plot.quad(top='counts', bottom=0, left='bin_left', right='bin_right',
-                           fill_color="#C85108", line_color="#C85108", alpha=0.75, source=source)
-
-            hover_tool = HoverTool(tooltips=f'@bin_centers DN: @counts')
-            self.plot.tools.append(hover_tool)
-            self.plot.xaxis.axis_label = x_label
-            self.plot.yaxis.axis_label = y_label
+        # Be sure data is not empty
+        if len(data) > 0:
+            # Be sure the array of histogram information is not empty
+            if len(self.data['counts'].iloc[0]) > 0:
+
+                # In order to use Bokeh's quad, we need left and right bin edges, rather than bin centers
+                bin_centers = np.array(self.data['bin_centers'][0])
+                half_widths = (bin_centers[1:] - bin_centers[0:-1]) / 2
+                half_widths = np.insert(half_widths, 0, half_widths[0])
+                self.data['bin_left'] = [bin_centers - half_widths]
+                self.data['bin_right'] = [bin_centers + half_widths]
 
+                datestr = self.data['expstart_str'].iloc[0]
+                self.plot = figure(title=f'Calibrated data: Histogram, {datestr}', tools='pan,box_zoom,reset,wheel_zoom,save',
+                                   background_fill_color="#fafafa")
+
+                # Keep only the columns where the data are a list
+                series = self.data.iloc[0]
+                series = series[['counts', 'bin_left', 'bin_right', 'bin_centers']]
+                source = ColumnDataSource(dict(series))
+                self.plot.quad(top='counts', bottom=0, left='bin_left', right='bin_right',
+                               fill_color="#C85108", line_color="#C85108", alpha=0.75, source=source)
+
+                hover_tool = HoverTool(tooltips=f'@bin_centers DN: @counts')
+                self.plot.tools.append(hover_tool)
+                self.plot.xaxis.axis_label = x_label
+                self.plot.yaxis.axis_label = y_label
+
+            else:
+                self.plot = PlaceholderPlot('Calibrated data: Histogram', x_label, y_label).plot
         else:
             self.plot = PlaceholderPlot('Calibrated data: Histogram', x_label, y_label).plot
 

From 40bd0dae87e1c704c87914b56ae55c81d7ded34c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 2 Aug 2023 15:46:38 -0400
Subject: [PATCH 177/256] typo

---
 jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
index 47a85b8b8..4b5a6a6de 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bias_bokeh.py
@@ -383,7 +383,7 @@ def create_plot(self):
         x_label = 'Signal (DN)'
         y_label = '# Pixels'
         # Be sure data is not empty
-        if len(data) > 0:
+        if len(self.data) > 0:
             # Be sure the array of histogram information is not empty
             if len(self.data['counts'].iloc[0]) > 0:
 

From 45c1f8f64ade3edb4c1818a2334dd8f1c6ee593d Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 4 Aug 2023 14:02:46 -0400
Subject: [PATCH 178/256] Check fidelity of file copy from mast

---
 .../common_monitors/dark_monitor.py                 | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 75cdf1062..c1c2ecc4c 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -1022,6 +1022,19 @@ def run(self):
                             # Copy files from filesystem
                             dark_files, not_copied = copy_files(new_filenames, self.data_dir)
 
+                            # Check that there were no problems with the file copying. If any of the copied
+                            # files have different sizes between the MAST filesystem and the JWQL filesystem,
+                            # then throw them out.
+                            for dark_file in dark_files:
+                                copied_size = os.stat(dark_file).st_size
+                                orig_size = os.stat(filesystem_path(os.path.basename(dark_file))).st_size
+                                if orig_size != copied_size:
+                                    logging.info(f"\tProblem copying {os.path.basename(dark_file)} from the filesystem.")
+                                    logging.info(f"Size in filesystem: {orig_size}, size of copy: {copied_size}. Skipping file.")
+                                    not_copied.append(dark_file)
+                                    dark_files.remove(dark_file)
+                                    os.remove(dark_file)
+
                             logging.info('\tNew_filenames: {}'.format(new_filenames))
                             logging.info('\tData dir: {}'.format(self.data_dir))
                             logging.info('\tCopied to working dir: {}'.format(dark_files))

From dc7e2720f1ea7675936e76bb2bdaa6ae97702608 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 17 Aug 2023 12:01:15 -0400
Subject: [PATCH 179/256] env updates with correct nodejs

---
 environment_python_3.10.yml | 40 +++++++++++++++----------------
 environment_python_3.9.yml  | 40 +++++++++++++++----------------
 pyproject.toml              |  3 +--
 requirements.txt            | 48 ++++++++++++++++++-------------------
 rtd_requirements.txt        | 20 ++++++++--------
 5 files changed, 75 insertions(+), 76 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 32bef050d..291e94710 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -22,51 +22,51 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.1
+  - astropy=5.3.2
   - bokeh=2.4.3
   - beautifulsoup4=4.12.2
   - celery=5.3.1
-  - cryptography=41.0.2
+  - cryptography=41.0.3
   - django=4.2.3
   - inflection=0.5.1
   - ipython=8.14.0
   - jinja2=3.1.2
-  - jsonschema=4.17.3
+  - jsonschema=4.19.0
   - matplotlib=3.7.2
-  - nodejs=18.16.1
-  - numpy=1.25.1
+  - nodejs=18.16.0
+  - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.0.3
   - pip=23.2.1
-  - postgresql=15.3
-  - psycopg2=2.9.6
+  - postgresql=15.2
+  - psycopg2=2.9.3
   - pytest=7.4.0
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.10.12
   - pyyaml=6.0
   - redis
-  - ruff=0.0.280
+  - ruff=0.0.284
   - scipy=1.9.3
   - setuptools=68.0.0
   - sphinx=6.2.1
   - sphinx_rtd_theme=1.2.2
-  - sqlalchemy=2.0.19
+  - sqlalchemy=2.0.20
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
-    - astroquery=0.4.6
-    - bandit=1.7.5
-    - jwst=1.11.3
-    - pysiaf=0.20.0
-    - pysqlite3=0.5.1
-    - pyvo=1.4.1
-    - redis=4.6.0
-    - selenium=4.10.0
-    - stdatamodels=1.7.1
-    - stsci_rtd_theme=1.0.0
-    - vine=5.0.0
+    - astroquery==0.4.6
+    - bandit==1.7.5
+    - jwst==1.11.4
+    - pysiaf==0.20.0
+    - pysqlite3==0.5.1
+    - pyvo==1.4.2
+    - redis==5.0.0
+    - selenium==4.11.2
+    - stdatamodels==1.7.2
+    - stsci_rtd_theme==1.0.0
+    - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
 
     # Current package
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index dd677fa50..a42b9e770 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -22,51 +22,51 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.1
+  - astropy=5.3.2
   - bokeh=2.4.3
   - beautifulsoup4=4.12.2
   - celery=5.3.1
-  - cryptography=41.0.2
+  - cryptography=41.0.3
   - django=4.2.3
   - inflection=0.5.1
   - ipython=8.14.0
   - jinja2=3.1.2
-  - jsonschema=4.17.3
+  - jsonschema=4.19.0
   - matplotlib=3.7.2
-  - nodejs=18.16.1
-  - numpy=1.25.1
+  - nodejs=18.16.0
+  - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.0.3
   - pip=23.2.1
-  - postgresql=15.3
-  - psycopg2=2.9.6
+  - postgresql=15.2
+  - psycopg2=2.9.3
   - pytest=7.4.0
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.9.17
   - pyyaml=6.0
   - redis
-  - ruff=0.0.280
+  - ruff=0.0.284
   - scipy=1.9.3
   - setuptools=68.0.0
   - sphinx=6.2.1
   - sphinx_rtd_theme=1.2.2
-  - sqlalchemy=2.0.19
+  - sqlalchemy=2.0.20
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
-    - astroquery=0.4.6
-    - bandit=1.7.5
-    - jwst=1.11.3
-    - pysiaf=0.20.0
-    - pysqlite3=0.5.1
-    - pyvo=1.4.1
-    - redis=4.6.0
-    - selenium=4.10.0
-    - stdatamodels=1.7.1
-    - stsci_rtd_theme=1.0.0
-    - vine=5.0.0
+    - astroquery==0.4.6
+    - bandit==1.7.5
+    - jwst==1.11.4
+    - pysiaf==0.20.0
+    - pysqlite3==0.5.1
+    - pyvo==1.4.2
+    - redis==5.0.0
+    - selenium==4.11.2
+    - stdatamodels==1.7.2
+    - stsci_rtd_theme==1.0.0
+    - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
 
     # Current package
diff --git a/pyproject.toml b/pyproject.toml
index f17cefdf9..c17d3dc28 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,7 +27,6 @@ dependencies = [
   "jwst",
   "jwst_reffiles",
   "matplotlib",
-  "nodejs",
   "numpy",
   "numpydoc",
   "pandas",
@@ -46,7 +45,7 @@ test = [
   "pytest",
   "pytest-cov",
   "pytest-mock",
-]  
+]
 docs = [
   "sphinx",
   "sphinx_rtd_theme",
diff --git a/requirements.txt b/requirements.txt
index 501641339..9571c33c9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,38 +1,38 @@
-astropy==5.2.2
+astropy==5.3.2
 astroquery==0.4.6
 bandit==1.7.5
 beautifulsoup4==4.12.2
 bokeh==2.4.3
-celery==5.2.7
-cryptography==40.0.2
-django==4.2.1
+celery==5.3.1
+cryptography==41.0.3
+django==4.2.3
 inflection==0.5.1
-ipython==8.13.2
+ipython==8.14.0
 jinja2==3.1.2
-jsonschema==4.17.3
-jwst==1.10.2
-matplotlib==3.7.0
-nodejs==0.1.1
-numpy==1.24.3
+jsonschema==4.19.0
+jwst==1.11.4
+matplotlib==3.7.2
+nodejs==18.16.0
+numpy==1.25.2
 numpydoc==1.5.0
-pandas==2.0.1
+pandas==2.0.3
 psycopg2-binary==2.9.3
-pysiaf==0.19.1
-pysqlite3==0.5.0
-pytest==7.3.1
-pytest-cov==4.0.0
-pytest-mock==3.10.0
-pyvo==1.4.1
+pysiaf==0.20.0
+pysqlite3==0.5.1
+pytest==7.4.0
+pytest-cov==4.1.0
+pytest-mock==3.11.1
+pyvo==1.4.2
 pyyaml==6.0
-redis==4.5.5
-ruff==0.0.269
+redis==5.0.0
+ruff==0.0.284
 scipy==1.9.3
-selenium==4.9.1
-setuptools==67.7.2
+selenium==4.11.2
+setuptools==68.0.0
 sphinx==6.2.1
-sphinx_rtd_theme==1.2.0
-sqlalchemy==2.0.15
-stdatamodels==1.3.1
+sphinx_rtd_theme==1.2.2
+sqlalchemy==2.0.20
+stdatamodels==1.7.2
 stsci_rtd_theme==1.0.0
 twine==4.0.2
 vine==5.0.0
diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index ba1ecf587..aa1047b5e 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -1,15 +1,15 @@
-sphinx_automodapi==0.14.1
+sphinx_automodapi==0.15.0
 bokeh==2.4.3
-celery==5.2.7
-cython==0.29.33
-django==4.2.1
+celery==5.3.1
+cython==3.0.0
+django==4.2.3
 docutils==0.18.1
-jwst==1.10.2
-pygments==2.14.0
-pytest==7.3.1
-redis==4.5.5
-sphinx>=2
-sphinx_rtd_theme==1.2.0
+jwst==1.11.4
+pygments==2.16.1
+pytest==7.4.0
+redis==5.0.0
+sphinx==6.2.1
+sphinx_rtd_theme==1.2.2
 stsci_rtd_theme==1.0.0
 tomli==2.0.1
 git+https://github.com/spacetelescope/jwst_reffiles

From bc3b546ad514d5b2b73e4279ee3214a9774f2e10 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 17 Aug 2023 12:02:00 -0400
Subject: [PATCH 180/256] Return env name back to nominal

---
 environment_python_3.10.yml | 2 +-
 environment_python_3.9.yml  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 291e94710..f554506b0 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -15,7 +15,7 @@
 # To remove the environment entirely, run the following command:
 # $ conda env remove -n jwql-3.10
 
-name: jwql-3.10-deptest
+name: jwql-3.10
 
 channels:
   - conda-forge
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index a42b9e770..3986b0829 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -15,7 +15,7 @@
 # To remove the environment entirely, run the following command:
 # $ conda env remove -n jwql-3.9
 
-name: jwql-3.9-deptest
+name: jwql-3.9
 
 channels:
   - conda-forge

From a6beb9415c8e2bf6335f24a6f47f42becc4b3078 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 18 Aug 2023 12:29:09 -0400
Subject: [PATCH 181/256] bring nodejs up to most recent version

---
 environment_python_3.10.yml | 14 +++++++-------
 environment_python_3.9.yml  | 14 +++++++-------
 requirements.txt            |  6 +++---
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index f554506b0..f61973680 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -1,10 +1,10 @@
 # This file describes a conda environment that can be to install jwql
 #
 # Run the following command to set up this environment:
-# $ conda env create -f environment_python_3_10.yml
+# $ conda env create -f environment_python_3.10.yml
 #
 # The environment name can be overridden with the following command:
-# $ conda env create -n <custom name> -f environment_python_3_10.yml
+# $ conda env create -n <custom name> -f environment_python_3.10.yml
 #
 # Run the following command to activate the environment:
 # $ source activate jwql-3.10
@@ -23,8 +23,8 @@ channels:
 
 dependencies:
   - astropy=5.3.2
-  - bokeh=2.4.3
   - beautifulsoup4=4.12.2
+  - bokeh=2.4.3
   - celery=5.3.1
   - cryptography=41.0.3
   - django=4.2.3
@@ -33,20 +33,20 @@ dependencies:
   - jinja2=3.1.2
   - jsonschema=4.19.0
   - matplotlib=3.7.2
-  - nodejs=18.16.0
+  - nodejs=20.5.1
   - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.0.3
   - pip=23.2.1
-  - postgresql=15.2
-  - psycopg2=2.9.3
+  - postgresql=15.4
+  - psycopg2=2.9.6
   - pytest=7.4.0
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.10.12
   - pyyaml=6.0
   - redis
-  - ruff=0.0.284
+  - ruff=0.0.285
   - scipy=1.9.3
   - setuptools=68.0.0
   - sphinx=6.2.1
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index 3986b0829..d3724b1d8 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -1,10 +1,10 @@
 # This file describes a conda environment that can be to install jwql
 #
 # Run the following command to set up this environment:
-# $ conda env create -f environment_python_3_9.yml
+# $ conda env create -f environment_python_3.9.yml
 #
 # The environment name can be overridden with the following command:
-# $ conda env create -n <custom name> -f environment_python_3_9.yml
+# $ conda env create -n <custom name> -f environment_python_3.9.yml
 #
 # Run the following command to activate the environment:
 # $ source activate jwql-3.9
@@ -23,8 +23,8 @@ channels:
 
 dependencies:
   - astropy=5.3.2
-  - bokeh=2.4.3
   - beautifulsoup4=4.12.2
+  - bokeh=2.4.3
   - celery=5.3.1
   - cryptography=41.0.3
   - django=4.2.3
@@ -33,20 +33,20 @@ dependencies:
   - jinja2=3.1.2
   - jsonschema=4.19.0
   - matplotlib=3.7.2
-  - nodejs=18.16.0
+  - nodejs=20.5.1
   - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.0.3
   - pip=23.2.1
-  - postgresql=15.2
-  - psycopg2=2.9.3
+  - postgresql=15.4
+  - psycopg2=2.9.6
   - pytest=7.4.0
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.9.17
   - pyyaml=6.0
   - redis
-  - ruff=0.0.284
+  - ruff=0.0.285
   - scipy=1.9.3
   - setuptools=68.0.0
   - sphinx=6.2.1
diff --git a/requirements.txt b/requirements.txt
index 9571c33c9..4391be8b3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,11 +12,11 @@ jinja2==3.1.2
 jsonschema==4.19.0
 jwst==1.11.4
 matplotlib==3.7.2
-nodejs==18.16.0
+nodejs==20.5.1
 numpy==1.25.2
 numpydoc==1.5.0
 pandas==2.0.3
-psycopg2-binary==2.9.3
+psycopg2-binary==2.9.7
 pysiaf==0.20.0
 pysqlite3==0.5.1
 pytest==7.4.0
@@ -25,7 +25,7 @@ pytest-mock==3.11.1
 pyvo==1.4.2
 pyyaml==6.0
 redis==5.0.0
-ruff==0.0.284
+ruff==0.0.285
 scipy==1.9.3
 selenium==4.11.2
 setuptools==68.0.0

From fd7c838616b7518db13ea035cfdd79dcd71638f1 Mon Sep 17 00:00:00 2001
From: "york@stsci.edu" <york@stsci.edu>
Date: Mon, 11 Sep 2023 15:23:09 -0400
Subject: [PATCH 182/256] Adding more logging

---
 jwql/shared_tasks/run_pipeline.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 0531f0337..e73c5bf1e 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -51,18 +51,27 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         status_f.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir)))
         status_f.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file)))
         status_f.write(f"\t outputs is {outputs}\n")
+        sys.stderr.write("Running run_pipe\n")
+        sys.stderr.write("\t input_file_basename is {} ({})\n".format(input_file_basename, type(input_file_basename)))
+        sys.stderr.write("\t start_dir is {} ({})\n".format(start_dir, type(start_dir)))
+        sys.stderr.write("\t uncal_file is {} ({})\n".format(uncal_file, type(uncal_file)))
+        sys.stderr.write(f"\t outputs is {outputs}\n")
 
     try:
+        sys.stderr.write("Copying file {} to working directory.\n".format(input_file))
         copy_files([input_file], work_directory)
+        sys.stderr.write("Setting permissions on {}\n".format(uncal_file))
         set_permissions(uncal_file)
 
         steps = get_pipeline_steps(instrument)
+        sys.stderr.write("Pipeline steps initialized to {}\n".format(steps))
 
         # If the input file is a file other than uncal.fits, then we may only need to run a
         # subset of steps. Check the completed steps in the input file. Find the latest step
         # that has been completed, and skip that plus all prior steps
         if 'uncal' not in input_file:
             completed_steps = completed_pipeline_steps(input_file)
+            sys.stderr.write("Steps {} already completed.\n".format(completed_steps))
 
             # Reverse the boolean value, so that now steps answers the question: "Do we need
             # to run this step?""
@@ -80,18 +89,21 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
 
         for step in steps:
             if not steps[step]:
+                sys.stderr.write("Setting last_run to {}.\n".format(step))
                 last_run = deepcopy(step)
 
         for step in steps:
             if step == last_run:
                 break
             if step != last_run:
+                sys.stderr.write("Setting {} to skip while looking for last_run.\n".format(step))
                 steps[step] = False
 
         # Set any steps the user specifically asks to skip
         for step, step_dict in step_args.items():
             if 'skip' in step_dict:
                 if step_dict['skip']:
+                    sys.stderr.write("Setting step {} to skip by user request.\n".format(step))
                     steps[step] = False
 
         # Run each specified step

From 1834232cfa409f4339b333b368fd9ea2ddb308ce Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Wed, 13 Sep 2023 15:12:15 -0400
Subject: [PATCH 183/256] Add step_args to calwebb_detector1_save_jump

---
 jwql/shared_tasks/shared_tasks.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index be232935d..1e8e95549 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -408,7 +408,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
 
 
 @celery_app.task(name='jwql.shared_tasks.shared_tasks.calwebb_detector1_save_jump')
-def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True):
+def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True, step_args={}):
     """Call ``calwebb_detector1`` on the provided file, running all
     steps up to the ``ramp_fit`` step, and save the result. Optionally
     run the ``ramp_fit`` step and save the resulting slope file as well.
@@ -430,6 +430,13 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
         If ``True``, the file of optional outputs from the ramp fitting
         step of the pipeline is saved.
 
+    step_args : dict
+        A dictionary containing custom arguments to supply to individual pipeline steps.
+        When a step is run, the dictionary will be checked for a key matching the step
+        name (as defined in jwql.utils.utils.get_pipeline_steps() for the provided
+        instrument). The value matching the step key should, itself, be a dictionary that
+        can be spliced in to step.call() via dereferencing (**dict)
+
     Returns
     -------
     jump_output : str
@@ -469,7 +476,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
 
     cores = 'all'
     status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file,
-                            short_name, result_file, cores)
+                            short_name, result_file, cores, step_args)
 
     if status[-1].strip() == "SUCCEEDED":
         logging.info("Subprocess reports successful finish.")
@@ -484,7 +491,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
         if core_fail:
             cores = "half"
             status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
-                                    input_file, short_name, result_file, cores)
+                                    input_file, short_name, result_file, cores, step_args)
             if status[-1].strip() == "SUCCEEDED":
                 logging.info("Subprocess reports successful finish.")
                 managed = True
@@ -498,7 +505,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
                 if core_fail:
                     cores = "none"
                     status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument,
-                                            input_file, short_name, result_file, cores)
+                                            input_file, short_name, result_file, cores, step_args)
                     if status[-1].strip() == "SUCCEEDED":
                         logging.info("Subprocess reports successful finish.")
                         managed = True

From 477072f37d93b36ed907d77d43204d74f1f600f8 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 21 Sep 2023 15:22:52 -0400
Subject: [PATCH 184/256] create wrapper func for saving png files

---
 .../common_monitors/dark_monitor.py           |  5 ++--
 jwql/utils/utils.py                           | 23 ++++++++++++++++++-
 .../monitor_pages/monitor_bad_pixel_bokeh.py  |  6 ++---
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index c1c2ecc4c..467a8bd1c 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -84,7 +84,6 @@
 from astropy.modeling import models
 from astropy.stats import sigma_clipped_stats
 from astropy.time import Time
-from bokeh.io import export_png
 from bokeh.models import ColorBar, ColumnDataSource, HoverTool, Legend
 from bokeh.models import LinearColorMapper
 from bokeh.plotting import figure
@@ -106,7 +105,7 @@
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
+from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png
 
 THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt')
 
@@ -350,7 +349,7 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
                 self.plot.add_layout(legend, 'below')
 
             # Save the plot in a png
-            export_png(self.plot, filename=output_filename)
+            save_png(self.plot, filename=output_filename)
             set_permissions(output_filename)
 
 
diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
index 33d0c272c..75cf49ca6 100644
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -45,6 +45,7 @@
 from bokeh.plotting import figure
 import numpy as np
 from PIL import Image
+from selenium import webdriver
 
 from jwql.utils import permissions
 from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \
@@ -170,7 +171,7 @@ def create_png_from_fits(filename, outdir):
 
         # Save the plot in a png
         output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png'))
-        export_png(plot, filename=output_filename)
+        save_png(plot, filename=output_filename)
         permissions.set_permissions(output_filename)
         return output_filename
     else:
@@ -756,6 +757,26 @@ def read_png(filename):
     return img
 
 
+def save_png(fig, filename=''):
+    """Starting with selenium version 4.10.0, our testing has shown that on the JWQL
+    servers, we need to specify an instance of a web driver when exporting a Bokeh
+    figure as a png. This is a wrapper function that creates the web driver instance
+    and calls Bokeh's export_png function.
+
+    Parameters
+    ----------
+    fig : bokeh.plotting.figure
+        Bokeh figure to be saved as a png
+
+    filename : str
+        Filename to use for the png file
+    """
+    options = webdriver.FirefoxOptions()
+    options.add_argument('-headless')
+    driver = webdriver.Firefox(options=options)
+    export_png(fig, filename=filename, webdriver=driver)
+
+
 def grouper(iterable, chunksize):
     """
     Take a list of items (iterable), and group it into chunks of chunksize, with the
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
index 48c47e216..b172df2d4 100755
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
@@ -22,7 +22,7 @@
 from astropy.stats import sigma_clipped_stats
 from astropy.time import Time
 from bokeh.embed import components, file_html
-from bokeh.io import export_png, show
+from bokeh.io import show
 from bokeh.layouts import layout
 from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text, Title
 from bokeh.plotting import figure
@@ -40,7 +40,7 @@
 from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES
 from jwql.utils.constants import DETECTOR_PER_INSTRUMENT, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import filesystem_path, get_config, read_png
+from jwql.utils.utils import filesystem_path, get_config, read_png, save_png
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 OUTPUT_DIR = get_config()['outputs']
@@ -593,7 +593,7 @@ def switch_to_png(self, filename, title):
             Title to add to the Figure
         """
         # Save the figure as a png
-        export_png(self.plot, filename=filename)
+        save_png(self.plot, filename=filename)
         set_permissions(filename)
 
         # Read in the png and insert into a replacement figure

From 03da9a8916d3dc438a7caaae96c0710a2708092b Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 21 Sep 2023 15:38:38 -0400
Subject: [PATCH 185/256] Quit driver when done. Add test

---
 jwql/tests/test_utils.py | 19 ++++++++++++++++++-
 jwql/utils/utils.py      |  1 +
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py
index c2f6795c1..27bee1b5e 100644
--- a/jwql/tests/test_utils.py
+++ b/jwql/tests/test_utils.py
@@ -22,7 +22,11 @@
 from pathlib import Path
 import pytest
 
-from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, _validate_config
+from bokeh.models import LinearColorMapper
+from bokeh.plotting import figure
+import numpy as np
+
+from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, save_png, _validate_config
 
 
 # Determine if tests are being run on Github Actions
@@ -479,6 +483,19 @@ def test_filesystem_path():
     assert check == location
 
 
+def test_save_png():
+    """Test that we can create a png file"""
+    plot = figure(title='test',tools='')
+    image = np.zeros((200,200))
+    image[100:105, 100:105] = 1
+    ny, nx = image.shape
+    mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1)
+    imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image")
+    #save_png(plot, filename='test.png')
+    from bokeh.io import export_png
+    export_png(plot, filename='test.png')
+
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_validate_config():
     """Test that the config validator works."""
diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
index 75cf49ca6..182c18191 100644
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -775,6 +775,7 @@ def save_png(fig, filename=''):
     options.add_argument('-headless')
     driver = webdriver.Firefox(options=options)
     export_png(fig, filename=filename, webdriver=driver)
+    driver.quit()
 
 
 def grouper(iterable, chunksize):

From f239ef2ccc290c3f05c657611e0d7903044fcf7a Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 21 Sep 2023 15:54:07 -0400
Subject: [PATCH 186/256] RTD needs selenium now

---
 rtd_requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index aa1047b5e..d4b88e1fd 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -8,6 +8,7 @@ jwst==1.11.4
 pygments==2.16.1
 pytest==7.4.0
 redis==5.0.0
+selenium==4.11.3
 sphinx==6.2.1
 sphinx_rtd_theme==1.2.2
 stsci_rtd_theme==1.0.0

From 23d02e79a7a3dd9752c4505fdccc2a36f791059c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 21 Sep 2023 16:05:08 -0400
Subject: [PATCH 187/256] Fix test. Adjust rtd version

---
 jwql/tests/test_utils.py | 4 +---
 rtd_requirements.txt     | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py
index 27bee1b5e..813bb0432 100644
--- a/jwql/tests/test_utils.py
+++ b/jwql/tests/test_utils.py
@@ -491,9 +491,7 @@ def test_save_png():
     ny, nx = image.shape
     mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1)
     imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image")
-    #save_png(plot, filename='test.png')
-    from bokeh.io import export_png
-    export_png(plot, filename='test.png')
+    save_png(plot, filename='test.png')
 
 
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index d4b88e1fd..39500c2b6 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -8,7 +8,7 @@ jwst==1.11.4
 pygments==2.16.1
 pytest==7.4.0
 redis==5.0.0
-selenium==4.11.3
+selenium==4.11.2
 sphinx==6.2.1
 sphinx_rtd_theme==1.2.2
 stsci_rtd_theme==1.0.0

From b85a1b0f7080afc1a58a1e8070b22377cf878b58 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 21 Sep 2023 21:21:42 +0000
Subject: [PATCH 188/256] Bump cryptography from 41.0.3 to 41.0.4

Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 4391be8b3..d4f3a5616 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ bandit==1.7.5
 beautifulsoup4==4.12.2
 bokeh==2.4.3
 celery==5.3.1
-cryptography==41.0.3
+cryptography==41.0.4
 django==4.2.3
 inflection==0.5.1
 ipython==8.14.0

From 2f0e8258c8edb0952c1b95c6a80bcf5470b4d3b6 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Fri, 22 Sep 2023 12:37:11 -0400
Subject: [PATCH 189/256] Better logging. Fix param def. Allow empty dark file
 list

---
 .../common_monitors/bad_pixel_monitor.py                  | 7 ++++---
 jwql/shared_tasks/run_pipeline.py                         | 8 ++++++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py
index 546fc6d80..cc4b7fb68 100755
--- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py
@@ -984,9 +984,10 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun
                 else:
                     index += 1
 
-            min_dark_time = min(dark_obstimes)
-            max_dark_time = max(dark_obstimes)
-            mid_dark_time = instrument_properties.mean_time(dark_obstimes)
+            if len(dark_slope_files) > 0:
+                min_dark_time = min(dark_obstimes)
+                max_dark_time = max(dark_obstimes)
+                mid_dark_time = instrument_properties.mean_time(dark_obstimes)
 
         # Check whether there are still enough files left to meet the threshold
         if illuminated_slope_files is None:
diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 0531f0337..82ea0689e 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -10,6 +10,7 @@
 import shutil
 import sys
 import time
+import traceback
 
 from jwst import datamodels
 from jwst.dq_init import DQInitStep
@@ -161,7 +162,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         with open(status_file, "a+") as status_f:
             status_f.write("EXCEPTION\n")
             status_f.write("{}\n".format(e))
-            status_f.write("FAILED")
+            status_f.write("FAILED\n")
+            status_f.write(traceback.format_exc())
         sys.exit(1)
 
     with open(status_file, "a+") as status_f:
@@ -217,6 +219,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
             params['refpix'] = dict(odd_even_rows=False)
 
         # Default CR rejection threshold is too low
+        params['jump'] = {}
         params['jump']['rejection_threshold'] = 15
 
         # Set up to save jump step output
@@ -284,7 +287,8 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         with open(status_file, "a+") as status_f:
             status_f.write("EXCEPTION\n")
             status_f.write("{}\n".format(e))
-            status_f.write("FAILED")
+            status_f.write("FAILED\n")
+            status_f.write(traceback.format_exc())
         sys.exit(1)
 
     with open(status_file, "a+") as status_f:

From 7ce0d9c89e9ace520c98f2487762e5cb208156e4 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 25 Sep 2023 13:19:07 -0400
Subject: [PATCH 190/256] Fix typo in env file names in readme

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 70b4e173b..377005a11 100644
--- a/README.md
+++ b/README.md
@@ -83,16 +83,16 @@ source activate base/root
 
 **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step.
 
-Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
+Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
 
 ```
-conda env create -f environment_python_3_9.yml
+conda env create -f environment_python_3.9.yml
 ```
 
 or
 
 ```
-conda env create -f environment_python_3_10.yml
+conda env create -f environment_python_3.10.yml
 ```
 
 ### Configuration File

From 55b12f399cc7504e05879157e2c3bfae7b85866c Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Mon, 25 Sep 2023 13:49:40 -0400
Subject: [PATCH 191/256] pep8 fixes

---
 jwql/shared_tasks/run_pipeline.py |  2 +-
 jwql/shared_tasks/shared_tasks.py | 19 ++++++++++---------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py
index 82ea0689e..991e08d47 100755
--- a/jwql/shared_tasks/run_pipeline.py
+++ b/jwql/shared_tasks/run_pipeline.py
@@ -348,7 +348,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     outputs = args.outputs
     step_args = args.step_args
 
-    status_file = os.path.join(working_path, short_name+"_status.txt")
+    status_file = os.path.join(working_path, short_name + "_status.txt")
     with open(status_file, 'w') as out_file:
         out_file.write("Starting Process\n")
         out_file.write("\tpipeline is {} ({})\n".format(pipe_type, type(pipe_type)))
diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py
index 1e8e95549..f076e2035 100644
--- a/jwql/shared_tasks/shared_tasks.py
+++ b/jwql/shared_tasks/shared_tasks.py
@@ -1,4 +1,4 @@
- #! /usr/bin/env python
+#! /usr/bin/env python
 
 """This module contains code for the celery application, which is used for any demanding
 work which should be restricted in terms of how many iterations are run simultaneously, or
@@ -204,7 +204,7 @@ def log_subprocess_output(pipe):
     If a subprocess STDOUT has been set to subprocess.PIPE, this function will log each
     line to the logging output.
     """
-    for line in iter(pipe.readline, b''): # b'\n'-separated lines
+    for line in iter(pipe.readline, b''):  # b'\n'-separated lines
         logging.info("\t{}".format(line.decode('UTF-8').strip()))
 
 
@@ -224,6 +224,7 @@ def after_setup_celery_logger(logger, **kwargs):
 def collect_after_task(**kwargs):
     gc.collect()
 
+
 def convert_step_args_to_string(args_dict):
     """Convert the nested dictionary containing pipeline step parameter keyword/value pairs
     to a string so that it can be passed via command line
@@ -239,17 +240,17 @@ def convert_step_args_to_string(args_dict):
     args_str : str
         String representation of ``args_dict``
     """
-    args_str="'{"
+    args_str = "'{"
 
     for i, step in enumerate(args_dict):
         args_str += f'"{step}":'
         args_str += '{'
         for j, (param, val) in enumerate(args_dict[step].items()):
             args_str += f'"{param}":"{val}"'
-            if j < len(args_dict[step])-1:
+            if j < len(args_dict[step]) - 1:
                 args_str += ', '
         args_str += "}"
-        if i < len(args_dict)-1:
+        if i < len(args_dict) - 1:
             args_str += ','
     args_str += "}'"
     return args_str
@@ -331,7 +332,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
     current_dir = os.path.dirname(__file__)
     cmd_name = os.path.join(current_dir, "run_pipeline.py")
     outputs = ",".join(ext_or_exts)
-    result_file = os.path.join(cal_dir, short_name+"_status.txt")
+    result_file = os.path.join(cal_dir, short_name + "_status.txt")
     if "all" in ext_or_exts:
         logging.info("All outputs requested")
         if instrument.lower() != 'miri':
@@ -399,7 +400,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument,
         set_permissions(os.path.join(output_dir, file))
 
     logging.info("Removing local files.")
-    files_to_remove = glob(os.path.join(cal_dir, short_name+"*"))
+    files_to_remove = glob(os.path.join(cal_dir, short_name + "*"))
     for file_name in files_to_remove:
         logging.info("\tRemoving {}".format(file_name))
         os.remove(file_name)
@@ -472,7 +473,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
     output_dir = os.path.join(config["transfer_dir"], "outgoing")
 
     cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py")
-    result_file = os.path.join(cal_dir, short_name+"_status.txt")
+    result_file = os.path.join(cal_dir, short_name + "_status.txt")
 
     cores = 'all'
     status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file,
@@ -531,7 +532,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save
                 files["fitopt_output"] = os.path.join(output_dir, file)
 
     logging.info("Removing local files.")
-    files_to_remove = glob(os.path.join(cal_dir, short_name+"*"))
+    files_to_remove = glob(os.path.join(cal_dir, short_name + "*"))
     for file_name in files_to_remove:
         logging.info("\tRemoving {}".format(file_name))
         os.remove(file_name)

From ebf05a9e22ca555ee3f7d9b48fcd48dd0068de5c Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Thu, 28 Sep 2023 10:59:37 -0400
Subject: [PATCH 192/256] Include migrations in source control

---
 .../apps/jwql/migrations/0001_initial.py      | 45 +++++++++
 .../migrations/0002_auto_20220913_1525.py     | 27 ++++++
 .../migrations/0003_auto_20220921_0955.py     | 58 +++++++++++
 .../migrations/0004_auto_20220922_0911.py     | 33 +++++++
 .../migrations/0005_auto_20220922_1422.py     | 23 +++++
 .../migrations/0006_auto_20230214_1624.py     | 95 +++++++++++++++++++
 .../migrations/0007_auto_20230222_1157.py     | 22 +++++
 .../migrations/0008_rootfileinfo_exp_type.py  | 18 ++++
 .../migrations/0009_auto_20230303_0930.py     | 63 ++++++++++++
 .../migrations/0010_auto_20230313_1053.py     | 23 +++++
 jwql/website/apps/jwql/migrations/__init__.py |  0
 11 files changed, 407 insertions(+)
 create mode 100644 jwql/website/apps/jwql/migrations/0001_initial.py
 create mode 100644 jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py
 create mode 100644 jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py
 create mode 100644 jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py
 create mode 100644 jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py
 create mode 100644 jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py
 create mode 100644 jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py
 create mode 100644 jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py
 create mode 100644 jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py
 create mode 100644 jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py
 create mode 100644 jwql/website/apps/jwql/migrations/__init__.py

diff --git a/jwql/website/apps/jwql/migrations/0001_initial.py b/jwql/website/apps/jwql/migrations/0001_initial.py
new file mode 100644
index 000000000..9de5ca6a9
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0001_initial.py
@@ -0,0 +1,45 @@
+# Generated by Django 3.1.7 on 2022-09-09 17:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ImageData',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('inst', models.CharField(choices=[('FGS', 'FGS'), ('MIRI', 'MIRI'), ('NIRCam', 'NIRCam'), ('NIRISS', 'NIRISS'), ('NIRSpec', 'NIRSpec')], default=None, max_length=7, verbose_name='instrument')),
+                ('pub_date', models.DateTimeField(verbose_name='date published')),
+                ('filepath', models.FilePathField(path='/user/lchambers/jwql/')),
+            ],
+            options={
+                'verbose_name_plural': 'image data',
+                'db_table': 'imagedata',
+            },
+        ),
+        migrations.CreateModel(
+            name='InstrumentFilterHandler',
+            fields=[
+                ('instrument', models.CharField(max_length=10, primary_key=True, serialize=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ThumbnailFilterInfo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('observation', models.PositiveIntegerField()),
+                ('proposal', models.PositiveIntegerField()),
+                ('root_name', models.CharField(max_length=300)),
+                ('marked_viewed', models.BooleanField(default=False)),
+                ('inst_handler', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.instrumentfilterhandler')),
+            ],
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py
new file mode 100644
index 000000000..049901811
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py
@@ -0,0 +1,27 @@
+# Generated by Django 3.1.7 on 2022-09-13 20:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='thumbnailfilterinfo',
+            name='id',
+        ),
+        migrations.AlterField(
+            model_name='instrumentfilterhandler',
+            name='instrument',
+            field=models.TextField(max_length=10, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='thumbnailfilterinfo',
+            name='root_name',
+            field=models.TextField(max_length=300, primary_key=True, serialize=False),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py
new file mode 100644
index 000000000..df7bc6965
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py
@@ -0,0 +1,58 @@
+# Generated by Django 3.1.7 on 2022-09-21 14:55
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0002_auto_20220913_1525'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Archive',
+            fields=[
+                ('instrument', models.CharField(help_text='Instrument name', max_length=7, primary_key=True, serialize=False)),
+            ],
+            options={
+                'ordering': ['instrument'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Observation',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('obsnum', models.CharField(help_text='Observation number, as a 3 digit string', max_length=3)),
+                ('number_of_files', models.IntegerField(default=0, help_text='Number of files in the proposal')),
+                ('obsstart', models.FloatField(default=0.0, help_text='Time of the beginning of the observation in MJD')),
+                ('obsend', models.FloatField(default=0.0, help_text='Time of the end of the observation in MJD')),
+                ('exptypes', models.CharField(default='', help_text='Comma-separated list of exposure types', max_length=100)),
+            ],
+            options={
+                'ordering': ['-obsnum'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Proposal',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('prop_id', models.CharField(help_text='5-digit proposal ID string', max_length=5)),
+                ('thumbnail_path', models.CharField(default='', help_text='Path to the proposal thumbnail', max_length=100)),
+                ('archive', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.archive')),
+            ],
+            options={
+                'ordering': ['-prop_id'],
+                'unique_together': {('prop_id', 'archive')},
+            },
+        ),
+        migrations.DeleteModel(
+            name='ImageData',
+        ),
+        migrations.AddField(
+            model_name='observation',
+            name='proposal',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.proposal'),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py
new file mode 100644
index 000000000..998dd2ce4
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py
@@ -0,0 +1,33 @@
+# Generated by Django 3.1.7 on 2022-09-22 14:11
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0003_auto_20220921_0955'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='thumbnailfilterinfo',
+            name='obsnum',
+            field=models.CharField(default=11, help_text='Observation number, as a 3 digit string', max_length=3),
+            preserve_default=False,
+        ),
+        migrations.CreateModel(
+            name='RootFileInfo',
+            fields=[
+                ('instrument', models.CharField(help_text='Instrument name', max_length=7)),
+                ('proposal', models.CharField(help_text='5-digit proposal ID string', max_length=5)),
+                ('root_name', models.TextField(max_length=300, primary_key=True, serialize=False)),
+                ('viewed', models.BooleanField(default=False)),
+                ('obsnum', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.observation')),
+            ],
+            options={
+                'ordering': ['-root_name'],
+            },
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py
new file mode 100644
index 000000000..3c51cbe23
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.1.7 on 2022-09-22 19:22
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0004_auto_20220922_0911'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='thumbnailfilterinfo',
+            name='inst_handler',
+        ),
+        migrations.DeleteModel(
+            name='InstrumentFilterHandler',
+        ),
+        migrations.DeleteModel(
+            name='ThumbnailFilterInfo',
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py
new file mode 100644
index 000000000..493608df3
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py
@@ -0,0 +1,95 @@
+# Generated by Django 3.1.7 on 2023-02-14 21:24
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0005_auto_20220922_1422'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Anomalies',
+            fields=[
+                ('root_file_info', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='jwql.rootfileinfo')),
+                ('cosmic_ray_shower', models.BooleanField(default=False)),
+                ('diffraction_spike', models.BooleanField(default=False)),
+                ('excessive_saturation', models.BooleanField(default=False)),
+                ('guidestar_failure', models.BooleanField(default=False)),
+                ('persistence', models.BooleanField(default=False)),
+                ('crosstalk', models.BooleanField(default=False)),
+                ('data_transfer_error', models.BooleanField(default=False)),
+                ('ghost', models.BooleanField(default=False)),
+                ('snowball', models.BooleanField(default=False)),
+                ('column_pull_up', models.BooleanField(default=False)),
+                ('column_pull_down', models.BooleanField(default=False)),
+                ('dominant_msa_leakage', models.BooleanField(default=False)),
+                ('dragons_breath', models.BooleanField(default=False)),
+                ('mrs_glow', models.BooleanField(default=False)),
+                ('mrs_zipper', models.BooleanField(default=False)),
+                ('internal_reflection', models.BooleanField(default=False)),
+                ('optical_short', models.BooleanField(default=False)),
+                ('row_pull_up', models.BooleanField(default=False)),
+                ('row_pull_down', models.BooleanField(default=False)),
+                ('lrs_contamination', models.BooleanField(default=False)),
+                ('tree_rings', models.BooleanField(default=False)),
+                ('scattered_light', models.BooleanField(default=False)),
+                ('claws', models.BooleanField(default=False)),
+                ('wisps', models.BooleanField(default=False)),
+                ('tilt_event', models.BooleanField(default=False)),
+                ('light_saber', models.BooleanField(default=False)),
+                ('other', models.BooleanField(default=False)),
+            ],
+            options={
+                'ordering': ['-root_file_info'],
+            },
+        ),
+        migrations.AddField(
+            model_name='proposal',
+            name='cat_type',
+            field=models.CharField(default='', help_text='Category Type', max_length=10),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='aperature',
+            field=models.CharField(default='', help_text='Aperature', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='detector',
+            field=models.CharField(default='', help_text='Detector', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='filter',
+            field=models.CharField(default='', help_text='Instrument name', max_length=7),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='grating',
+            field=models.CharField(default='', help_text='Grating', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='pupil',
+            field=models.CharField(default='', help_text='Pupil', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='read_patt',
+            field=models.CharField(default='', help_text='Read Pattern', max_length=40),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='read_patt_num',
+            field=models.IntegerField(default=0, help_text='Read Pattern Number'),
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='subarray',
+            field=models.CharField(default='', help_text='Subarray', max_length=40),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py
new file mode 100644
index 000000000..2d4ffdba3
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py
@@ -0,0 +1,22 @@
+# Generated by Django 3.1.7 on 2023-02-22 16:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0006_auto_20230214_1624'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='rootfileinfo',
+            name='aperature',
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='aperture',
+            field=models.CharField(default='', help_text='Aperture', max_length=40),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py
new file mode 100644
index 000000000..2c67cd955
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.1.7 on 2023-02-22 17:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0007_auto_20230222_1157'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='exp_type',
+            field=models.CharField(default='', help_text='Exposure Type', max_length=40),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py
new file mode 100644
index 000000000..d0dbafc76
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py
@@ -0,0 +1,63 @@
+# Generated by Django 3.1.7 on 2023-03-03 14:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0008_rootfileinfo_exp_type'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='proposal',
+            old_name='cat_type',
+            new_name='category',
+        ),
+        migrations.AddField(
+            model_name='rootfileinfo',
+            name='expstart',
+            field=models.FloatField(default=0.0, help_text='Exposure Start Time'),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='aperture',
+            field=models.CharField(blank=True, default='', help_text='Aperture', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='detector',
+            field=models.CharField(blank=True, default='', help_text='Detector', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='exp_type',
+            field=models.CharField(blank=True, default='', help_text='Exposure Type', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='filter',
+            field=models.CharField(blank=True, default='', help_text='Instrument name', max_length=7, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='grating',
+            field=models.CharField(blank=True, default='', help_text='Grating', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='pupil',
+            field=models.CharField(blank=True, default='', help_text='Pupil', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='read_patt',
+            field=models.CharField(blank=True, default='', help_text='Read Pattern', max_length=40, null=True),
+        ),
+        migrations.AlterField(
+            model_name='rootfileinfo',
+            name='subarray',
+            field=models.CharField(blank=True, default='', help_text='Subarray', max_length=40, null=True),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py
new file mode 100644
index 000000000..3d80e95fe
--- /dev/null
+++ b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.1.7 on 2023-03-13 15:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0009_auto_20230303_0930'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='anomalies',
+            name='flag_date',
+            field=models.DateTimeField(blank=True, help_text='flag date', null=True),
+        ),
+        migrations.AddField(
+            model_name='anomalies',
+            name='user',
+            field=models.CharField(blank=True, default='', help_text='user', max_length=50, null=True),
+        ),
+    ]
diff --git a/jwql/website/apps/jwql/migrations/__init__.py b/jwql/website/apps/jwql/migrations/__init__.py
new file mode 100644
index 000000000..e69de29bb

From a22b9ccc95958ff5a29f6ddbdc35978cbdd64682 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 28 Sep 2023 14:51:41 -0400
Subject: [PATCH 193/256] pep8 fixes

---
 .../common_monitors/dark_monitor.py           | 20 ++++---------
 jwql/utils/utils.py                           | 10 +++++--
 .../monitor_pages/monitor_bad_pixel_bokeh.py  | 28 +++++--------------
 3 files changed, 20 insertions(+), 38 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index 467a8bd1c..c8d30c92a 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -275,14 +275,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
 
             # Create figure
             start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y")
-            end_time  = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y")
+            end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y")
 
             self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='')
             #                   tools='pan,box_zoom,reset,wheel_zoom,save')
             self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0
 
             # Create the color mapper that will be used to scale the image
-            mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
+            mapper = LinearColorMapper(palette='Viridis256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev)))
 
             # Plot image and add color bar
             imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny,
@@ -291,13 +291,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
             color_bar = ColorBar(color_mapper=mapper, width=8, title='DN/sec')
             self.plot.add_layout(color_bar, 'right')
 
-            # Add hover tool for all pixel values
-            #hover_tool = HoverTool(tooltips=[('(x, y):', '($x{int}, $y{int})'),
-            #                                 ('value:', '@image')
-            #                                 ],
-            #                       renderers=[imgplot])
-            #self.plot.tools.append(hover_tool)
-
             if (('FULL' in self.aperture) or ('_CEN' in self.aperture)):
 
                 if hotxy is not None:
@@ -337,14 +330,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
                     base_start = Time(float(base_parts[3]), format='mjd').tt.datetime
                     base_end = Time(float(base_parts[5]), format='mjd').tt.datetime
                     base_start_time = base_start.strftime("%m/%d/%Y")
-                    base_end_time  = base_end.strftime("%m/%d/%Y")
+                    base_end_time = base_end.strftime("%m/%d/%Y")
                     legend_title = f'Compared to dark from {base_start_time} to {base_end_time}'
                 else:
                     legend_title = 'Compared to previous mean dark'
                 legend = Legend(items=[hot_legend, dead_legend, noisy_legend],
                                 location="center",
                                 orientation='vertical',
-                                title = legend_title)
+                                title=legend_title)
 
                 self.plot.add_layout(legend, 'below')
 
@@ -352,7 +345,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
             save_png(self.plot, filename=output_filename)
             set_permissions(output_filename)
 
-
     def get_metadata(self, filename):
         """Collect basic metadata from a fits file
 
@@ -650,14 +642,14 @@ def overplot_bad_pix(self, pix_type, coords, values):
         sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0],
                                                        pixels_y=coords[1]
                                                       )
-                                             )
+                                            )
 
         # Overplot the bad pixel locations
         badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y',
                                                  source=sources[pix_type], color=colors[pix_type])
 
         # Add to the legend
-        if  numpix > 0:
+        if numpix > 0:
             if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT:
                 text = f"{numpix} pix {adjective[pix_type]} than baseline"
             else:
diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
index 182c18191..4b8951717 100644
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -158,8 +158,7 @@ def create_png_from_fits(filename, outdir):
         plot.ygrid.visible = False
 
         # Create the color mapper that will be used to scale the image
-        #mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
-        mapper = LogColorMapper(palette='Greys256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
+        mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)),high=(img_med + (5 * img_dev)))
 
         # Plot image
         imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny,
@@ -170,8 +169,13 @@ def create_png_from_fits(filename, outdir):
         plot.yaxis.visible = False
 
         # Save the plot in a png
+<<<<<<< Updated upstream
         output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png'))
         save_png(plot, filename=output_filename)
+=======
+        output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png'))
+        export_png(plot, filename=output_filename)
+>>>>>>> Stashed changes
         permissions.set_permissions(output_filename)
         return output_filename
     else:
@@ -750,7 +754,7 @@ def read_png(filename):
 
         # Copy the RGBA image into view, flipping it so it comes right-side up
         # with a lower-left origin
-        view[:,:,:] = np.flipud(np.asarray(rgba_img))
+        view[:, :, :] = np.flipud(np.asarray(rgba_img))
     else:
         view = None
     # Return the 2D version
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
index b172df2d4..385efaa1f 100755
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
@@ -74,8 +74,6 @@ def __init__(self, instrument):
         # Get the relevant database tables
         self.identify_tables()
 
-        #self.detectors = get_unique_values_per_column(self.pixel_table, 'detector')
-
         self.detectors = sorted(DETECTOR_PER_INSTRUMENT[self.instrument])
         if self.instrument == 'miri':
             self.detectors = ['MIRIMAGE']
@@ -128,7 +126,6 @@ def modify_bokeh_saved_html(self):
 
         self._html = "".join(newlines)
 
-
     def run(self):
 
         # Right now, the aperture name in the query history table is used as the title of the
@@ -163,7 +160,7 @@ def run(self):
         # Insert into our html template and save
         template_dir = os.path.join(os.path.dirname(__file__), '../templates')
         template_file = os.path.join(template_dir, 'bad_pixel_monitor_savefile_basic.html')
-        temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div}
+        temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div': div}
         self._html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars)
 
         # Modify the html such that our Django-related lines are kept in place,
@@ -264,7 +261,6 @@ def __init__(self, pixel_table, instrument, detector):
         for badtype in self.badtypes:
             self.get_trending_data(badtype)
 
-
     def get_most_recent_entry(self):
         """Get all nedded data from the database tables.
         """
@@ -312,7 +308,7 @@ def get_trending_data(self, badpix_type):
         # Query database for all data in the table with a matching detector and bad pixel type
         all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1),
                                             self.pixel_table.obs_mid_time) \
-                              .filter(and_(self.pixel_table.detector == self.detector,  self.pixel_table.type == badpix_type)) \
+                              .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \
                               .all()
 
         # Organize the results
@@ -463,7 +459,7 @@ def create_plot(self):
             self.plot = figure(title=title_text, tools=tools,
                                x_axis_label="Pixel Number", y_axis_label="Pixel Number")
         else:
-            self.plot = figure(tools='') #, x_axis_label="Pixel Number", y_axis_label="Pixel Number")
+            self.plot = figure(tools='')
             self.plot.toolbar.logo = None
             self.plot.toolbar_location = None
             self.plot.min_border = 0
@@ -475,18 +471,9 @@ def create_plot(self):
         # Plot image
         if image is not None:
             ny, nx = image.shape
-            #imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image")
-
-
-
             # Shift the figure title slightly right in this case to get it
             # to align with the axes
-            #self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim*,
-            #                   tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number")
             self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen, alpha=0.5)
-
-
-
         else:
             # If the background image is not present, manually set the x and y range
             self.plot.x_range.start = 0
@@ -511,11 +498,10 @@ def create_plot(self):
         legend = Legend(items=[plot_legend],
                         location="center",
                         orientation='vertical',
-                        title = legend_title)
+                        title=legend_title)
 
         self.plot.add_layout(legend, 'below')
 
-
     def overplot_bad_pix(self):
         """Add a scatter plot of potential new bad pixels to the plot
 
@@ -537,7 +523,7 @@ def overplot_bad_pix(self):
             # If there are no new bad pixels, write text within the figure mentioning that
             txt_source = ColumnDataSource(data=dict(x=[self._detlen / 10], y=[self._detlen / 2],
                                           text=[f'No new {self.badpix_type} pixels found']))
-            glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'})
+            glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value': '20px'})
             self.plot.add_glyph(txt_source, glyph)
 
             # Insert a fake one, in order to get the plot to be made
@@ -668,7 +654,7 @@ def create_plot(self):
                                             string_time=string_times,
                                             value=[self.badpix_type] * len(self.num_pix)
                                             )
-                                    )
+                                 )
 
         self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save',
                            background_fill_color="#fafafa")
@@ -696,7 +682,7 @@ def create_plot(self):
             time_pad = datetime.timedelta(days=1)
         self.plot.x_range.start = min(self.time) - time_pad
         self.plot.x_range.end = max(self.time) + time_pad
-        self.plot.grid.grid_line_color="white"
+        self.plot.grid.grid_line_color = "white"
         self.plot.xaxis.axis_label = 'Date'
         self.plot.yaxis.axis_label = f'Number of {self.badpix_type} pixels'
 

From 7c2cd6618ee81e39cb7b9b09723f96f4beca4c51 Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 28 Sep 2023 15:03:00 -0400
Subject: [PATCH 194/256] more pep8

---
 jwql/instrument_monitors/common_monitors/dark_monitor.py   | 4 ++--
 jwql/tests/test_utils.py                                   | 6 +++---
 jwql/utils/utils.py                                        | 7 +------
 .../apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py     | 6 +++---
 4 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
index c8d30c92a..c73dd3543 100755
--- a/jwql/instrument_monitors/common_monitors/dark_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -641,8 +641,8 @@ def overplot_bad_pix(self, pix_type, coords, values):
 
         sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0],
                                                        pixels_y=coords[1]
-                                                      )
-                                            )
+                                                       )
+                                             )
 
         # Overplot the bad pixel locations
         badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y',
diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py
index 813bb0432..cd50c6a01 100644
--- a/jwql/tests/test_utils.py
+++ b/jwql/tests/test_utils.py
@@ -485,11 +485,11 @@ def test_filesystem_path():
 
 def test_save_png():
     """Test that we can create a png file"""
-    plot = figure(title='test',tools='')
-    image = np.zeros((200,200))
+    plot = figure(title='test', tools='')
+    image = np.zeros((200, 200))
     image[100:105, 100:105] = 1
     ny, nx = image.shape
-    mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1)
+    mapper = LinearColorMapper(palette='Viridis256', low=0, high=1.1)
     imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image")
     save_png(plot, filename='test.png')
 
diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
index 4b8951717..ca4df70a3 100644
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -169,13 +169,8 @@ def create_png_from_fits(filename, outdir):
         plot.yaxis.visible = False
 
         # Save the plot in a png
-<<<<<<< Updated upstream
-        output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png'))
-        save_png(plot, filename=output_filename)
-=======
         output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png'))
-        export_png(plot, filename=output_filename)
->>>>>>> Stashed changes
+        save_png(plot, filename=output_filename)
         permissions.set_permissions(output_filename)
         return output_filename
     else:
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
index 385efaa1f..5e522f7cd 100755
--- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py
@@ -308,8 +308,8 @@ def get_trending_data(self, badpix_type):
         # Query database for all data in the table with a matching detector and bad pixel type
         all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1),
                                             self.pixel_table.obs_mid_time) \
-                              .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \
-                              .all()
+            .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \
+            .all()
 
         # Organize the results
         num_pix = []
@@ -654,7 +654,7 @@ def create_plot(self):
                                             string_time=string_times,
                                             value=[self.badpix_type] * len(self.num_pix)
                                             )
-                                 )
+                                  )
 
         self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save',
                            background_fill_color="#fafafa")

From 6fc7ad384071ed1bdd884c63a6c7ed22aa33174f Mon Sep 17 00:00:00 2001
From: Bryan Hilbert <hilbert@stsci.edu>
Date: Thu, 28 Sep 2023 15:04:49 -0400
Subject: [PATCH 195/256] pep8, make up your mind

---
 jwql/utils/utils.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
index ca4df70a3..113dd66c9 100644
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -49,11 +49,11 @@
 
 from jwql.utils import permissions
 from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \
-                                 FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \
-                                 FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \
-                                 FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \
-                                 FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \
-                                 JWST_INSTRUMENT_NAMES_SHORTHAND
+    FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \
+    FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \
+    FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \
+    FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \
+    JWST_INSTRUMENT_NAMES_SHORTHAND
 
 __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 
@@ -158,7 +158,7 @@ def create_png_from_fits(filename, outdir):
         plot.ygrid.visible = False
 
         # Create the color mapper that will be used to scale the image
-        mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)),high=(img_med + (5 * img_dev)))
+        mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev)))
 
         # Plot image
         imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny,

From f81de385fa3105aa2470b774014bddcb243d00d4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 2 Oct 2023 12:50:14 +0000
Subject: [PATCH 196/256] Bump scipy from 1.9.3 to 1.10.0

Bumps [scipy](https://github.com/scipy/scipy) from 1.9.3 to 1.10.0.
- [Release notes](https://github.com/scipy/scipy/releases)
- [Commits](https://github.com/scipy/scipy/compare/v1.9.3...v1.10.0)

---
updated-dependencies:
- dependency-name: scipy
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index d4f3a5616..4270e9d45 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -26,7 +26,7 @@ pyvo==1.4.2
 pyyaml==6.0
 redis==5.0.0
 ruff==0.0.285
-scipy==1.9.3
+scipy==1.10.0
 selenium==4.11.2
 setuptools==68.0.0
 sphinx==6.2.1

From 1e6073ec1b0083ebd9366d7241bc4dc63b6e8516 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 2 Oct 2023 15:03:05 -0400
Subject: [PATCH 197/256] Removed all templating, adding new calls to monitor
 views to display proper html

---
 .../monitor_pages/monitor_readnoise_bokeh.py  | 118 ++++++++++++++++--
 jwql/website/apps/jwql/monitor_views.py       |   4 +-
 2 files changed, 109 insertions(+), 13 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 3a5d22a6a..5e2b0c80f 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -12,7 +12,7 @@
 
     ::
 
-        from jwql.website.apps.jwql import monitor_pages
+        .
         monitor_template = monitor_pages.ReadnoiseMonitor()
         monitor_template.input_parameters = ('NIRCam', 'NRCA1_FULL')
 """
@@ -20,12 +20,17 @@
 from datetime import datetime, timedelta
 import os
 
+from bokeh.embed import components
+from bokeh.layouts import column, row
+from bokeh.models import Panel, Tabs  # bokeh <= 3.0
+from bokeh.models import ColumnDataSource, HoverTool
+# from bokeh.models import TabPanel, Tabs  # bokeh >= 3.0
+from bokeh.plotting import figure
 import numpy as np
 
-from jwql.bokeh_templating import BokehTemplate
 from jwql.database.database_interface import session
 from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE
 
 
 class ReadnoiseMonitorData():
@@ -74,8 +79,24 @@ def load_data(self):
 
         session.close()
 
-class ReadNoisePlots():
-    """Class to make readnoise plots.
+
+class ReadNoiseFigure():
+    """Generate tabbed plot displayed in JWQL web application
+    """
+    def __init__(self, instrument):
+        instrument_apertures = FULL_FRAME_APERTURES[instrument.upper()]
+
+        self.tabs = []
+        for aperture in instrument_apertures:
+            readnoise_tab = ReadNoisePlotTab(instrument, aperture)
+            self.tabs.append(readnoise_tab.tab)
+
+        self.plot = Tabs(tabs=self.tabs)
+        self.tab_components = components(self.plot)
+
+
+class ReadNoisePlotTab():
+    """Class to make instrument/aperture panels 
     """
     def __init__(self, instrument, aperture):
         self.instrument = instrument
@@ -83,16 +104,89 @@ def __init__(self, instrument, aperture):
 
         self.db = ReadnoiseMonitorData(self.instrument, self.aperture)
 
+        self.plot_readnoise_amplifers()
+        self.plot_readnoise_difference_image()
+        self.plot_readnoise_histogram()
 
-    def read_noise_amp_plots(self):
-        """Make read noise monitor trending plots per amplifier
-        """
-        print('make plot here')
-        # for amp in ['1', '2', '3', '4']:
-        #     expstarts_iso = np.array([result.expstart for result in self.db])
-        #     readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.query_results])
+        self.tab = Panel(child=column(row(*self.amp_plots), self.diff_image_plot, self.readnoise_histogram), title= self.aperture)
+
+    def plot_readnoise_amplifers(self):
+
+        self.amp_plots = []
+        for amp in ['1', '2', '3', '4']:
+
+            amp_plot = figure(width=350, height=350, x_axis_type='datetime')
+
+            readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results])
             
+            filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results]
+            expstarts_iso = np.array([result.expstart for result in self.db.query_results])
+            expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso])
+            nints = [result.nints for result in self.db.query_results]
+            ngroups = [result.ngroups for result in self.db.query_results]
+
+            source = ColumnDataSource(data=dict(
+                        file=filenames,
+                        expstarts=expstarts,
+                        nints=nints,
+                        ngroups=ngroups,
+                        readnoise=readnoise_vals,
+                        ))
+
+            amp_plot.add_tools(HoverTool(tooltips=[("file", "@filenames"),
+                                                    ("time", "@expstarts"),
+                                                    ("nints", "@nints"),
+                                                    ("ngroups", "@ngroups"),
+                                                    ("readnoise", "@readnoise")]))
+
+            amp_plot.circle(x='expstarts', y='readnoise', source=source)
+
+            amp_plot.xaxis.axis_label = 'Date'
+            amp_plot.yaxis.axis_label = 'Mean Readnoise [DN]'
+
+            self.amp_plots.append(amp_plot)
+
+    def plot_readnoise_difference_image(self):
+        """Updates the readnoise difference image"""
+
+        # Update the readnoise difference image and histogram, if data exists
+
+        self.diff_image_plot = figure(height=700, width=700)
+
+        if len(self.db.query_results) != 0:
+            diff_image_png = self.db.query_results[-1].readnoise_diff_image
+            self.diff_image_plot.image_url(url=[diff_image_png], x=1, y=1, w=2048, h=2048, anchor="bottom_left")
+            self.diff_image_plot.xaxis.visible = False
+            self.diff_image_plot.yaxis.visible = False
+            self.diff_image_plot.xgrid.grid_line_color = None
+            self.diff_image_plot.ygrid.grid_line_color = None
+            self.diff_image_plot.title.text_font_size = '22px'
+            self.diff_image_plot.title.align = 'center'
+
+    def plot_readnoise_histogram(self):
+        """Updates the readnoise histogram"""
+
+        diff_image_n = np.array(self.db.query_results[-1].diff_image_n)
+        diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers)
+
+        hist_xr_start = diff_image_bin_centers.min()
+        hist_xr_end = diff_image_bin_centers.max()
+        hist_yr_start = diff_image_n.min()
+        hist_yr_end = diff_image_n.max() + diff_image_n.max() * 0.05
+
+        self.readnoise_histogram = figure(height=800, width=800,
+                                          x_range=(hist_xr_start, hist_xr_end),
+                                          y_range=(hist_yr_start, hist_yr_end))
+
 
+        source = ColumnDataSource(data=dict(
+                        x=diff_image_bin_centers,
+                        y=diff_image_n,
+                        ))
 
+        self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"),]))
 
+        self.readnoise_histogram.circle(x='x', y='y', source=source)
 
+        self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]'
+        self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels'
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index b62f339de..09cf423f6 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -37,12 +37,14 @@
 
 from . import bokeh_containers
 from jwql.website.apps.jwql import bokeh_containers
+from jwql.website.apps.jwql.monitor_pages.monitor_readnoise_bokeh import ReadNoiseFigure
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.utils import get_config, get_base_url
 from jwql.instrument_monitors.nirspec_monitors.ta_monitors import msata_monitor
 from jwql.instrument_monitors.nirspec_monitors.ta_monitors import wata_monitor
 from jwql.utils import monitor_utils
 
+
 CONFIG = get_config()
 FILESYSTEM_DIR = os.path.join(CONFIG['jwql_dir'], 'filesystem')
 
@@ -219,7 +221,7 @@ def readnoise_monitor(request, inst):
     inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()]
 
     # Get the html and JS needed to render the readnoise tab plots
-    tabs_components = bokeh_containers.readnoise_monitor_tabs(inst)
+    tabs_components = ReadNoiseFigure(inst).tab_components
 
     template = "readnoise_monitor.html"
 

From d4db60902588ceb844ac06218274c0cb6e466b45 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 2 Oct 2023 15:50:28 -0400
Subject: [PATCH 198/256] Remove import from __init__ and updating typos for
 plotting, updating source for readnoise scatter

---
 jwql/website/apps/jwql/monitor_pages/__init__.py              | 3 +--
 .../apps/jwql/monitor_pages/monitor_readnoise_bokeh.py        | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py
index 1405523c0..8a4054c76 100644
--- a/jwql/website/apps/jwql/monitor_pages/__init__.py
+++ b/jwql/website/apps/jwql/monitor_pages/__init__.py
@@ -1,2 +1 @@
-from .monitor_cosmic_rays_bokeh import CosmicRayMonitor
-from .monitor_readnoise_bokeh import ReadnoiseMonitor
+from .monitor_cosmic_rays_bokeh import CosmicRayMonitor
\ No newline at end of file
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 5e2b0c80f..c4e4a350f 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -133,7 +133,7 @@ def plot_readnoise_amplifers(self):
                         readnoise=readnoise_vals,
                         ))
 
-            amp_plot.add_tools(HoverTool(tooltips=[("file", "@filenames"),
+            amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"),
                                                     ("time", "@expstarts"),
                                                     ("nints", "@nints"),
                                                     ("ngroups", "@ngroups"),
@@ -155,7 +155,7 @@ def plot_readnoise_difference_image(self):
 
         if len(self.db.query_results) != 0:
             diff_image_png = self.db.query_results[-1].readnoise_diff_image
-            self.diff_image_plot.image_url(url=[diff_image_png], x=1, y=1, w=2048, h=2048, anchor="bottom_left")
+            self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left")
             self.diff_image_plot.xaxis.visible = False
             self.diff_image_plot.yaxis.visible = False
             self.diff_image_plot.xgrid.grid_line_color = None

From 8c2fd616cdaf51ca280ff0dd77cf0a2dbf922e82 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 3 Oct 2023 16:07:04 -0400
Subject: [PATCH 199/256] update deps in environment files

---
 environment_python_3.10.yml | 50 ++++++++++++------------
 environment_python_3.9.yml  | 46 +++++++++++-----------
 requirements.txt            | 78 ++++++++++++++++++-------------------
 3 files changed, 87 insertions(+), 87 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index f61973680..7856cf201 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -22,49 +22,49 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.2
+  - astropy=5.3.3
   - beautifulsoup4=4.12.2
   - bokeh=2.4.3
-  - celery=5.3.1
-  - cryptography=41.0.3
-  - django=4.2.3
+  - celery=5.3.4
+  - cryptography=41.0.4
+  - django=4.2.5
   - inflection=0.5.1
-  - ipython=8.14.0
+  - ipython=8.16.1
   - jinja2=3.1.2
-  - jsonschema=4.19.0
-  - matplotlib=3.7.2
-  - nodejs=20.5.1
+  - jsonschema=4.19.1
+  - matplotlib=3.8.0
+  - nodejs>=18.15.0
   - numpy=1.25.2
   - numpydoc=1.5.0
-  - pandas=2.0.3
+  - pandas=2.1.1
   - pip=23.2.1
-  - postgresql=15.4
-  - psycopg2=2.9.6
-  - pytest=7.4.0
+  - postgresql>=15.2
+  - psycopg2>=2.9.3
+  - pytest=7.4.2
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
-  - python=3.10.12
-  - pyyaml=6.0
-  - redis
-  - ruff=0.0.285
+  - python=3.10.13
+  - pyyaml=6.0.1
+  - redis=5.0.1
+  - ruff=0.0.292
   - scipy=1.9.3
-  - setuptools=68.0.0
-  - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.2
-  - sqlalchemy=2.0.20
+  - setuptools=68.2.2
+  - sphinx=7.2.6
+  - sphinx_rtd_theme=1.3.0
+  - sqlalchemy=2.0.21
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
     - astroquery==0.4.6
     - bandit==1.7.5
-    - jwst==1.11.4
+    - jwst==1.12.2
     - pysiaf==0.20.0
-    - pysqlite3==0.5.1
+    - pysqlite3==0.5.2
     - pyvo==1.4.2
-    - redis==5.0.0
-    - selenium==4.11.2
-    - stdatamodels==1.7.2
+    - redis==5.0.1
+    - selenium==4.13.0
+    - stdatamodels==1.8.3
     - stsci_rtd_theme==1.0.0
     - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index d3724b1d8..554355df3 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -22,49 +22,49 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.2
+  - astropy=5.3.3
   - beautifulsoup4=4.12.2
   - bokeh=2.4.3
-  - celery=5.3.1
-  - cryptography=41.0.3
-  - django=4.2.3
+  - celery=5.3.4
+  - cryptography=41.0.4
+  - django=4.2.5
   - inflection=0.5.1
-  - ipython=8.14.0
+  - ipython=8.16.1
   - jinja2=3.1.2
-  - jsonschema=4.19.0
-  - matplotlib=3.7.2
-  - nodejs=20.5.1
+  - jsonschema=4.19.1
+  - matplotlib=3.8.0
+  - nodejs=20.8.0
   - numpy=1.25.2
   - numpydoc=1.5.0
-  - pandas=2.0.3
+  - pandas=2.1.1
   - pip=23.2.1
   - postgresql=15.4
-  - psycopg2=2.9.6
-  - pytest=7.4.0
+  - psycopg2=2.9.7
+  - pytest=7.4.2
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.9.17
-  - pyyaml=6.0
-  - redis
-  - ruff=0.0.285
+  - pyyaml=6.0.1
+  - redis=5.0.1
+  - ruff=0.0.292
   - scipy=1.9.3
-  - setuptools=68.0.0
-  - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.2
-  - sqlalchemy=2.0.20
+  - setuptools=68.2.2
+  - sphinx=7.2.6
+  - sphinx_rtd_theme=1.3.0
+  - sqlalchemy=2.0.21
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
     - astroquery==0.4.6
     - bandit==1.7.5
-    - jwst==1.11.4
+    - jwst==1.12.2
     - pysiaf==0.20.0
-    - pysqlite3==0.5.1
+    - pysqlite3==0.5.2
     - pyvo==1.4.2
-    - redis==5.0.0
-    - selenium==4.11.2
-    - stdatamodels==1.7.2
+    - redis==5.0.1
+    - selenium==4.13.0
+    - stdatamodels==1.8.3
     - stsci_rtd_theme==1.0.0
     - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles
diff --git a/requirements.txt b/requirements.txt
index 4270e9d45..6b8d88d41 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,40 +1,40 @@
-astropy==5.3.2
-astroquery==0.4.6
-bandit==1.7.5
-beautifulsoup4==4.12.2
-bokeh==2.4.3
-celery==5.3.1
-cryptography==41.0.4
-django==4.2.3
-inflection==0.5.1
-ipython==8.14.0
-jinja2==3.1.2
-jsonschema==4.19.0
-jwst==1.11.4
-matplotlib==3.7.2
-nodejs==20.5.1
-numpy==1.25.2
-numpydoc==1.5.0
-pandas==2.0.3
-psycopg2-binary==2.9.7
-pysiaf==0.20.0
-pysqlite3==0.5.1
-pytest==7.4.0
-pytest-cov==4.1.0
-pytest-mock==3.11.1
-pyvo==1.4.2
-pyyaml==6.0
-redis==5.0.0
-ruff==0.0.285
-scipy==1.10.0
-selenium==4.11.2
-setuptools==68.0.0
-sphinx==6.2.1
-sphinx_rtd_theme==1.2.2
-sqlalchemy==2.0.20
-stdatamodels==1.7.2
-stsci_rtd_theme==1.0.0
-twine==4.0.2
-vine==5.0.0
-wtforms==3.0.1
+astropy
+astroquery
+bandit
+beautifulsoup4
+bokeh
+celery
+cryptography
+django
+inflection
+ipython
+jinja2
+jsonschema
+jwst
+matplotlib
+nodejs
+numpy
+numpydoc
+pandas
+psycopg2-binary
+pysiaf
+pysqlite3
+pytest
+pytest-cov
+pytest-mock
+pyvo
+pyyaml
+redis
+ruff
+scipy
+selenium
+setuptools
+sphinx
+sphinx_rtd_theme
+sqlalchemy
+stdatamodels
+stsci_rtd_theme
+twine
+vine
+wtforms
 git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles

From c0de0112325fa00644caa9cbe619d25bb63aa670 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 3 Oct 2023 16:18:34 -0400
Subject: [PATCH 200/256] update versions in requirements.txt

---
 requirements.txt | 78 ++++++++++++++++++++++++------------------------
 1 file changed, 39 insertions(+), 39 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 6b8d88d41..bb289de5a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,40 +1,40 @@
-astropy
-astroquery
-bandit
-beautifulsoup4
-bokeh
-celery
-cryptography
-django
-inflection
-ipython
-jinja2
-jsonschema
-jwst
-matplotlib
-nodejs
-numpy
-numpydoc
-pandas
-psycopg2-binary
-pysiaf
-pysqlite3
-pytest
-pytest-cov
-pytest-mock
-pyvo
-pyyaml
-redis
-ruff
-scipy
-selenium
-setuptools
-sphinx
-sphinx_rtd_theme
-sqlalchemy
-stdatamodels
-stsci_rtd_theme
-twine
-vine
-wtforms
+astropy==5.3.3
+astroquery==0.4.6
+bandit==1.7.5
+beautifulsoup4==4.12.2
+bokeh==2.4.3
+celery==5.3.4
+cryptography==41.0.4
+django==4.2.5
+inflection==0.5.1
+ipython==8.16.1
+jinja2==3.1.2
+jsonschema==4.19.1
+jwst==1.12.2
+matplotlib==3.8.0
+nodejs==20.8.0
+numpy==1.25.2
+numpydoc==1.5.0
+pandas==2.1.1
+psycopg2-binary==2.9.7
+pysiaf==0.20.0
+pysqlite3==0.5.2
+pytest==7.4.2
+pytest-cov==4.1.0
+pytest-mock==3.11.1
+pyvo==1.4.2
+pyyaml==6.0.1
+redis==5.0.1
+ruff==0.0.292
+scipy==1.9.3
+selenium==4.13.0
+setuptools==68.2.2
+sphinx==7.2.6
+sphinx_rtd_theme==1.3.0
+sqlalchemy==2.0.21
+stdatamodels==1.8.3
+stsci_rtd_theme==1.0.0
+twine==4.0.2
+vine==5.0.0
+wtforms==3.0.1
 git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles

From 8ae4da424b1cf99c8a6e20cabe504c938285c228 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 3 Oct 2023 16:21:14 -0400
Subject: [PATCH 201/256] put redis back to 5.0.0

---
 environment_python_3.10.yml     | 2 +-
 jwql/utils/logging_functions.py | 2 +-
 requirements.txt                | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 7856cf201..505bdb7c0 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -45,7 +45,7 @@ dependencies:
   - pytest-mock=3.11.1
   - python=3.10.13
   - pyyaml=6.0.1
-  - redis=5.0.1
+  - redis=5.0.0
   - ruff=0.0.292
   - scipy=1.9.3
   - setuptools=68.2.2
diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py
index abd82a45e..3ef464a40 100644
--- a/jwql/utils/logging_functions.py
+++ b/jwql/utils/logging_functions.py
@@ -106,7 +106,7 @@ def configure_logging(module):
     logging.basicConfig(filename=log_file,
                         format='%(asctime)s %(levelname)s: %(message)s',
                         datefmt='%m/%d/%Y %H:%M:%S %p',
-                        level=logging.INFO)
+                        level=logging.DEBUG)
     print('Log file initialized to {}'.format(log_file))
     set_permissions(log_file)
 
diff --git a/requirements.txt b/requirements.txt
index bb289de5a..d82a3409f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,7 @@ pytest-cov==4.1.0
 pytest-mock==3.11.1
 pyvo==1.4.2
 pyyaml==6.0.1
-redis==5.0.1
+redis==5.0.0.
 ruff==0.0.292
 scipy==1.9.3
 selenium==4.13.0

From 9ef1cd9356335913eab0562282cad68e16f1f579 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 3 Oct 2023 16:23:10 -0400
Subject: [PATCH 202/256] whoops, accidental logging change

---
 jwql/utils/logging_functions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py
index 3ef464a40..abd82a45e 100644
--- a/jwql/utils/logging_functions.py
+++ b/jwql/utils/logging_functions.py
@@ -106,7 +106,7 @@ def configure_logging(module):
     logging.basicConfig(filename=log_file,
                         format='%(asctime)s %(levelname)s: %(message)s',
                         datefmt='%m/%d/%Y %H:%M:%S %p',
-                        level=logging.DEBUG)
+                        level=logging.INFO)
     print('Log file initialized to {}'.format(log_file))
     set_permissions(log_file)
 

From b4e4c38e856adfa88e82483dcfc4f81b6e5bbb52 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 6 Oct 2023 11:04:47 -0400
Subject: [PATCH 203/256] revert all redis references to 5.0.0

---
 environment_python_3.10.yml | 2 +-
 environment_python_3.9.yml  | 4 ++--
 requirements.txt            | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 505bdb7c0..518fbc902 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -62,7 +62,7 @@ dependencies:
     - pysiaf==0.20.0
     - pysqlite3==0.5.2
     - pyvo==1.4.2
-    - redis==5.0.1
+    - redis==5.0.0
     - selenium==4.13.0
     - stdatamodels==1.8.3
     - stsci_rtd_theme==1.0.0
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index 554355df3..5b70010a8 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -45,7 +45,7 @@ dependencies:
   - pytest-mock=3.11.1
   - python=3.9.17
   - pyyaml=6.0.1
-  - redis=5.0.1
+  - redis=5.0.0
   - ruff=0.0.292
   - scipy=1.9.3
   - setuptools=68.2.2
@@ -62,7 +62,7 @@ dependencies:
     - pysiaf==0.20.0
     - pysqlite3==0.5.2
     - pyvo==1.4.2
-    - redis==5.0.1
+    - redis==5.0.0
     - selenium==4.13.0
     - stdatamodels==1.8.3
     - stsci_rtd_theme==1.0.0
diff --git a/requirements.txt b/requirements.txt
index d82a3409f..01aeb4f7f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,7 @@ pytest-cov==4.1.0
 pytest-mock==3.11.1
 pyvo==1.4.2
 pyyaml==6.0.1
-redis==5.0.0.
+redis==5.0.0
 ruff==0.0.292
 scipy==1.9.3
 selenium==4.13.0

From b874639b0e45e0a67b8813a9b36a4d12f02af7a1 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 6 Oct 2023 12:51:46 -0400
Subject: [PATCH 204/256] mamba didn't like 3.10.13

---
 environment_python_3.10.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index 518fbc902..f60c31d27 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -43,7 +43,7 @@ dependencies:
   - pytest=7.4.2
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
-  - python=3.10.13
+  - python=3.10.12
   - pyyaml=6.0.1
   - redis=5.0.0
   - ruff=0.0.292

From 6edf1262793467580a131f5c745682e980d8cecb Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 6 Oct 2023 15:32:45 -0400
Subject: [PATCH 205/256] update for 3.10.12

---
 environment_python_3.10.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index f60c31d27..fd927ce30 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -22,24 +22,24 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.3
+  - astropy=5.3.4
   - beautifulsoup4=4.12.2
   - bokeh=2.4.3
   - celery=5.3.4
   - cryptography=41.0.4
-  - django=4.2.5
+  - django=4.2.6
   - inflection=0.5.1
   - ipython=8.16.1
   - jinja2=3.1.2
   - jsonschema=4.19.1
   - matplotlib=3.8.0
-  - nodejs>=18.15.0
+  - nodejs=20.8.0
   - numpy=1.25.2
   - numpydoc=1.5.0
   - pandas=2.1.1
   - pip=23.2.1
-  - postgresql>=15.2
-  - psycopg2>=2.9.3
+  - postgresql=15.4
+  - psycopg2=2.9.7
   - pytest=7.4.2
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1

From 91feef45fdbf47a9e994c28ef8b3921ecae7d68f Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 6 Oct 2023 16:29:43 -0400
Subject: [PATCH 206/256] update rtd_requirements.txt

---
 rtd_requirements.txt | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index 39500c2b6..f4f42830e 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -1,16 +1,16 @@
-sphinx_automodapi==0.15.0
+sphinx_automodapi>=0.15.0
 bokeh==2.4.3
-celery==5.3.1
-cython==3.0.0
-django==4.2.3
-docutils==0.18.1
-jwst==1.11.4
+celery==5.3.4
+cython>=3.0.0
+django==4.2.5
+docutils>=0.18.1
+jwst==1.12.3
 pygments==2.16.1
-pytest==7.4.0
+pytest==7.4.2
 redis==5.0.0
-selenium==4.11.2
-sphinx==6.2.1
-sphinx_rtd_theme==1.2.2
+selenium==4.13.0
+sphinx==7.2.6
+sphinx_rtd_theme==1.30
 stsci_rtd_theme==1.0.0
 tomli==2.0.1
 git+https://github.com/spacetelescope/jwst_reffiles

From 059310ac90171af382c1c91b130861e20cdb0c6d Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 6 Oct 2023 16:35:59 -0400
Subject: [PATCH 207/256] jwst changed out from under me

---
 environment_python_3.10.yml | 2 +-
 environment_python_3.9.yml  | 2 +-
 requirements.txt            | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml
index fd927ce30..9f2ed409b 100644
--- a/environment_python_3.10.yml
+++ b/environment_python_3.10.yml
@@ -58,7 +58,7 @@ dependencies:
   - pip:
     - astroquery==0.4.6
     - bandit==1.7.5
-    - jwst==1.12.2
+    - jwst==1.12.3
     - pysiaf==0.20.0
     - pysqlite3==0.5.2
     - pyvo==1.4.2
diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml
index 5b70010a8..4f4159244 100644
--- a/environment_python_3.9.yml
+++ b/environment_python_3.9.yml
@@ -58,7 +58,7 @@ dependencies:
   - pip:
     - astroquery==0.4.6
     - bandit==1.7.5
-    - jwst==1.12.2
+    - jwst==1.12.3
     - pysiaf==0.20.0
     - pysqlite3==0.5.2
     - pyvo==1.4.2
diff --git a/requirements.txt b/requirements.txt
index 01aeb4f7f..6a7f670cd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,7 +10,7 @@ inflection==0.5.1
 ipython==8.16.1
 jinja2==3.1.2
 jsonschema==4.19.1
-jwst==1.12.2
+jwst==1.12.3
 matplotlib==3.8.0
 nodejs==20.8.0
 numpy==1.25.2

From fdfd4b674a9a6db8468fe16e72a235e537894079 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 6 Oct 2023 16:40:32 -0400
Subject: [PATCH 208/256] update setuptools version

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index c17d3dc28..133af8c79 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ file = "LICENSE"
 content-type = "text/plain"
 
 [build-system]
-requires = ["setuptools>=61.2,<=65.5.0", "numpy", "wheel", "setuptools_scm"]
+requires = ["setuptools=68.2.2", "numpy", "wheel", "setuptools_scm"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools]

From f365329f7023f0d18596e59ee6f1979d76d51f31 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 6 Oct 2023 16:51:14 -0400
Subject: [PATCH 209/256] setuptools greater than

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 133af8c79..24f701717 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ file = "LICENSE"
 content-type = "text/plain"
 
 [build-system]
-requires = ["setuptools=68.2.2", "numpy", "wheel", "setuptools_scm"]
+requires = ["setuptools>=68.0.0", "numpy", "wheel", "setuptools_scm"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools]

From f7ecd691e9034e3828a21406ebe00372721ec09a Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Fri, 6 Oct 2023 16:54:54 -0400
Subject: [PATCH 210/256] no thank you sphinx_theme. stsci_theme 4ever

---
 rtd_requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index f4f42830e..abac6b128 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -10,7 +10,6 @@ pytest==7.4.2
 redis==5.0.0
 selenium==4.13.0
 sphinx==7.2.6
-sphinx_rtd_theme==1.30
 stsci_rtd_theme==1.0.0
 tomli==2.0.1
 git+https://github.com/spacetelescope/jwst_reffiles

From 9af57328c240ecec34e836ac12b0bf38cc3ce102 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 12 Oct 2023 11:59:54 -0400
Subject: [PATCH 211/256] Adding nircam claw databases

---
 jwql/database/database_interface.py             |  7 +++++--
 .../nircam/nircam_claw_query_history.txt        |  5 +++++
 .../nircam/nircam_claw_stats.txt                | 17 +++++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 jwql/database/monitor_table_definitions/nircam/nircam_claw_query_history.txt
 create mode 100644 jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt

diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 8f660209c..47800f647 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -509,6 +509,8 @@ class : obj
 NIRSpecCosmicRayStats = monitor_orm_factory('nirspec_cosmic_ray_stats')
 NIRSpecTAQueryHistory = monitor_orm_factory('nirspec_ta_query_history')
 NIRSpecTAStats = monitor_orm_factory('nirspec_ta_stats')
+NIRCamClawQueryHistory = monitor_orm_factory('nircam_claw_query_history')
+NIRCamClawStats = monitor_orm_factory('nircam_claw_stats')
 
 INSTRUMENT_TABLES = {
     'nircam': [NIRCamDarkQueryHistory, NIRCamDarkPixelStats, NIRCamDarkDarkCurrent,
@@ -516,7 +518,7 @@ class : obj
                NIRCamBadPixelStats, NIRCamReadnoiseQueryHistory, NIRCamReadnoiseStats,
                NIRCamAnomaly, NIRCamCosmicRayQueryHistory, NIRCamCosmicRayStats,
                NIRCamEDBDailyStats, NIRCamEDBBlockStats, NIRCamEDBTimeIntervalStats,
-               NIRCamEDBEveryChangeStats],
+               NIRCamEDBEveryChangeStats, NIRCamClawQueryHistory, NIRCamClawStats],
     'niriss': [NIRISSDarkQueryHistory, NIRISSDarkPixelStats, NIRISSDarkDarkCurrent,
                NIRISSBiasQueryHistory, NIRISSBiasStats, NIRISSBadPixelQueryHistory,
                NIRISSBadPixelStats, NIRISSReadnoiseQueryHistory, NIRISSReadnoiseStats,
@@ -570,7 +572,8 @@ class : obj
             MIRIEDBBlockStats, MIRIEDBTimeIntervalStats, MIRIEDBEveryChangeStats,
             NIRSpecEDBDailyStats, NIRSpecEDBBlockStats, NIRSpecEDBTimeIntervalStats,
             NIRSpecEDBEveryChangeStats, FGSEDBDailyStats, FGSEDBBlockStats,
-            FGSEDBTimeIntervalStats, FGSEDBEveryChangeStats]}
+            FGSEDBTimeIntervalStats, FGSEDBEveryChangeStats],
+    'claw': [NIRCamClawQueryHistory, NIRCamClawStats],}
 
 if __name__ == '__main__':
     base.metadata.create_all(engine)
diff --git a/jwql/database/monitor_table_definitions/nircam/nircam_claw_query_history.txt b/jwql/database/monitor_table_definitions/nircam/nircam_claw_query_history.txt
new file mode 100644
index 000000000..d116f3c2f
--- /dev/null
+++ b/jwql/database/monitor_table_definitions/nircam/nircam_claw_query_history.txt
@@ -0,0 +1,5 @@
+INSTRUMENT, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+RUN_MONITOR, bool
+ENTRY_DATE, datetime
\ No newline at end of file
diff --git a/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt b/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
new file mode 100644
index 000000000..f6fc1131a
--- /dev/null
+++ b/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
@@ -0,0 +1,17 @@
+FILENAME, string
+PROPOSAL, string
+OBS, string
+DETECTOR, string
+FILTER, string
+PUPIL, string
+EXPSTART, string
+EXPSTART_MJD, float
+EFFEXPTM, float
+RA, float
+DEC, float
+PA_V3, float
+MEAN, float
+MEDIAN, float
+STDDEV, float
+FRAC_MASKED, float
+SKYFLAT_FILENAME, string
\ No newline at end of file

From 385ec3d6243ffba2399d0f66951169a91c99c249 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Fri, 13 Oct 2023 11:22:39 -0400
Subject: [PATCH 212/256] Adding string slicing for path to readnoise diff
 image

---
 .../monitor_pages/monitor_readnoise_bokeh.py  | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index c4e4a350f..551623606 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -108,14 +108,18 @@ def __init__(self, instrument, aperture):
         self.plot_readnoise_difference_image()
         self.plot_readnoise_histogram()
 
-        self.tab = Panel(child=column(row(*self.amp_plots), self.diff_image_plot, self.readnoise_histogram), title= self.aperture)
+        self.tab = Panel(child=column(row(*self.amp_plots), 
+                                          self.diff_image_plot, 
+                                          self.readnoise_histogram), 
+                        title= self.aperture)
 
     def plot_readnoise_amplifers(self):
 
         self.amp_plots = []
         for amp in ['1', '2', '3', '4']:
 
-            amp_plot = figure(width=350, height=350, x_axis_type='datetime')
+            amp_plot = figure(title='Amp {}'.format(amp), width=280, height=280, x_axis_type='datetime')
+            amp_plot.xaxis[0].ticker.desired_num_ticks = 4
 
             readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results])
             
@@ -151,10 +155,12 @@ def plot_readnoise_difference_image(self):
 
         # Update the readnoise difference image and histogram, if data exists
 
-        self.diff_image_plot = figure(height=700, width=700)
+        self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', 
+                                      height=500, width=500, sizing_mode='scale_width')
 
         if len(self.db.query_results) != 0:
             diff_image_png = self.db.query_results[-1].readnoise_diff_image
+            diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:]))
             self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left")
             self.diff_image_plot.xaxis.visible = False
             self.diff_image_plot.yaxis.visible = False
@@ -174,9 +180,10 @@ def plot_readnoise_histogram(self):
         hist_yr_start = diff_image_n.min()
         hist_yr_end = diff_image_n.max() + diff_image_n.max() * 0.05
 
-        self.readnoise_histogram = figure(height=800, width=800,
+        self.readnoise_histogram = figure(height=500, width=500,
                                           x_range=(hist_xr_start, hist_xr_end),
-                                          y_range=(hist_yr_start, hist_yr_end))
+                                          y_range=(hist_yr_start, hist_yr_end),
+                                          sizing_mode='scale_width')
 
 
         source = ColumnDataSource(data=dict(
@@ -190,3 +197,5 @@ def plot_readnoise_histogram(self):
 
         self.readnoise_histogram.xaxis.axis_label = 'Readnoise Difference [DN]'
         self.readnoise_histogram.yaxis.axis_label = 'Number of Pixels'
+        self.readnoise_histogram.xaxis.axis_label_text_font_size = "15pt"
+        self.readnoise_histogram.yaxis.axis_label_text_font_size = "15pt"

From 6cdb3f7527cb680514ae04f2862992ee8eab6d07 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Fri, 13 Oct 2023 13:11:23 -0400
Subject: [PATCH 213/256] pep8speaks

---
 .../apps/jwql/monitor_pages/__init__.py       |  2 +-
 .../monitor_pages/monitor_readnoise_bokeh.py  | 34 +++++++++----------
 jwql/website/apps/jwql/monitor_views.py       |  2 +-
 3 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/__init__.py b/jwql/website/apps/jwql/monitor_pages/__init__.py
index 8a4054c76..ed184d7ff 100644
--- a/jwql/website/apps/jwql/monitor_pages/__init__.py
+++ b/jwql/website/apps/jwql/monitor_pages/__init__.py
@@ -1 +1 @@
-from .monitor_cosmic_rays_bokeh import CosmicRayMonitor
\ No newline at end of file
+from .monitor_cosmic_rays_bokeh import CosmicRayMonitor
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 551623606..7657aeeed 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -35,21 +35,21 @@
 
 class ReadnoiseMonitorData():
     """Class to hold bias data to be plotted
-    
+
     Parameters
     ----------
-    
+
     instrument : str
         Instrument name (e.g. nircam)
-    
+
     Attributes
     ----------
-    
+
     instrument : str
         Instrument name (e.g. nircam)
     aperture : str
         Aperture name (e.g. apername)
-    
+
     """
 
     def __init__(self, instrument, aperture):
@@ -57,7 +57,6 @@ def __init__(self, instrument, aperture):
         self.aperture = aperture
         self.load_data()
 
-
     def identify_tables(self):
         """Determine which database tables to use for the given instrument"""
 
@@ -96,7 +95,7 @@ def __init__(self, instrument):
 
 
 class ReadNoisePlotTab():
-    """Class to make instrument/aperture panels 
+    """Class to make instrument/aperture panels
     """
     def __init__(self, instrument, aperture):
         self.instrument = instrument
@@ -108,10 +107,10 @@ def __init__(self, instrument, aperture):
         self.plot_readnoise_difference_image()
         self.plot_readnoise_histogram()
 
-        self.tab = Panel(child=column(row(*self.amp_plots), 
-                                          self.diff_image_plot, 
-                                          self.readnoise_histogram), 
-                        title= self.aperture)
+        self.tab = Panel(child=column(row(*self.amp_plots),
+                                      self.diff_image_plot,
+                                      self.readnoise_histogram),
+                         title=self.aperture)
 
     def plot_readnoise_amplifers(self):
 
@@ -122,7 +121,7 @@ def plot_readnoise_amplifers(self):
             amp_plot.xaxis[0].ticker.desired_num_ticks = 4
 
             readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results])
-            
+
             filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results]
             expstarts_iso = np.array([result.expstart for result in self.db.query_results])
             expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso])
@@ -138,10 +137,10 @@ def plot_readnoise_amplifers(self):
                         ))
 
             amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"),
-                                                    ("time", "@expstarts"),
-                                                    ("nints", "@nints"),
-                                                    ("ngroups", "@ngroups"),
-                                                    ("readnoise", "@readnoise")]))
+                                                   ("time", "@expstarts"),
+                                                   ("nints", "@nints"),
+                                                   ("ngroups", "@ngroups"),
+                                                   ("readnoise", "@readnoise")]))
 
             amp_plot.circle(x='expstarts', y='readnoise', source=source)
 
@@ -155,7 +154,7 @@ def plot_readnoise_difference_image(self):
 
         # Update the readnoise difference image and histogram, if data exists
 
-        self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)', 
+        self.diff_image_plot = figure(title='Readnoise Difference (most recent dark - pipeline reffile)',
                                       height=500, width=500, sizing_mode='scale_width')
 
         if len(self.db.query_results) != 0:
@@ -185,7 +184,6 @@ def plot_readnoise_histogram(self):
                                           y_range=(hist_yr_start, hist_yr_end),
                                           sizing_mode='scale_width')
 
-
         source = ColumnDataSource(data=dict(
                         x=diff_image_bin_centers,
                         y=diff_image_n,
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index 09cf423f6..3835f744b 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -69,7 +69,7 @@ def bad_pixel_monitor(request, inst):
 
     context = {
         'inst': inst,
-        }
+    }
 
     return render(request, template, context)
 

From d6be2ccc12f04d884a0e76010ff49ab523be37c3 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Mon, 16 Oct 2023 17:00:38 -0400
Subject: [PATCH 214/256] Updates after spot checking new data run

---
 .../nircam_monitors/claw_monitor.py           | 58 +++++++++++++------
 1 file changed, 39 insertions(+), 19 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index ab1c10bfc..ef1c508ca 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -20,7 +20,7 @@
 
 from astropy.convolution import Gaussian2DKernel, convolve
 from astropy.io import fits
-from astropy.stats import gaussian_fwhm_to_sigma
+from astropy.stats import gaussian_fwhm_to_sigma, sigma_clipped_stats
 from astropy.time import Time
 from astropy.visualization import ZScaleInterval
 from astroquery.mast import Mast
@@ -28,10 +28,13 @@
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import numpy as np
-from photutils import detect_sources, detect_threshold
+from photutils.segmentation import detect_sources, detect_threshold
 
-#from jwql.utils import monitor_utils  # todo uncomment
-#from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
+from jwql.database.database_interface import session
+from jwql.database.database_interface import NIRCamClawQueryHistory, NIRCamClawStats
+from jwql.utils import monitor_utils  # todo uncomment
+from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
 
 class ClawMonitor():
@@ -63,17 +66,20 @@ def process(self):
             fs = 20
         
         # Make source-masked, median-stack of each detector's images
+        logging.info('Working on claw stack: {}'.format(self.outfile))
         print(self.outfile)
         print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
         found_scale = False
         for i,det in enumerate(detectors_to_run):
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
-            files = [f for f in files if os.path.exists(f)][0:2]  # todo change index value?
+            files = [f for f in files if os.path.exists(f)][0:5]  # todo change index value?
             stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
             print(files)
             print('------')
+            backgrounds = []
+            fraction_masked = []
             for n,f in enumerate(files):
                 # Get pointing and other info from first image
                 if n == 0:
@@ -81,18 +87,23 @@ def process(self):
                     obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
                     obs_start_mjd = h[0].header['EXPSTART']
                     targname, ra_v1, dec_v1, pa_v3 = h[0].header['TARGPROP'], h[1].header['RA_V1'], h[1].header['DEC_V1'], h[1].header['PA_V3']
+                    print(obs_start, obs_start_mjd)
+                    print(targname, ra_v1, dec_v1, pa_v3)
                     h.close()
 
-                # Make source segmap, and add the masked data to the stack
+                # Make source segmap, add the masked data to the stack, and get background stats
                 data = fits.getdata(f, 'SCI')
-                threshold = detect_threshold(data, 1.25)
+                threshold = detect_threshold(data, 1.0)
                 sigma = 3.0 * gaussian_fwhm_to_sigma  # FWHM = 3.
                 kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
                 kernel.normalize()
                 data_conv = convolve(data, kernel)
-                segmap = detect_sources(data_conv, threshold, npixels=3)
+                segmap = detect_sources(data_conv, threshold, npixels=6)
                 segmap = segmap.data
                 stack[n] = np.ma.masked_array(data, mask=segmap!=0)
+                fraction_masked.append(len(segmap[segmap!=0]) / (segmap.shape[0]*segmap.shape[1]))
+                mean, med, stddev = sigma_clipped_stats(data[segmap!=0])
+                backgrounds.append(med)
 
             # Make the normalized skyflat for this detector
             skyflat = np.ma.median(stack, axis=0)
@@ -130,6 +141,7 @@ def process(self):
             fig.savefig(self.outfile, dpi=100, bbox_inches='tight')
         fig.clf()
         plt.close()
+        logging.info('Claw stack complete: {}'.format(self.outfile))
 
     def query_mast(self):
         """Query MAST for new nircam full-frame imaging data.
@@ -166,30 +178,38 @@ def query_mast(self):
 
         return t
 
-    #@log_fail  # todo uncomment
-    #@log_info  # todo uncomment
+    @log_fail  # todo uncomment
+    @log_info  # todo uncomment
     def run(self):
         """The main method.  See module docstrings for further details."""
 
         logging.info('Begin logging for claw_monitor')
-        self.output_dir = '/Users/bsunnquist/Documents/nircam/claw_monitor_testing/'  # todo change this to os.path.join(get_config()['outputs'], 'claw_monitor')
+        self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+        ensure_dir_exists(self.output_dir)
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
+        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
 
         # Query MAST for new imaging data from the last 3 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 3
-        self.query_end_mjd, self.query_start_mjd = 59878.986, 59878.934  # todo remove these test datess
+        #self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
+        self.query_start_mjd, self.query_end_mjd = 60150, 60152  # todo remove
+        #self.query_start_mjd = 59985  # last run was may 25; todo remove
+
         print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
-        print(t)
+        #print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
+        n_combos = len(np.unique(combos))
+        print('unique combos:')
         print(np.unique(combos))
         t['combos'] = combos
-        for combo in np.unique(combos)[0:2]:  # todo take off 0:2
+        for nnn,combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
+            print(combo, '{}/{}'.format(nnn,n_combos))
             tt = t[t['combos']==combo]
-            print(tt)
+            #print(tt)
             if 'long' in tt['filename'][0]:
                 self.wv = 'LW'
             else:
@@ -197,12 +217,12 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
             self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
-            print(self.files)
+            #print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 self.process()
             else:
-                print('{} already exists'.format(self.outfile))
+                logging.info('{} already exists'.format(self.outfile))
 
         logging.info('Claw Monitor completed successfully.')
 
@@ -210,9 +230,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    #start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # todo uncomment
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # todo uncomment
 
     monitor = ClawMonitor()
     monitor.run()
 
-    #monitor_utils.update_monitor_table(module, start_time, log_file)   # todo uncomment
+    monitor_utils.update_monitor_table(module, start_time, log_file)   # todo uncomment

From ab2d8036a9a0f42426d07e120d781eff1a16f5da Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Tue, 17 Oct 2023 12:22:41 -0400
Subject: [PATCH 215/256] adding new columns and other database issues

---
 .../nircam/nircam_claw_stats.txt              |  3 +-
 .../nircam_monitors/claw_monitor.py           | 67 +++++++++++++++----
 2 files changed, 55 insertions(+), 15 deletions(-)

diff --git a/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt b/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
index f6fc1131a..0730d1e3f 100644
--- a/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
+++ b/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
@@ -14,4 +14,5 @@ MEAN, float
 MEDIAN, float
 STDDEV, float
 FRAC_MASKED, float
-SKYFLAT_FILENAME, string
\ No newline at end of file
+SKYFLAT_FILENAME, string
+ENTRY_DATE, datetime
\ No newline at end of file
diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index ef1c508ca..a6957d370 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -15,6 +15,7 @@
         python claw_monitor.py
 """
 
+import datetime
 import logging
 import os
 
@@ -30,7 +31,7 @@
 import numpy as np
 from photutils.segmentation import detect_sources, detect_threshold
 
-from jwql.database.database_interface import session
+from jwql.database.database_interface import session, engine
 from jwql.database.database_interface import NIRCamClawQueryHistory, NIRCamClawStats
 from jwql.utils import monitor_utils  # todo uncomment
 from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
@@ -73,26 +74,22 @@ def process(self):
         for i,det in enumerate(detectors_to_run):
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
-            files = [f for f in files if os.path.exists(f)][0:5]  # todo change index value?
+            files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
             stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
             print(files)
             print('------')
-            backgrounds = []
-            fraction_masked = []
             for n,f in enumerate(files):
-                # Get pointing and other info from first image
+                h = fits.open(f)
+                
+                # Get plot label info from first image
                 if n == 0:
-                    h = fits.open(f)
                     obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
-                    obs_start_mjd = h[0].header['EXPSTART']
-                    targname, ra_v1, dec_v1, pa_v3 = h[0].header['TARGPROP'], h[1].header['RA_V1'], h[1].header['DEC_V1'], h[1].header['PA_V3']
-                    print(obs_start, obs_start_mjd)
-                    print(targname, ra_v1, dec_v1, pa_v3)
-                    h.close()
+                    pa_v3 = h[1].header['PA_V3']
 
                 # Make source segmap, add the masked data to the stack, and get background stats
-                data = fits.getdata(f, 'SCI')
+                data = h['SCI'].data
+                dq = h['DQ'].data
                 threshold = detect_threshold(data, 1.0)
                 sigma = 3.0 * gaussian_fwhm_to_sigma  # FWHM = 3.
                 kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
@@ -100,10 +97,35 @@ def process(self):
                 data_conv = convolve(data, kernel)
                 segmap = detect_sources(data_conv, threshold, npixels=6)
                 segmap = segmap.data
+                segmap[dq&1!=0] = 1  # flag DO_NOT_USE pixels
                 stack[n] = np.ma.masked_array(data, mask=segmap!=0)
-                fraction_masked.append(len(segmap[segmap!=0]) / (segmap.shape[0]*segmap.shape[1]))
                 mean, med, stddev = sigma_clipped_stats(data[segmap!=0])
-                backgrounds.append(med)
+                
+                # Add this file's stats to the claw database table.
+                # Can't insert values with numpy.float32 datatypes into database
+                # so need to change the datatypes of these values.
+                claw_db_entry = {'filename': os.path.basename(f),
+                                 'proposal': self.proposal,
+                                 'obs': self.obs,
+                                 'detector': det,
+                                 'filter': self.fltr,
+                                 'pupil': self.pupil,
+                                 'expstart': '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS']),
+                                 'expstart_mjd': h[0].header['EXPSTART'],
+                                 'effexptm': h[0].header['EFFEXPTM'],
+                                 'ra': h[1].header['RA_V1'],
+                                 'dec': h[1].header['DEC_V1'],
+                                 'pa_v3': h[1].header['PA_V3'],
+                                 'mean': float(mean),
+                                 'median': float(med),
+                                 'stddev': float(stddev),
+                                 'frac_masked': len(segmap[segmap!=0]) / (segmap.shape[0]*segmap.shape[1]),
+                                 'skyflat_filename': os.path.basename(self.outfile),
+                                 'entry_date': datetime.datetime.now()
+                                }
+                with engine.begin() as connection:
+                    connection.execute(self.stats_table.__table__.insert(), claw_db_entry)
+                h.close()
 
             # Make the normalized skyflat for this detector
             skyflat = np.ma.median(stack, axis=0)
@@ -126,9 +148,11 @@ def process(self):
                 found_scale = True
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
+                print(det, vmin, vmax)
             else:
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
+                print(det, vmin, vmax)
             ax.axes.get_xaxis().set_ticks([])
             ax.axes.get_yaxis().set_ticks([])
         
@@ -189,6 +213,10 @@ def run(self):
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
         self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
 
+        # Get the claw monitor database tables
+        self.query_table = eval('NIRCamClawQueryHistory')
+        self.stats_table = eval('NIRCamClawStats')
+
         # Query MAST for new imaging data from the last 3 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 3
@@ -206,6 +234,7 @@ def run(self):
         print('unique combos:')
         print(np.unique(combos))
         t['combos'] = combos
+        monitor_run = False
         for nnn,combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
             print(combo, '{}/{}'.format(nnn,n_combos))
             tt = t[t['combos']==combo]
@@ -221,9 +250,19 @@ def run(self):
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 self.process()
+                monitor_run = True
             else:
                 logging.info('{} already exists'.format(self.outfile))
 
+        # Update the query history
+        new_entry = {'instrument': 'nircam',
+                     'start_time_mjd': self.query_start_mjd,
+                     'end_time_mjd': self.query_end_mjd,
+                     'run_monitor': monitor_run,
+                     'entry_date': datetime.datetime.now()}
+        with engine.begin() as connection:
+            connection.execute(self.query_table.__table__.insert(), new_entry)
+
         logging.info('Claw Monitor completed successfully.')
 
 

From d8219c73f7d10c35d0a9f3e2847bb3bd41963bd5 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Tue, 17 Oct 2023 16:36:11 -0400
Subject: [PATCH 216/256] claw monitor webpages and databases working

---
 .../nircam_monitors/claw_monitor.py           | 11 +++---
 jwql/utils/constants.py                       |  1 +
 jwql/website/apps/jwql/monitor_views.py       | 35 +++++++++++++++++++
 .../apps/jwql/templates/claw_monitor.html     | 25 +++++++++++++
 jwql/website/apps/jwql/urls.py                |  3 ++
 5 files changed, 70 insertions(+), 5 deletions(-)
 create mode 100644 jwql/website/apps/jwql/templates/claw_monitor.html

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index a6957d370..6edcafd38 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -67,11 +67,11 @@ def process(self):
             fs = 20
         
         # Make source-masked, median-stack of each detector's images
-        logging.info('Working on claw stack: {}'.format(self.outfile))
         print(self.outfile)
         print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
         found_scale = False
         for i,det in enumerate(detectors_to_run):
+            logging.info('Working on {}'.format(det))
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
             files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
@@ -80,6 +80,7 @@ def process(self):
             print(files)
             print('------')
             for n,f in enumerate(files):
+                logging.info('Working on: {}'.format(f))
                 h = fits.open(f)
                 
                 # Get plot label info from first image
@@ -99,7 +100,7 @@ def process(self):
                 segmap = segmap.data
                 segmap[dq&1!=0] = 1  # flag DO_NOT_USE pixels
                 stack[n] = np.ma.masked_array(data, mask=segmap!=0)
-                mean, med, stddev = sigma_clipped_stats(data[segmap!=0])
+                mean, med, stddev = sigma_clipped_stats(data[segmap==0])
                 
                 # Add this file's stats to the claw database table.
                 # Can't insert values with numpy.float32 datatypes into database
@@ -148,11 +149,9 @@ def process(self):
                 found_scale = True
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
-                print(det, vmin, vmax)
             else:
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
-                print(det, vmin, vmax)
             ax.axes.get_xaxis().set_ticks([])
             ax.axes.get_yaxis().set_ticks([])
         
@@ -165,7 +164,7 @@ def process(self):
             fig.savefig(self.outfile, dpi=100, bbox_inches='tight')
         fig.clf()
         plt.close()
-        logging.info('Claw stack complete: {}'.format(self.outfile))
+        logging.info('Claw stacks complete: {}'.format(self.outfile))
 
     def query_mast(self):
         """Query MAST for new nircam full-frame imaging data.
@@ -226,6 +225,7 @@ def run(self):
 
         print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
+        logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
         #print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
@@ -249,6 +249,7 @@ def run(self):
             #print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
+                logging.info('Working on {}'.format(self.outfile))
                 self.process()
                 monitor_run = True
             else:
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index dee4123d2..b5728a8c4 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -378,6 +378,7 @@
              ('Readnoise Monitor', '/miri/readnoise_monitor')],
     'nircam': [('Bad Pixel Monitor', '/nircam/bad_pixel_monitor'),
                ('Bias Monitor', '/nircam/bias_monitor'),
+               ('Claw Monitor', '/nircam/claw_monitor'),
                ('Cosmic Ray Monitor', '#'),
                ('Dark Current Monitor', '/nircam/dark_monitor'),
                ('EDB Telemetry Monitor', '/nircam/edb_monitor'),
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index b62f339de..f271e69b8 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -30,12 +30,16 @@
 
 import os
 
+from astropy.time import Time
 from bokeh.resources import CDN, INLINE
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
 import json
+import pandas as pd
 
 from . import bokeh_containers
+from jwql.database.database_interface import session
+from jwql.database.database_interface import NIRCamClawStats
 from jwql.website.apps.jwql import bokeh_containers
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.utils import get_config, get_base_url
@@ -100,6 +104,37 @@ def bias_monitor(request, inst):
     return render(request, template, context)
 
 
+def claw_monitor(request):
+    """Generate the NIRCam claw monitor page
+
+    Parameters
+    ----------
+    request : HttpRequest object
+        Incoming request from the webpage
+
+    Returns
+    -------
+    HttpResponse object
+        Outgoing response sent to the webpage
+    """
+
+    template = "claw_monitor.html"
+
+    # Get all recent claw stack images
+    query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
+    df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd']>Time.now().mjd-100]))  # todo change 100 to 10 days back?
+    claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
+
+    context = {
+        'inst': 'NIRCam',
+        'claw_stacks': claw_stacks
+        }
+
+    # Return a HTTP response with the template and dictionary of variables
+    return render(request, template, context)
+
+
 def cosmic_ray_monitor(request, inst):
     """Generate the cosmic ray monitor page for a given instrument
 
diff --git a/jwql/website/apps/jwql/templates/claw_monitor.html b/jwql/website/apps/jwql/templates/claw_monitor.html
new file mode 100644
index 000000000..f4af8d8be
--- /dev/null
+++ b/jwql/website/apps/jwql/templates/claw_monitor.html
@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+
+{% block preamble %}
+
+<title>{{ inst }} Claw Monitor - JWQL</title>
+
+{% endblock %}
+
+{% block content %}
+
+    <main role="main" class="container">
+
+		<h1>{{ inst }} Claw Monitor</h1>
+		<hr>
+
+		<div width=100%>
+			{% for claw_stack in claw_stacks %}
+				<img src={{ claw_stack | safe }} width="100%">
+				<p></p>
+			{% endfor %}
+		</div>
+
+	</main>
+
+{% endblock %}
\ No newline at end of file
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index 0b4420080..e483cff9d 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -58,6 +58,9 @@
     # Home
     path('', views.home, name='home'),
 
+    # NIRCam-specific views
+    path('nircam/claw_monitor/', monitor_views.claw_monitor, name='claw_monitor'), 
+
     # NIRSpec-specific views
     path('nirspec/msata_monitor/', monitor_views.msata_monitoring, name='msata_monitor'),
     path('nirspec/wata_monitor/', monitor_views.wata_monitoring, name='wata_monitor'),

From 8e9a1dc83c2cacac4cc3b40799d77b5f365e8622 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 18 Oct 2023 13:26:33 -0400
Subject: [PATCH 217/256] added background monitoring plots and webpages

---
 .../nircam_monitors/claw_monitor.py           | 148 ++++++++++++++++--
 jwql/utils/constants.py                       |   3 +-
 jwql/website/apps/jwql/monitor_views.py       |  28 ++++
 .../jwql/templates/background_monitor.html    |  25 +++
 jwql/website/apps/jwql/urls.py                |   1 +
 5 files changed, 188 insertions(+), 17 deletions(-)
 create mode 100644 jwql/website/apps/jwql/templates/background_monitor.html

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 6edcafd38..9e3e2c2fa 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -29,6 +29,7 @@
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import numpy as np
+import pandas as pd
 from photutils.segmentation import detect_sources, detect_threshold
 
 from jwql.database.database_interface import session, engine
@@ -40,12 +41,118 @@
 
 class ClawMonitor():
     """Class for executing the claw monitor.
+
+    This class searches for all new NIRCam full-frame imaging data
+    and creates observation-level, source-masked, median stacks
+    for each filter/pupil combination. These stacks are then plotted
+    in on-sky orientation and the results are used to identify new
+    instances of claws - a scattered light effect seen in NIRCam
+    data. Background statistics are also stored for each individual
+    image, which are then plotted to track the NIRCam background
+    levels over time. Results are all saved to database tables.
+
+    Attributes
+    ----------
+    outfile : str
+        The name of the output plot for a given claw stack combination.
+
+    output_dir : str
+        Path into which claw stack plots will be placed.
+
+    output_dir_bkg : str
+        Path into which background trending plots will be placed.
+
+    query_start : float
+        MJD start date to use for querying MAST.
+
+    query_end : float
+        MJD end date to use for querying MAST.
+
+    wv : str
+        NIRCam channel for a given claw stack, either ``SW`` or ``LW``.
+
+    proposal : str
+        NIRCam proposal number for a given claw stack.
+
+    obs : str
+        NIRCam observation number for a given claw stack.
+
+    fltr : str
+        NIRCam filter used for a given claw stack.
+
+    pupil : str
+        NIRCam pupil used for a given claw stack.
+
+    detectors : str
+        The detectors used for a given claw stack combination.
+
+    files : numpy.ndarray
+        The names of the individual files belonging to a given claw stack combination.
     """
 
     def __init__(self):
         """Initialize an instance of the ``ClawMonitor`` class.
         """
 
+    def make_background_plots(self):
+        """Makes plots of the background levels over time in NIRCam data.
+        """
+
+        # Get all of the background data.
+        query = session.query(NIRCamClawStats.filename, NIRCamClawStats.filter, NIRCamClawStats.pupil, NIRCamClawStats.detector,
+                              NIRCamClawStats.effexptm, NIRCamClawStats.expstart_mjd, NIRCamClawStats.entry_date, NIRCamClawStats.mean, 
+                              NIRCamClawStats.median, NIRCamClawStats.frac_masked).all()
+        df_orig = pd.DataFrame(query, columns=['filename', 'filter', 'pupil', 'detector', 'effexptm', 'expstart_mjd', 
+                                               'entry_date', 'mean', 'median', 'frac_masked'])
+        df_orig = df_orig.drop_duplicates(subset='filename', keep="last")  # remove any duplicate filename entries, keep the most recent
+
+        # Make backgroud trending plots for all SW wide filters
+        for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W']:
+            logging.info('Working on background trending plots for {}'.format(fltr))
+            grid = plt.GridSpec(2, 4, hspace=.2, wspace=.2, width_ratios=[1,1,1,1])
+            fig = plt.figure(figsize=(40, 20))
+            fig.suptitle(fltr, fontsize=45)
+            for i,det in enumerate(['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']):  # in on-sky order, don't change order
+                logging.info('Working on {}'.format(det))
+                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
+                # extended objects, nebulas, short exposures.
+                df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
+                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<0.075) & (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
+                
+                # Plot the background levels over time
+                ax = fig.add_subplot(grid[i])
+                ax.scatter(df['expstart_mjd'], df['median'])
+                ax.set_title(det, fontsize=30)
+                ax.set_ylabel('Background Level [MJy/sr]')
+                ax.set_xlabel('Date [MJD]')
+            fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
+            fig.clf()
+            plt.close()
+
+
+        # Make backgroud trending plots for all LW wide filters
+        for fltr in ['F277W', 'F356W', 'F444W']:
+            logging.info('Working on background trending plots for {}'.format(fltr))
+            grid = plt.GridSpec(1, 2, hspace=.2, wspace=.2, width_ratios=[1,1])
+            fig = plt.figure(figsize=(20, 10))
+            fig.suptitle(fltr, fontsize=30)
+            for i,det in enumerate(['NRCALONG', 'NRCBLONG']):
+                logging.info('Working on {}'.format(det))
+                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
+                # extended objects, nebulas, short exposures.
+                df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
+                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<0.15) & (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
+                
+                # Plot the background levels over time
+                ax = fig.add_subplot(grid[i])
+                ax.scatter(df['expstart_mjd'], df['median'])
+                ax.set_title(det, fontsize=20)
+                ax.set_ylabel('Background Level [MJy/sr]')
+                ax.set_xlabel('Date [MJD]')
+            fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
+            fig.clf()
+            plt.close()
+
     def process(self):
         """The main method for processing.  See module docstrings for further details.
         """
@@ -73,7 +180,8 @@ def process(self):
         for i,det in enumerate(detectors_to_run):
             logging.info('Working on {}'.format(det))
             files = self.files[self.detectors == det]
-            # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
+            # Remove missing files; to avoid memory/speed issues, only use the first 20 files, 
+            # which should be plenty to see any claws todo change value?
             files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
             stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
@@ -102,15 +210,14 @@ def process(self):
                 stack[n] = np.ma.masked_array(data, mask=segmap!=0)
                 mean, med, stddev = sigma_clipped_stats(data[segmap==0])
                 
-                # Add this file's stats to the claw database table.
-                # Can't insert values with numpy.float32 datatypes into database
-                # so need to change the datatypes of these values.
+                # Add this file's stats to the claw database table. Can't insert values with numpy.float32
+                # datatypes into database so need to change the datatypes of these values.
                 claw_db_entry = {'filename': os.path.basename(f),
                                  'proposal': self.proposal,
                                  'obs': self.obs,
-                                 'detector': det,
-                                 'filter': self.fltr,
-                                 'pupil': self.pupil,
+                                 'detector': det.upper(),
+                                 'filter': self.fltr.upper(),
+                                 'pupil': self.pupil.upper(),
                                  'expstart': '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS']),
                                  'expstart_mjd': h[0].header['EXPSTART'],
                                  'effexptm': h[0].header['EFFEXPTM'],
@@ -134,7 +241,7 @@ def process(self):
             skyflat = skyflat / np.nanmedian(skyflat)
             skyflat[~np.isfinite(skyflat)] = 1  # fill missing values
 
-            # Add the skyflat for this detector to the plot
+            # Add the skyflat for this detector to the claw stack plot
             if (self.wv=='SW') & (i>3):  # skip colobar axis
                 idx = i+1
             else:
@@ -155,7 +262,7 @@ def process(self):
             ax.axes.get_xaxis().set_ticks([])
             ax.axes.get_yaxis().set_ticks([])
         
-        # Add colobar, save figure if any detector stacks exist
+        # Add colobar, save figure if any claw stacks exist
         if found_scale:
             fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
             cax = fig.add_subplot(grid[0:rows, cols-1:cols])
@@ -167,7 +274,7 @@ def process(self):
         logging.info('Claw stacks complete: {}'.format(self.outfile))
 
     def query_mast(self):
-        """Query MAST for new nircam full-frame imaging data.
+        """Query MAST for new NIRCam full-frame imaging data.
 
         Returns
         -------
@@ -207,22 +314,25 @@ def run(self):
         """The main method.  See module docstrings for further details."""
 
         logging.info('Begin logging for claw_monitor')
+
+        # Define and setup the output directories for the claw and background plots.
         self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
         ensure_dir_exists(self.output_dir)
-        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
+        self.output_dir_bkg =  os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
+        ensure_dir_exists(self.output_dir_bkg)
+        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
         self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
 
         # Get the claw monitor database tables
         self.query_table = eval('NIRCamClawQueryHistory')
         self.stats_table = eval('NIRCamClawStats')
 
-        # Query MAST for new imaging data from the last 3 days
+        # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
-        self.query_start_mjd = self.query_end_mjd - 3
+        self.query_start_mjd = self.query_end_mjd - 2
         #self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
         self.query_start_mjd, self.query_end_mjd = 60150, 60152  # todo remove
         #self.query_start_mjd = 59985  # last run was may 25; todo remove
-
         print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
@@ -246,6 +356,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
             self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
             #print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
@@ -255,6 +366,11 @@ def run(self):
             else:
                 logging.info('{} already exists'.format(self.outfile))
 
+        # Update the background trending plots, if any new data exists
+        if len(t)>0:
+            logging.info('Making background trending plots.')
+            self.make_background_plots()
+
         # Update the query history
         new_entry = {'instrument': 'nircam',
                      'start_time_mjd': self.query_start_mjd,
@@ -270,9 +386,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # todo uncomment
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
 
     monitor = ClawMonitor()
     monitor.run()
 
-    monitor_utils.update_monitor_table(module, start_time, log_file)   # todo uncomment
+    monitor_utils.update_monitor_table(module, start_time, log_file)
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index b5728a8c4..52679037b 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -376,7 +376,8 @@
              ('Dark Current Monitor', '/miri/dark_monitor'),
              ('EDB Telemetry Monitor', '/miri/edb_monitor'),
              ('Readnoise Monitor', '/miri/readnoise_monitor')],
-    'nircam': [('Bad Pixel Monitor', '/nircam/bad_pixel_monitor'),
+    'nircam': [('Background Monitor', '/nircam/background_monitor'),
+               ('Bad Pixel Monitor', '/nircam/bad_pixel_monitor'),
                ('Bias Monitor', '/nircam/bias_monitor'),
                ('Claw Monitor', '/nircam/claw_monitor'),
                ('Cosmic Ray Monitor', '#'),
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index f271e69b8..d36c5ad07 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -51,6 +51,34 @@
 FILESYSTEM_DIR = os.path.join(CONFIG['jwql_dir'], 'filesystem')
 
 
+def background_monitor(request):
+    """Generate the NIRCam background monitor page
+
+    Parameters
+    ----------
+    request : HttpRequest object
+        Incoming request from the webpage
+
+    Returns
+    -------
+    HttpResponse object
+        Outgoing response sent to the webpage
+    """
+
+    template = "background_monitor.html"
+
+    # Get the background trending filters to display
+    fltrs = ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']
+    bkg_plots = ['/static/outputs/claw_monitor/backgrounds/{}_backgrounds.png'.format(fltr) for fltr in fltrs]
+
+    context = {
+        'inst': 'NIRCam',
+        'bkg_plots': bkg_plots
+        }
+
+    # Return a HTTP response with the template and dictionary of variables
+    return render(request, template, context)
+
 def bad_pixel_monitor(request, inst):
     """Generate the dark monitor page for a given instrument
 
diff --git a/jwql/website/apps/jwql/templates/background_monitor.html b/jwql/website/apps/jwql/templates/background_monitor.html
new file mode 100644
index 000000000..efe0c122b
--- /dev/null
+++ b/jwql/website/apps/jwql/templates/background_monitor.html
@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+
+{% block preamble %}
+
+<title>{{ inst }} Background Monitor - JWQL</title>
+
+{% endblock %}
+
+{% block content %}
+
+    <main role="main" class="container">
+
+		<h1>{{ inst }} Background Monitor</h1>
+		<hr>
+
+		<div width=100%>
+			{% for bkg_plot in bkg_plots %}
+				<img src={{ bkg_plot | safe }} width="100%">
+				<p></p>
+			{% endfor %}
+		</div>
+
+	</main>
+
+{% endblock %}
\ No newline at end of file
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index e483cff9d..638e7501f 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -59,6 +59,7 @@
     path('', views.home, name='home'),
 
     # NIRCam-specific views
+    path('nircam/background_monitor/', monitor_views.background_monitor, name='background_monitor'), 
     path('nircam/claw_monitor/', monitor_views.claw_monitor, name='claw_monitor'), 
 
     # NIRSpec-specific views

From b2bd048679a43dde9ab91cc82d83e62eff0342cf Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 18 Oct 2023 17:09:07 -0400
Subject: [PATCH 218/256] updates to background trending plots

---
 .../nircam_monitors/claw_monitor.py           | 58 +++++++++++++++----
 .../jwql/templates/background_monitor.html    |  2 +-
 .../apps/jwql/templates/claw_monitor.html     |  2 +-
 3 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 9e3e2c2fa..379f6e35f 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -106,12 +106,18 @@ def make_background_plots(self):
                                                'entry_date', 'mean', 'median', 'frac_masked'])
         df_orig = df_orig.drop_duplicates(subset='filename', keep="last")  # remove any duplicate filename entries, keep the most recent
 
+        # Use the same time xlimits/xticks for all plots
+        start_mjd = 59650  # March 2022, middle of commissioning
+        end_mjd = Time.now().mjd + 0.05*(Time.now().mjd - start_mjd)
+        time_tick_vals = np.linspace(start_mjd, end_mjd, 5)  # Make April 2022 the first tick
+        time_tick_labels = [Time(m, format='mjd').isot.split('T')[0] for m in time_tick_vals]
+
         # Make backgroud trending plots for all SW wide filters
         for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
-            grid = plt.GridSpec(2, 4, hspace=.2, wspace=.2, width_ratios=[1,1,1,1])
+            grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1,1,1,1])
             fig = plt.figure(figsize=(40, 20))
-            fig.suptitle(fltr, fontsize=45)
+            fig.suptitle(fltr, fontsize=70)
             for i,det in enumerate(['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']):  # in on-sky order, don't change order
                 logging.info('Working on {}'.format(det))
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
@@ -122,9 +128,25 @@ def make_background_plots(self):
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
-                ax.set_title(det, fontsize=30)
-                ax.set_ylabel('Background Level [MJy/sr]')
-                ax.set_xlabel('Date [MJD]')
+
+                # Match scaling in all plots to the first detector. Shade median+/-10% region.
+                if len(df)>0:
+                    if i==0:
+                        first_med = np.nanmedian(df['median'])
+                    ax.set_ylim(first_med-first_med*0.5, first_med+first_med*0.5)
+                    med = np.nanmedian(df['median'])
+                    ax.axhline(med, ls='-', color='black')
+                    ax.axhspan(med-med*0.1, med+med*0.1, color='gray', alpha=0.4, lw=0)
+
+                # Axis formatting
+                ax.set_title(det, fontsize=40)
+                ax.set_xlim(start_mjd, end_mjd)
+                ax.set_xticks(time_tick_vals)
+                ax.set_xticklabels(time_tick_labels, fontsize=20, rotation=45)
+                ax.yaxis.set_tick_params(labelsize=20)
+                ax.set_ylabel('Background [MJy/sr]', fontsize=30)
+                #ax.set_xlabel('Date [YYYY-MM-DD]')
+                ax.grid(ls='--', color='gray')
             fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
             fig.clf()
             plt.close()
@@ -133,9 +155,9 @@ def make_background_plots(self):
         # Make backgroud trending plots for all LW wide filters
         for fltr in ['F277W', 'F356W', 'F444W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
-            grid = plt.GridSpec(1, 2, hspace=.2, wspace=.2, width_ratios=[1,1])
+            grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1,1])
             fig = plt.figure(figsize=(20, 10))
-            fig.suptitle(fltr, fontsize=30)
+            fig.suptitle(fltr, fontsize=70, y=1.05)
             for i,det in enumerate(['NRCALONG', 'NRCBLONG']):
                 logging.info('Working on {}'.format(det))
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
@@ -146,9 +168,25 @@ def make_background_plots(self):
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
-                ax.set_title(det, fontsize=20)
-                ax.set_ylabel('Background Level [MJy/sr]')
-                ax.set_xlabel('Date [MJD]')
+
+                # Match y scaling in all plots to the first detector. Shade median+/-10% region.
+                if len(df)>0:
+                    if i==0:
+                        first_med = np.nanmedian(df['median'])
+                    ax.set_ylim(first_med-first_med*0.5, first_med+first_med*0.5)
+                    med = np.nanmedian(df['median'])
+                    ax.axhline(med, ls='-', color='black')
+                    ax.axhspan(med-med*0.1, med+med*0.1, color='gray', alpha=0.4, lw=0)
+
+                # Axis formatting
+                ax.set_title(det, fontsize=40)
+                ax.set_xlim(start_mjd, end_mjd)
+                ax.set_xticks(time_tick_vals)
+                ax.set_xticklabels(time_tick_labels, fontsize=20, rotation=45)
+                ax.yaxis.set_tick_params(labelsize=20)
+                ax.set_ylabel('Background [MJy/sr]', fontsize=30)
+                #ax.set_xlabel('Date [YYYY-MM-DD]')
+                ax.grid(ls='--', color='gray')
             fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
             fig.clf()
             plt.close()
diff --git a/jwql/website/apps/jwql/templates/background_monitor.html b/jwql/website/apps/jwql/templates/background_monitor.html
index efe0c122b..ddbf53c6f 100644
--- a/jwql/website/apps/jwql/templates/background_monitor.html
+++ b/jwql/website/apps/jwql/templates/background_monitor.html
@@ -16,7 +16,7 @@ <h1>{{ inst }} Background Monitor</h1>
 		<div width=100%>
 			{% for bkg_plot in bkg_plots %}
 				<img src={{ bkg_plot | safe }} width="100%">
-				<p></p>
+				<br><br><hr>
 			{% endfor %}
 		</div>
 
diff --git a/jwql/website/apps/jwql/templates/claw_monitor.html b/jwql/website/apps/jwql/templates/claw_monitor.html
index f4af8d8be..0bae20e65 100644
--- a/jwql/website/apps/jwql/templates/claw_monitor.html
+++ b/jwql/website/apps/jwql/templates/claw_monitor.html
@@ -16,7 +16,7 @@ <h1>{{ inst }} Claw Monitor</h1>
 		<div width=100%>
 			{% for claw_stack in claw_stacks %}
 				<img src={{ claw_stack | safe }} width="100%">
-				<p></p>
+				<br><br><hr>
 			{% endfor %}
 		</div>
 

From b47dadb434b6d03d30833907e415b89b2beb8a53 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 10:33:32 -0400
Subject: [PATCH 219/256] combined sw and lw plotting in single loop

---
 .../nircam_monitors/claw_monitor.py           | 107 +++++++-----------
 1 file changed, 44 insertions(+), 63 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 379f6e35f..b912534ff 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -13,6 +13,13 @@
     ::
 
         python claw_monitor.py
+
+    To only update the background trending plots:
+
+    ::
+
+        m = ClawMonitor()
+        m.make_background_plots()
 """
 
 import datetime
@@ -94,6 +101,18 @@ def __init__(self):
         """Initialize an instance of the ``ClawMonitor`` class.
         """
 
+        # Define and setup the output directories for the claw and background plots.
+        self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+        ensure_dir_exists(self.output_dir)
+        self.output_dir_bkg =  os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
+        ensure_dir_exists(self.output_dir_bkg)
+        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
+        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
+
+        # Get the claw monitor database tables
+        self.query_table = eval('NIRCamClawQueryHistory')
+        self.stats_table = eval('NIRCamClawStats')
+
     def make_background_plots(self):
         """Makes plots of the background levels over time in NIRCam data.
         """
@@ -109,67 +128,38 @@ def make_background_plots(self):
         # Use the same time xlimits/xticks for all plots
         start_mjd = 59650  # March 2022, middle of commissioning
         end_mjd = Time.now().mjd + 0.05*(Time.now().mjd - start_mjd)
-        time_tick_vals = np.linspace(start_mjd, end_mjd, 5)  # Make April 2022 the first tick
+        time_tick_vals = np.linspace(start_mjd, end_mjd, 5)
         time_tick_labels = [Time(m, format='mjd').isot.split('T')[0] for m in time_tick_vals]
 
-        # Make backgroud trending plots for all SW wide filters
-        for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W']:
+        # Make backgroud trending plots for all wide filters
+        for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
-            grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1,1,1,1])
-            fig = plt.figure(figsize=(40, 20))
-            fig.suptitle(fltr, fontsize=70)
-            for i,det in enumerate(['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']):  # in on-sky order, don't change order
+            if int(fltr[1:4])<250:  # i.e. SW
+                detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']   # in on-sky order, don't change order
+                grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1,1,1,1])
+                fig = plt.figure(figsize=(40, 20))
+                fig.suptitle(fltr, fontsize=70)
+                frack_masked_thresh = 0.075
+            else:  # i.e. LW
+                detectors_to_run = ['NRCALONG', 'NRCBLONG']
+                grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1,1])
+                fig = plt.figure(figsize=(20, 10))
+                fig.suptitle(fltr, fontsize=70, y=1.05)
+                frack_masked_thresh = 0.15
+            for i,det in enumerate(detectors_to_run):
                 logging.info('Working on {}'.format(det))
-                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
-                # extended objects, nebulas, short exposures.
-                df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
-                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<0.075) & (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
-                
-                # Plot the background levels over time
-                ax = fig.add_subplot(grid[i])
-                ax.scatter(df['expstart_mjd'], df['median'])
 
-                # Match scaling in all plots to the first detector. Shade median+/-10% region.
-                if len(df)>0:
-                    if i==0:
-                        first_med = np.nanmedian(df['median'])
-                    ax.set_ylim(first_med-first_med*0.5, first_med+first_med*0.5)
-                    med = np.nanmedian(df['median'])
-                    ax.axhline(med, ls='-', color='black')
-                    ax.axhspan(med-med*0.1, med+med*0.1, color='gray', alpha=0.4, lw=0)
-
-                # Axis formatting
-                ax.set_title(det, fontsize=40)
-                ax.set_xlim(start_mjd, end_mjd)
-                ax.set_xticks(time_tick_vals)
-                ax.set_xticklabels(time_tick_labels, fontsize=20, rotation=45)
-                ax.yaxis.set_tick_params(labelsize=20)
-                ax.set_ylabel('Background [MJy/sr]', fontsize=30)
-                #ax.set_xlabel('Date [YYYY-MM-DD]')
-                ax.grid(ls='--', color='gray')
-            fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
-            fig.clf()
-            plt.close()
-
-
-        # Make backgroud trending plots for all LW wide filters
-        for fltr in ['F277W', 'F356W', 'F444W']:
-            logging.info('Working on background trending plots for {}'.format(fltr))
-            grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1,1])
-            fig = plt.figure(figsize=(20, 10))
-            fig.suptitle(fltr, fontsize=70, y=1.05)
-            for i,det in enumerate(['NRCALONG', 'NRCBLONG']):
-                logging.info('Working on {}'.format(det))
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
                 # extended objects, nebulas, short exposures.
                 df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
-                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<0.15) & (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
+                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<frack_masked_thresh) & 
+                             (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
                 
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
 
-                # Match y scaling in all plots to the first detector. Shade median+/-10% region.
+                # Match scaling in all plots to the first detector. Shade median+/-10% region.
                 if len(df)>0:
                     if i==0:
                         first_med = np.nanmedian(df['median'])
@@ -302,7 +292,8 @@ def process(self):
         
         # Add colobar, save figure if any claw stacks exist
         if found_scale:
-            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
+            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), 
+                         self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
             cax = fig.add_subplot(grid[0:rows, cols-1:cols])
             cbar = fig.colorbar(im, cax=cax, orientation='vertical')
             cbar.ax.tick_params(labelsize=cbar_fs)
@@ -353,18 +344,6 @@ def run(self):
 
         logging.info('Begin logging for claw_monitor')
 
-        # Define and setup the output directories for the claw and background plots.
-        self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
-        ensure_dir_exists(self.output_dir)
-        self.output_dir_bkg =  os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
-        ensure_dir_exists(self.output_dir_bkg)
-        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
-        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
-
-        # Get the claw monitor database tables
-        self.query_table = eval('NIRCamClawQueryHistory')
-        self.stats_table = eval('NIRCamClawStats')
-
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
@@ -392,8 +371,10 @@ def run(self):
             else:
                 self.wv = 'SW'
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
-            self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
+                                        self.obs, self.fltr, self.pupil).lower())
+            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 
+                            'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
             #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
             #print(self.files)
             self.detectors = np.array(tt['detector'])

From a5ac0352621c0fe009eb35ec5afaf1b1a7aa76be Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 10:59:33 -0400
Subject: [PATCH 220/256] pep8

---
 .../nircam_monitors/claw_monitor.py           | 126 +++++++++---------
 jwql/website/apps/jwql/monitor_views.py       |   7 +-
 jwql/website/apps/jwql/urls.py                |   4 +-
 3 files changed, 68 insertions(+), 69 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index b912534ff..e71ec52fa 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -104,7 +104,7 @@ def __init__(self):
         # Define and setup the output directories for the claw and background plots.
         self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
         ensure_dir_exists(self.output_dir)
-        self.output_dir_bkg =  os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
+        self.output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
         ensure_dir_exists(self.output_dir_bkg)
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
         self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
@@ -119,54 +119,54 @@ def make_background_plots(self):
 
         # Get all of the background data.
         query = session.query(NIRCamClawStats.filename, NIRCamClawStats.filter, NIRCamClawStats.pupil, NIRCamClawStats.detector,
-                              NIRCamClawStats.effexptm, NIRCamClawStats.expstart_mjd, NIRCamClawStats.entry_date, NIRCamClawStats.mean, 
+                              NIRCamClawStats.effexptm, NIRCamClawStats.expstart_mjd, NIRCamClawStats.entry_date, NIRCamClawStats.mean,
                               NIRCamClawStats.median, NIRCamClawStats.frac_masked).all()
-        df_orig = pd.DataFrame(query, columns=['filename', 'filter', 'pupil', 'detector', 'effexptm', 'expstart_mjd', 
+        df_orig = pd.DataFrame(query, columns=['filename', 'filter', 'pupil', 'detector', 'effexptm', 'expstart_mjd',
                                                'entry_date', 'mean', 'median', 'frac_masked'])
         df_orig = df_orig.drop_duplicates(subset='filename', keep="last")  # remove any duplicate filename entries, keep the most recent
 
         # Use the same time xlimits/xticks for all plots
         start_mjd = 59650  # March 2022, middle of commissioning
-        end_mjd = Time.now().mjd + 0.05*(Time.now().mjd - start_mjd)
+        end_mjd = Time.now().mjd + 0.05 * (Time.now().mjd - start_mjd)
         time_tick_vals = np.linspace(start_mjd, end_mjd, 5)
         time_tick_labels = [Time(m, format='mjd').isot.split('T')[0] for m in time_tick_vals]
 
         # Make backgroud trending plots for all wide filters
         for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
-            if int(fltr[1:4])<250:  # i.e. SW
+            if int(fltr[1:4]) < 250:  # i.e. SW
                 detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']   # in on-sky order, don't change order
-                grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1,1,1,1])
+                grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1, 1, 1, 1])
                 fig = plt.figure(figsize=(40, 20))
                 fig.suptitle(fltr, fontsize=70)
                 frack_masked_thresh = 0.075
             else:  # i.e. LW
                 detectors_to_run = ['NRCALONG', 'NRCBLONG']
-                grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1,1])
+                grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1, 1])
                 fig = plt.figure(figsize=(20, 10))
                 fig.suptitle(fltr, fontsize=70, y=1.05)
                 frack_masked_thresh = 0.15
-            for i,det in enumerate(detectors_to_run):
+            for i, det in enumerate(detectors_to_run):
                 logging.info('Working on {}'.format(det))
 
-                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
+                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields,
                 # extended objects, nebulas, short exposures.
-                df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
-                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<frack_masked_thresh) & 
-                             (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
-                
+                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det) &
+                             (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh) &
+                             (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
+
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
 
                 # Match scaling in all plots to the first detector. Shade median+/-10% region.
-                if len(df)>0:
-                    if i==0:
+                if len(df) > 0:
+                    if i == 0:
                         first_med = np.nanmedian(df['median'])
-                    ax.set_ylim(first_med-first_med*0.5, first_med+first_med*0.5)
+                    ax.set_ylim(first_med - first_med * 0.5, first_med + first_med * 0.5)
                     med = np.nanmedian(df['median'])
                     ax.axhline(med, ls='-', color='black')
-                    ax.axhspan(med-med*0.1, med+med*0.1, color='gray', alpha=0.4, lw=0)
+                    ax.axhspan(med - med * 0.1, med + med * 0.1, color='gray', alpha=0.4, lw=0)
 
                 # Axis formatting
                 ax.set_title(det, fontsize=40)
@@ -175,7 +175,7 @@ def make_background_plots(self):
                 ax.set_xticklabels(time_tick_labels, fontsize=20, rotation=45)
                 ax.yaxis.set_tick_params(labelsize=20)
                 ax.set_ylabel('Background [MJy/sr]', fontsize=30)
-                #ax.set_xlabel('Date [YYYY-MM-DD]')
+                # ax.set_xlabel('Date [YYYY-MM-DD]')
                 ax.grid(ls='--', color='gray')
             fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
             fig.clf()
@@ -189,36 +189,36 @@ def process(self):
         if self.wv == 'SW':
             detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']  # in on-sky order, don't change order
             cols, rows = 5, 2
-            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,1,1,.1])
+            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1, 1, 1, 1, .1])
             fig = plt.figure(figsize=(40, 20))
             cbar_fs = 20
             fs = 30
         else:
             detectors_to_run = ['NRCALONG', 'NRCBLONG']
             cols, rows = 3, 1
-            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,.1])
+            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1, 1, .1])
             fig = plt.figure(figsize=(20, 10))
             cbar_fs = 10
             fs = 20
-        
+
         # Make source-masked, median-stack of each detector's images
         print(self.outfile)
         print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
         found_scale = False
-        for i,det in enumerate(detectors_to_run):
+        for i, det in enumerate(detectors_to_run):
             logging.info('Working on {}'.format(det))
             files = self.files[self.detectors == det]
-            # Remove missing files; to avoid memory/speed issues, only use the first 20 files, 
+            # Remove missing files; to avoid memory/speed issues, only use the first 20 files,
             # which should be plenty to see any claws todo change value?
             files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
             stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
             print(files)
             print('------')
-            for n,f in enumerate(files):
+            for n, f in enumerate(files):
                 logging.info('Working on: {}'.format(f))
                 h = fits.open(f)
-                
+
                 # Get plot label info from first image
                 if n == 0:
                     obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
@@ -234,10 +234,10 @@ def process(self):
                 data_conv = convolve(data, kernel)
                 segmap = detect_sources(data_conv, threshold, npixels=6)
                 segmap = segmap.data
-                segmap[dq&1!=0] = 1  # flag DO_NOT_USE pixels
-                stack[n] = np.ma.masked_array(data, mask=segmap!=0)
-                mean, med, stddev = sigma_clipped_stats(data[segmap==0])
-                
+                segmap[dq & 1 != 0] = 1  # flag DO_NOT_USE pixels
+                stack[n] = np.ma.masked_array(data, mask=segmap != 0)
+                mean, med, stddev = sigma_clipped_stats(data[segmap == 0])
+
                 # Add this file's stats to the claw database table. Can't insert values with numpy.float32
                 # datatypes into database so need to change the datatypes of these values.
                 claw_db_entry = {'filename': os.path.basename(f),
@@ -255,10 +255,10 @@ def process(self):
                                  'mean': float(mean),
                                  'median': float(med),
                                  'stddev': float(stddev),
-                                 'frac_masked': len(segmap[segmap!=0]) / (segmap.shape[0]*segmap.shape[1]),
+                                 'frac_masked': len(segmap[segmap != 0]) / (segmap.shape[0] * segmap.shape[1]),
                                  'skyflat_filename': os.path.basename(self.outfile),
                                  'entry_date': datetime.datetime.now()
-                                }
+                                 }
                 with engine.begin() as connection:
                     connection.execute(self.stats_table.__table__.insert(), claw_db_entry)
                 h.close()
@@ -270,15 +270,15 @@ def process(self):
             skyflat[~np.isfinite(skyflat)] = 1  # fill missing values
 
             # Add the skyflat for this detector to the claw stack plot
-            if (self.wv=='SW') & (i>3):  # skip colobar axis
-                idx = i+1
+            if (self.wv == 'SW') & (i > 3):  # skip colobar axis
+                idx = i + 1
             else:
                 idx = i
             ax = fig.add_subplot(grid[idx])
-            if len(skyflat[skyflat!=1])==0:
+            if len(skyflat[skyflat != 1]) == 0:
                 ax.set_title('N/A', fontsize=fs)
                 ax.imshow(skyflat, cmap='coolwarm', vmin=999, vmax=999, origin='lower')
-            elif (len(skyflat[skyflat!=1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
+            elif (len(skyflat[skyflat != 1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
                 z = ZScaleInterval()
                 vmin, vmax = z.get_limits(skyflat)
                 found_scale = True
@@ -289,12 +289,12 @@ def process(self):
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
             ax.axes.get_xaxis().set_ticks([])
             ax.axes.get_yaxis().set_ticks([])
-        
+
         # Add colobar, save figure if any claw stacks exist
         if found_scale:
-            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), 
-                         self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
-            cax = fig.add_subplot(grid[0:rows, cols-1:cols])
+            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(),
+                         self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs * 1.5)
+            cax = fig.add_subplot(grid[0:rows, cols - 1:cols])
             cbar = fig.colorbar(im, cax=cax, orientation='vertical')
             cbar.ax.tick_params(labelsize=cbar_fs)
             fig.savefig(self.outfile, dpi=100, bbox_inches='tight')
@@ -318,22 +318,20 @@ def query_mast(self):
         JwstObs._portal_api_connection.COLUMNS_CONFIG_URL = server + "/portal_jwst/Mashup/Mashup.asmx/columnsconfig"
         JwstObs._portal_api_connection.MAST_BUNDLE_URL = server + "/jwst/api/v0.1/download/bundle"
         service = 'Mast.Jwst.Filtered.Nircam'
-        FIELDS = ['filename','program', 'observtn','category','instrume', 'productLevel', 'filter', 
-                  'pupil', 'subarray', 'detector','datamodl','date_beg_mjd', 'effexptm']
-        params = {"columns":",".join(FIELDS),
-                "filters":[
-                {"paramName":"pupil","values":['CLEAR','F162M','F164N','F323N','F405N','F466N','F470N']},
-                {"paramName":"exp_type","values":['NRC_IMAGE']},
-                {"paramName":"datamodl", "values":['ImageModel']},  # exclude calints, which are cubemodel
-                {"paramName":"productLevel", "values":['2b']},  # i.e. cal.fits
-                {"paramName":"subarray", "values":['FULL']},
-                ]
-                }
+        FIELDS = ['filename', 'program', 'observtn', 'category', 'instrume', 'productLevel', 'filter',
+                  'pupil', 'subarray', 'detector', 'datamodl', 'date_beg_mjd', 'effexptm']
+        params = {"columns" : ",".join(FIELDS),
+                "filters":[{"paramName" : "pupil","values" : ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
+                           {"paramName" : "exp_type","values" : ['NRC_IMAGE']},
+                           {"paramName" : "datamodl", "values" : ['ImageModel']},  # exclude calints, which are cubemodel
+                           {"paramName" : "productLevel", "values" : ['2b']},  # i.e. cal.fits
+                           {"paramName" : "subarray", "values" : ['FULL']},]
+                 }
         t = JwstObs.service_request(service, params)
-        t = t[(t['date_beg_mjd']>self.query_start_mjd) & (t['date_beg_mjd']<self.query_end_mjd)]
+        t = t[(t['date_beg_mjd'] > self.query_start_mjd) & (t['date_beg_mjd'] < self.query_end_mjd)]
         t.sort('date_beg_mjd')
-        filetypes = np.array([row['filename'].split('_')[-1].replace('.fits','') for row in t])
-        t = t[filetypes=='cal']  # only want cal.fits files, no e.g. i2d.fits
+        filetypes = np.array([row['filename'].split('_')[-1].replace('.fits', '') for row in t])
+        t = t[filetypes == 'cal']  # only want cal.fits files, no e.g. i2d.fits
 
         return t
 
@@ -347,13 +345,13 @@ def run(self):
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
-        #self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
+        # self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
         self.query_start_mjd, self.query_end_mjd = 60150, 60152  # todo remove
-        #self.query_start_mjd = 59985  # last run was may 25; todo remove
+        # self.query_start_mjd = 59985  # last run was may 25; todo remove
         print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
-        #print(t)
+        # print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
@@ -362,10 +360,10 @@ def run(self):
         print(np.unique(combos))
         t['combos'] = combos
         monitor_run = False
-        for nnn,combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
-            print(combo, '{}/{}'.format(nnn,n_combos))
-            tt = t[t['combos']==combo]
-            #print(tt)
+        for nnn, combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
+            print(combo, '{}/{}'.format(nnn, n_combos))
+            tt = t[t['combos'] == combo]
+            # print(tt)
             if 'long' in tt['filename'][0]:
                 self.wv = 'LW'
             else:
@@ -374,9 +372,9 @@ def run(self):
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
             self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 
-                            'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
-            #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
-            #print(self.files)
+                                  'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            # self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
+            # print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
@@ -386,7 +384,7 @@ def run(self):
                 logging.info('{} already exists'.format(self.outfile))
 
         # Update the background trending plots, if any new data exists
-        if len(t)>0:
+        if len(t) > 0:
             logging.info('Making background trending plots.')
             self.make_background_plots()
 
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index d36c5ad07..86be368f6 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -74,11 +74,12 @@ def background_monitor(request):
     context = {
         'inst': 'NIRCam',
         'bkg_plots': bkg_plots
-        }
+    }
 
     # Return a HTTP response with the template and dictionary of variables
     return render(request, template, context)
 
+
 def bad_pixel_monitor(request, inst):
     """Generate the dark monitor page for a given instrument
 
@@ -151,13 +152,13 @@ def claw_monitor(request):
     # Get all recent claw stack images
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
-    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd']>Time.now().mjd-100]))  # todo change 100 to 10 days back?
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 100]))  # todo change 100 to 10 days back?
     claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
 
     context = {
         'inst': 'NIRCam',
         'claw_stacks': claw_stacks
-        }
+    }
 
     # Return a HTTP response with the template and dictionary of variables
     return render(request, template, context)
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index 638e7501f..64c00c53a 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -59,8 +59,8 @@
     path('', views.home, name='home'),
 
     # NIRCam-specific views
-    path('nircam/background_monitor/', monitor_views.background_monitor, name='background_monitor'), 
-    path('nircam/claw_monitor/', monitor_views.claw_monitor, name='claw_monitor'), 
+    path('nircam/background_monitor/', monitor_views.background_monitor, name='background_monitor'),
+    path('nircam/claw_monitor/', monitor_views.claw_monitor, name='claw_monitor'),
 
     # NIRSpec-specific views
     path('nirspec/msata_monitor/', monitor_views.msata_monitoring, name='msata_monitor'),

From 5b53632df4266fca18dee844acd432dc93a27b70 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 11:05:29 -0400
Subject: [PATCH 221/256] pep8

---
 .../nircam_monitors/claw_monitor.py           | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index e71ec52fa..a2c7ea8f7 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -107,7 +107,7 @@ def __init__(self):
         self.output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
         ensure_dir_exists(self.output_dir_bkg)
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
-        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
+        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  # todo remove
 
         # Get the claw monitor database tables
         self.query_table = eval('NIRCamClawQueryHistory')
@@ -151,9 +151,9 @@ def make_background_plots(self):
 
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields,
                 # extended objects, nebulas, short exposures.
-                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det) &
-                             (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh) &
-                             (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
+                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det)
+                             & (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh)
+                             & (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
 
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
@@ -320,13 +320,13 @@ def query_mast(self):
         service = 'Mast.Jwst.Filtered.Nircam'
         FIELDS = ['filename', 'program', 'observtn', 'category', 'instrume', 'productLevel', 'filter',
                   'pupil', 'subarray', 'detector', 'datamodl', 'date_beg_mjd', 'effexptm']
-        params = {"columns" : ",".join(FIELDS),
-                "filters":[{"paramName" : "pupil","values" : ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
-                           {"paramName" : "exp_type","values" : ['NRC_IMAGE']},
-                           {"paramName" : "datamodl", "values" : ['ImageModel']},  # exclude calints, which are cubemodel
-                           {"paramName" : "productLevel", "values" : ['2b']},  # i.e. cal.fits
-                           {"paramName" : "subarray", "values" : ['FULL']},]
-                 }
+        params = {"columns": ",".join(FIELDS),
+                  "filters":[{"paramName": "pupil", "values": ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
+                             {"paramName": "exp_type", "values": ['NRC_IMAGE']},
+                             {"paramName": "datamodl", "values": ['ImageModel']},  # exclude calints, which are cubemodel
+                             {"paramName": "productLevel", "values": ['2b']},  # i.e. cal.fits
+                             {"paramName": "subarray", "values": ['FULL']}, ]
+                  }
         t = JwstObs.service_request(service, params)
         t = t[(t['date_beg_mjd'] > self.query_start_mjd) & (t['date_beg_mjd'] < self.query_end_mjd)]
         t.sort('date_beg_mjd')
@@ -371,7 +371,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 
+            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)),
                                   'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
             # self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
             # print(self.files)

From 7e3bc889a55f5e7b2192fec91b46a38cef8d2054 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 11:07:15 -0400
Subject: [PATCH 222/256] pep8

---
 .../nircam_monitors/claw_monitor.py              | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index a2c7ea8f7..1a874b27b 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -151,9 +151,9 @@ def make_background_plots(self):
 
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields,
                 # extended objects, nebulas, short exposures.
-                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det)
-                             & (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh)
-                             & (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
+                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det) &
+                             (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh) &
+                             (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
 
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
@@ -321,11 +321,11 @@ def query_mast(self):
         FIELDS = ['filename', 'program', 'observtn', 'category', 'instrume', 'productLevel', 'filter',
                   'pupil', 'subarray', 'detector', 'datamodl', 'date_beg_mjd', 'effexptm']
         params = {"columns": ",".join(FIELDS),
-                  "filters":[{"paramName": "pupil", "values": ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
-                             {"paramName": "exp_type", "values": ['NRC_IMAGE']},
-                             {"paramName": "datamodl", "values": ['ImageModel']},  # exclude calints, which are cubemodel
-                             {"paramName": "productLevel", "values": ['2b']},  # i.e. cal.fits
-                             {"paramName": "subarray", "values": ['FULL']}, ]
+                  "filters": [{"paramName": "pupil", "values": ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
+                              {"paramName": "exp_type", "values": ['NRC_IMAGE']},
+                              {"paramName": "datamodl", "values": ['ImageModel']},  # exclude calints, which are cubemodel
+                              {"paramName": "productLevel", "values": ['2b']},  # i.e. cal.fits
+                              {"paramName": "subarray", "values": ['FULL']}, ]
                   }
         t = JwstObs.service_request(service, params)
         t = t[(t['date_beg_mjd'] > self.query_start_mjd) & (t['date_beg_mjd'] < self.query_end_mjd)]

From 582e2e832556278364f6d578a7903306890618a6 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 14:40:35 -0400
Subject: [PATCH 223/256] removed testing statements

---
 .../nircam_monitors/claw_monitor.py           | 36 +++++--------------
 jwql/website/apps/jwql/monitor_views.py       |  4 +--
 2 files changed, 10 insertions(+), 30 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 1a874b27b..30c2895b2 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -41,8 +41,8 @@
 
 from jwql.database.database_interface import session, engine
 from jwql.database.database_interface import NIRCamClawQueryHistory, NIRCamClawStats
-from jwql.utils import monitor_utils  # todo uncomment
-from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
+from jwql.utils import monitor_utils
+from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
 
@@ -106,8 +106,6 @@ def __init__(self):
         ensure_dir_exists(self.output_dir)
         self.output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
         ensure_dir_exists(self.output_dir_bkg)
-        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
-        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  # todo remove
 
         # Get the claw monitor database tables
         self.query_table = eval('NIRCamClawQueryHistory')
@@ -202,19 +200,14 @@ def process(self):
             fs = 20
 
         # Make source-masked, median-stack of each detector's images
-        print(self.outfile)
-        print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
         found_scale = False
         for i, det in enumerate(detectors_to_run):
             logging.info('Working on {}'.format(det))
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files,
-            # which should be plenty to see any claws todo change value?
-            files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
+            # which should be plenty to see any claws.
+            files = [f for f in files if os.path.exists(f)][0:20]
             stack = np.ma.ones((len(files), 2048, 2048))
-            print(det)
-            print(files)
-            print('------')
             for n, f in enumerate(files):
                 logging.info('Working on: {}'.format(f))
                 h = fits.open(f)
@@ -335,8 +328,8 @@ def query_mast(self):
 
         return t
 
-    @log_fail  # todo uncomment
-    @log_info  # todo uncomment
+    @log_fail
+    @log_info
     def run(self):
         """The main method.  See module docstrings for further details."""
 
@@ -345,25 +338,15 @@ def run(self):
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
-        # self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
-        self.query_start_mjd, self.query_end_mjd = 60150, 60152  # todo remove
-        # self.query_start_mjd = 59985  # last run was may 25; todo remove
-        print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
-        # print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
-        n_combos = len(np.unique(combos))
-        print('unique combos:')
-        print(np.unique(combos))
         t['combos'] = combos
         monitor_run = False
-        for nnn, combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
-            print(combo, '{}/{}'.format(nnn, n_combos))
+        for combo in np.unique(combos):
             tt = t[t['combos'] == combo]
-            # print(tt)
             if 'long' in tt['filename'][0]:
                 self.wv = 'LW'
             else:
@@ -371,10 +354,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)),
-                                  'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
-            # self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
-            # print(self.files)
+            self.files = np.array([filesystem_path(row['filename']) for row in tt])
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index 86be368f6..dc070b9fc 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -149,10 +149,10 @@ def claw_monitor(request):
 
     template = "claw_monitor.html"
 
-    # Get all recent claw stack images
+    # Get all recent claw stack images from the last 10 days
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
-    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 100]))  # todo change 100 to 10 days back?
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 10]))
     claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
 
     context = {

From e6e43bef746b0fabb2e2fdcf4b78439fdd7332d8 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Thu, 19 Oct 2023 15:21:28 -0400
Subject: [PATCH 224/256] Pep8 fixes

---
 .../jwql/monitor_pages/monitor_readnoise_bokeh.py     | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 7657aeeed..82f0fe914 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -129,12 +129,11 @@ def plot_readnoise_amplifers(self):
             ngroups = [result.ngroups for result in self.db.query_results]
 
             source = ColumnDataSource(data=dict(
-                        file=filenames,
-                        expstarts=expstarts,
-                        nints=nints,
-                        ngroups=ngroups,
-                        readnoise=readnoise_vals,
-                        ))
+                                      file=filenames,
+                                      expstarts=expstarts,
+                                      nints=nints,
+                                      ngroups=ngroups,
+                                      readnoise=readnoise_vals))
 
             amp_plot.add_tools(HoverTool(tooltips=[("file", "@file"),
                                                    ("time", "@expstarts"),

From 056295b2351d8f614aab0afdcc564220c8605fda Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Fri, 20 Oct 2023 10:16:58 -0400
Subject: [PATCH 225/256] a couple changes for testing purposes

---
 jwql/instrument_monitors/nircam_monitors/claw_monitor.py | 4 +++-
 jwql/website/apps/jwql/monitor_views.py                  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 30c2895b2..f9826cf81 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -338,6 +338,7 @@ def run(self):
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
+        self.query_start_mjd, self.query_end_mjd = 59715.28951771492, 59715.29771992559  # todo remove
         t = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
 
@@ -354,7 +355,8 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([filesystem_path(row['filename']) for row in tt])
+            #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment?
+            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in tt])
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index dc070b9fc..d8ff30f3f 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -152,7 +152,7 @@ def claw_monitor(request):
     # Get all recent claw stack images from the last 10 days
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
-    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 10]))
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 1000]))  # todo change from 1000 to 10
     claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
 
     context = {

From 33c8ddb5e5af9b3afce9d9e0fa694840495abdda Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Fri, 20 Oct 2023 11:03:29 -0400
Subject: [PATCH 226/256] Addressing Bryans comments

---
 .../monitor_pages/monitor_readnoise_bokeh.py  | 39 +++++++++++++------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 82f0fe914..552cac71f 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -41,6 +41,8 @@ class ReadnoiseMonitorData():
 
     instrument : str
         Instrument name (e.g. nircam)
+    aperture : str
+        Aperture name (e.g. apername)
 
     Attributes
     ----------
@@ -49,7 +51,12 @@ class ReadnoiseMonitorData():
         Instrument name (e.g. nircam)
     aperture : str
         Aperture name (e.g. apername)
-
+    query_results : list
+        Results from read noise statistics table based on
+        instrument, aperture and exposure start time
+    stats_table : sqlalchemy.orm.decl_api.DeclarativeMeta
+        Statistics table object to query based on instrument
+        and aperture
     """
 
     def __init__(self, instrument, aperture):
@@ -113,14 +120,18 @@ def __init__(self, instrument, aperture):
                          title=self.aperture)
 
     def plot_readnoise_amplifers(self):
-
+        """Class to create readnoise scatter plots per amplifier.
+        """
         self.amp_plots = []
         for amp in ['1', '2', '3', '4']:
 
             amp_plot = figure(title='Amp {}'.format(amp), width=280, height=280, x_axis_type='datetime')
             amp_plot.xaxis[0].ticker.desired_num_ticks = 4
 
-            readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results])
+            if self.db.query_results:
+                readnoise_vals = np.array([getattr(result, 'amp{}_mean'.format(amp)) for result in self.db.query_results])
+            else:
+                readnoise_vals = np.array(list())
 
             filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results]
             expstarts_iso = np.array([result.expstart for result in self.db.query_results])
@@ -160,18 +171,24 @@ def plot_readnoise_difference_image(self):
             diff_image_png = self.db.query_results[-1].readnoise_diff_image
             diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:]))
             self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left")
-            self.diff_image_plot.xaxis.visible = False
-            self.diff_image_plot.yaxis.visible = False
-            self.diff_image_plot.xgrid.grid_line_color = None
-            self.diff_image_plot.ygrid.grid_line_color = None
-            self.diff_image_plot.title.text_font_size = '22px'
-            self.diff_image_plot.title.align = 'center'
+            
+        self.diff_image_plot.xaxis.visible = False
+        self.diff_image_plot.yaxis.visible = False
+        self.diff_image_plot.xgrid.grid_line_color = None
+        self.diff_image_plot.ygrid.grid_line_color = None
+        self.diff_image_plot.title.text_font_size = '22px'
+        self.diff_image_plot.title.align = 'center'
+
 
     def plot_readnoise_histogram(self):
         """Updates the readnoise histogram"""
 
-        diff_image_n = np.array(self.db.query_results[-1].diff_image_n)
-        diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers)
+        if len(self.db.query_results) != 0:
+            diff_image_n = np.array(self.db.query_results[-1].diff_image_n)
+            diff_image_bin_centers = np.array(self.db.query_results[-1].diff_image_bin_centers)
+        else:
+            diff_image_n = np.array(list())
+            diff_image_bin_centers = np.array(list())
 
         hist_xr_start = diff_image_bin_centers.min()
         hist_xr_end = diff_image_bin_centers.max()

From 82109f5ed1b32d26c5a4813485f0c208ca3b39b2 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Fri, 20 Oct 2023 16:22:18 -0400
Subject: [PATCH 227/256] Removed zscaling; fixed potential future plot limit
 issue

---
 .../nircam_monitors/claw_monitor.py                 | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index f9826cf81..67a3bfc49 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -30,7 +30,6 @@
 from astropy.io import fits
 from astropy.stats import gaussian_fwhm_to_sigma, sigma_clipped_stats
 from astropy.time import Time
-from astropy.visualization import ZScaleInterval
 from astroquery.mast import Mast
 import matplotlib
 matplotlib.use('Agg')
@@ -132,6 +131,7 @@ def make_background_plots(self):
         # Make backgroud trending plots for all wide filters
         for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
+            found_limits = False
             if int(fltr[1:4]) < 250:  # i.e. SW
                 detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']   # in on-sky order, don't change order
                 grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1, 1, 1, 1])
@@ -157,10 +157,11 @@ def make_background_plots(self):
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
 
-                # Match scaling in all plots to the first detector. Shade median+/-10% region.
+                # Match scaling in all plots to the first detector with data. Shade median+/-10% region.
                 if len(df) > 0:
-                    if i == 0:
+                    if found_limits is False:
                         first_med = np.nanmedian(df['median'])
+                        found_limits = True
                     ax.set_ylim(first_med - first_med * 0.5, first_med + first_med * 0.5)
                     med = np.nanmedian(df['median'])
                     ax.axhline(med, ls='-', color='black')
@@ -272,8 +273,8 @@ def process(self):
                 ax.set_title('N/A', fontsize=fs)
                 ax.imshow(skyflat, cmap='coolwarm', vmin=999, vmax=999, origin='lower')
             elif (len(skyflat[skyflat != 1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
-                z = ZScaleInterval()
-                vmin, vmax = z.get_limits(skyflat)
+                mean, med, stddev = sigma_clipped_stats(skyflat)
+                vmin, vmax =  med - 3 * stddev, med + 3 * stddev
                 found_scale = True
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
@@ -356,7 +357,7 @@ def run(self):
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
             #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment?
-            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in tt])
+            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in tt]) # todo remove
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))

From 7206e33ab83d0df2f8cc6c043f47e0e91d9bf96d Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 24 Oct 2023 09:55:55 -0400
Subject: [PATCH 228/256] update actions/checkout to v4

---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3b047377a..54147a822 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -14,7 +14,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.x"
@@ -32,7 +32,7 @@ jobs:
         python-version: [3.9, "3.10"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: mamba-org/provision-with-micromamba@v15
         with:
@@ -43,7 +43,7 @@ jobs:
       - run: pip install -e .[test]
 
       - run: conda env export
-      
+
       - uses: supercharge/redis-github-action@1.4.0
         with:
           redis-version: 5.0

From fd08a85ebc818e3255f87b82f07ba3fabb96d6c1 Mon Sep 17 00:00:00 2001
From: "york@stsci.edu" <york@stsci.edu>
Date: Fri, 3 Nov 2023 11:57:57 -0400
Subject: [PATCH 229/256] Updated readnoise database entry and retrieval

---
 .../common_monitors/readnoise_monitor.py             |  9 ++++++---
 .../jwql/monitor_pages/monitor_readnoise_bokeh.py    | 12 ++++++++----
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py
index 5e4f7bb7f..60d889133 100755
--- a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py
@@ -498,7 +498,10 @@ def process(self, file_list):
             # Construct new entry for this file for the readnoise database table.
             # Can't insert values with numpy.float32 datatypes into database
             # so need to change the datatypes of these values.
-            readnoise_db_entry = {'uncal_filename': filename,
+            #
+            # Store files as file name only (file path will be built at retrieval based
+            # on runtime configuration)
+            readnoise_db_entry = {'uncal_filename': os.path.basename(filename),
                                   'aperture': self.aperture,
                                   'detector': self.detector,
                                   'subarray': self.subarray,
@@ -506,12 +509,12 @@ def process(self, file_list):
                                   'nints': self.nints,
                                   'ngroups': self.ngroups,
                                   'expstart': self.expstart,
-                                  'readnoise_filename': readnoise_outfile,
+                                  'readnoise_filename': os.path.basename(readnoise_outfile),
                                   'full_image_mean': float(full_image_mean),
                                   'full_image_stddev': float(full_image_stddev),
                                   'full_image_n': full_image_n.astype(float),
                                   'full_image_bin_centers': full_image_bin_centers.astype(float),
-                                  'readnoise_diff_image': readnoise_diff_png,
+                                  'readnoise_diff_image': os.path.basename(readnoise_diff_png),
                                   'diff_image_mean': float(diff_image_mean),
                                   'diff_image_stddev': float(diff_image_stddev),
                                   'diff_image_n': diff_image_n.astype(float),
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 552cac71f..1ece690f3 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -31,7 +31,9 @@
 from jwql.database.database_interface import session
 from jwql.database.database_interface import FGSReadnoiseStats, MIRIReadnoiseStats, NIRCamReadnoiseStats, NIRISSReadnoiseStats, NIRSpecReadnoiseStats
 from jwql.utils.constants import FULL_FRAME_APERTURES, JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.utils import get_config
 
+OUTPUTS_DIR = get_config()['outputs']
 
 class ReadnoiseMonitorData():
     """Class to hold bias data to be plotted
@@ -107,12 +109,15 @@ class ReadNoisePlotTab():
     def __init__(self, instrument, aperture):
         self.instrument = instrument
         self.aperture = aperture
+        self.ins_ap = "{}_{}".format(self.instrument.lower(), self.aperture.lower())
 
         self.db = ReadnoiseMonitorData(self.instrument, self.aperture)
 
         self.plot_readnoise_amplifers()
         self.plot_readnoise_difference_image()
         self.plot_readnoise_histogram()
+        
+        self.file_path = os.path.join(OUTPUT_DIR, "readnoise_monitor", "data", self.ins_ap)
 
         self.tab = Panel(child=column(row(*self.amp_plots),
                                       self.diff_image_plot,
@@ -133,7 +138,7 @@ def plot_readnoise_amplifers(self):
             else:
                 readnoise_vals = np.array(list())
 
-            filenames = [os.path.basename(result.uncal_filename).replace('_uncal.fits', '') for result in self.db.query_results]
+            filenames = [result.uncal_filename.replace('_uncal.fits', '') for result in self.db.query_results]
             expstarts_iso = np.array([result.expstart for result in self.db.query_results])
             expstarts = np.array([datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f') for date in expstarts_iso])
             nints = [result.nints for result in self.db.query_results]
@@ -151,7 +156,7 @@ def plot_readnoise_amplifers(self):
                                                    ("nints", "@nints"),
                                                    ("ngroups", "@ngroups"),
                                                    ("readnoise", "@readnoise")]))
-
+ 
             amp_plot.circle(x='expstarts', y='readnoise', source=source)
 
             amp_plot.xaxis.axis_label = 'Date'
@@ -168,8 +173,7 @@ def plot_readnoise_difference_image(self):
                                       height=500, width=500, sizing_mode='scale_width')
 
         if len(self.db.query_results) != 0:
-            diff_image_png = self.db.query_results[-1].readnoise_diff_image
-            diff_image_png = os.path.join('/static', '/'.join(diff_image_png.split('/')[-6:]))
+            diff_image_png = os.path.join(self.file_path, self.db.query_results[-1].readnoise_diff_image)
             self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left")
             
         self.diff_image_plot.xaxis.visible = False

From 8f7a65cb802a76b1028c2f28d65552046784824d Mon Sep 17 00:00:00 2001
From: "york@stsci.edu" <york@stsci.edu>
Date: Mon, 6 Nov 2023 15:28:12 -0500
Subject: [PATCH 230/256] PEP8

---
 .../common_monitors/readnoise_monitor.py          |  4 ++--
 .../jwql/monitor_pages/monitor_readnoise_bokeh.py | 15 ++++++---------
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py
index 60d889133..833db0fb7 100755
--- a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py
+++ b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py
@@ -413,7 +413,7 @@ def process(self, file_list):
             processed_file = file.replace("uncal", "refpix")
             if not os.path.isfile(processed_file):
                 files_to_calibrate.append(file)
-        
+
         # Run the files through the necessary pipeline steps
         outputs = run_parallel_pipeline(files_to_calibrate, "uncal", "refpix", self.instrument)
 
@@ -422,7 +422,7 @@ def process(self, file_list):
 
             # Get relevant header information for this file
             self.get_metadata(filename)
-            
+
             if filename in outputs:
                 processed_file = outputs[filename]
             else:
diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 1ece690f3..543dcac9a 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -35,6 +35,7 @@
 
 OUTPUTS_DIR = get_config()['outputs']
 
+
 class ReadnoiseMonitorData():
     """Class to hold bias data to be plotted
 
@@ -116,7 +117,7 @@ def __init__(self, instrument, aperture):
         self.plot_readnoise_amplifers()
         self.plot_readnoise_difference_image()
         self.plot_readnoise_histogram()
-        
+
         self.file_path = os.path.join(OUTPUT_DIR, "readnoise_monitor", "data", self.ins_ap)
 
         self.tab = Panel(child=column(row(*self.amp_plots),
@@ -156,7 +157,7 @@ def plot_readnoise_amplifers(self):
                                                    ("nints", "@nints"),
                                                    ("ngroups", "@ngroups"),
                                                    ("readnoise", "@readnoise")]))
- 
+
             amp_plot.circle(x='expstarts', y='readnoise', source=source)
 
             amp_plot.xaxis.axis_label = 'Date'
@@ -175,7 +176,7 @@ def plot_readnoise_difference_image(self):
         if len(self.db.query_results) != 0:
             diff_image_png = os.path.join(self.file_path, self.db.query_results[-1].readnoise_diff_image)
             self.diff_image_plot.image_url(url=[diff_image_png], x=0, y=0, w=2048, h=2048, anchor="bottom_left")
-            
+
         self.diff_image_plot.xaxis.visible = False
         self.diff_image_plot.yaxis.visible = False
         self.diff_image_plot.xgrid.grid_line_color = None
@@ -183,7 +184,6 @@ def plot_readnoise_difference_image(self):
         self.diff_image_plot.title.text_font_size = '22px'
         self.diff_image_plot.title.align = 'center'
 
-
     def plot_readnoise_histogram(self):
         """Updates the readnoise histogram"""
 
@@ -204,12 +204,9 @@ def plot_readnoise_histogram(self):
                                           y_range=(hist_yr_start, hist_yr_end),
                                           sizing_mode='scale_width')
 
-        source = ColumnDataSource(data=dict(
-                        x=diff_image_bin_centers,
-                        y=diff_image_n,
-                        ))
+        source = ColumnDataSource(data=dict(x=diff_image_bin_centers, y=diff_image_n, ))
 
-        self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"),]))
+        self.readnoise_histogram.add_tools(HoverTool(tooltips=[("Data (x, y)", "(@x, @y)"), ]))
 
         self.readnoise_histogram.circle(x='x', y='y', source=source)
 

From f927012f39a960373c31d74f41816638a72fc150 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 7 Nov 2023 16:54:08 -0500
Subject: [PATCH 231/256] implementing date range in query page

---
 jwql/utils/constants.py                       |  2 ++
 jwql/website/apps/jwql/data_containers.py     | 20 ++++++++++++++++++
 jwql/website/apps/jwql/forms.py               | 20 +++++++++++-------
 jwql/website/apps/jwql/templates/base.html    |  5 +++++
 .../apps/jwql/templates/jwql_query.html       | 21 +++++++++++++++++++
 jwql/website/apps/jwql/views.py               |  8 ++++---
 6 files changed, 66 insertions(+), 10 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index dee4123d2..92f189b5d 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -439,6 +439,7 @@ class QUERY_CONFIG_KEYS:
     INSTRUMENTS = "INSTRUMENTS"
     PROPOSAL_CATEGORY = "PROPOSAL_CATEGORY"
     LOOK_STATUS = "LOOK_STATUS"
+    DATE_RANGE = "DATE_RANGE"
     NUM_PER_PAGE = "NUM_PER_PAGE"
     SORT_TYPE = "SORT_TYPE"
     ANOMALIES = "ANOMALIES"
@@ -459,6 +460,7 @@ class QUERY_CONFIG_KEYS:
     QUERY_CONFIG_KEYS.LOOK_STATUS: [],
     QUERY_CONFIG_KEYS.NUM_PER_PAGE: 100,
     QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent',
+    QUERY_CONFIG_KEYS.DATE_RANGE: "",
     QUERY_CONFIG_KEYS.ANOMALIES: {},
     QUERY_CONFIG_KEYS.APERTURES: {},
     QUERY_CONFIG_KEYS.DETECTORS: {},
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index f4ecb6aa7..fb64f68da 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -48,6 +48,7 @@
 import pandas as pd
 import pyvo as vo
 import requests
+from datetime import datetime
 
 from jwql.database import database_interface as di
 from jwql.database.database_interface import load_connection
@@ -1500,6 +1501,18 @@ def get_rootnames_from_query(parameters):
     """
 
     filtered_rootnames = []
+    DATE_FORMAT = "%Y/%m/%d %I:%M%p" #noqa n806
+
+    # Parse DATE_RANGE string into correct format
+    date_range = parameters[QUERY_CONFIG_KEYS.DATE_RANGE]
+    start_date_range, stop_date_range = date_range.split(" - ")
+    # Parse the strings into datetime objects
+    start_datetime = datetime.strptime(start_date_range, DATE_FORMAT)
+    stop_datetime = datetime.strptime(stop_date_range, DATE_FORMAT)
+    # store as astroquery Time objects in isot format to be used in filter (with mjd format)
+    start_time  = Time(start_datetime.isoformat(), format="isot")
+    stop_time = Time(stop_datetime.isoformat(), format="isot")
+
     # Each Query Selection is Instrument specific
     for inst in parameters[QUERY_CONFIG_KEYS.INSTRUMENTS]:
         # Make sure instruments are of the proper format for the archive query
@@ -1509,6 +1522,13 @@ def get_rootnames_from_query(parameters):
         # General fields
         sort_type = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
         look_status = parameters[QUERY_CONFIG_KEYS.LOOK_STATUS]
+
+        # Get a queryset of all observations STARTING within our date range
+        current_ins_rootfileinfos = current_ins_rootfileinfos.filter(
+            expstart__gte=start_time.mjd)
+        current_ins_rootfileinfos = current_ins_rootfileinfos.filter(
+            expstart__lte=stop_time.mjd)
+
         if len(look_status) == 1:
             viewed = (look_status[0] == 'VIEWED')
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(viewed=viewed)
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index b0554b451..f156350b8 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -59,9 +59,10 @@ def view_function(request):
 from jwql.website.apps.jwql.models import Anomalies
 
 
-from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, ANOMALIES_PER_INSTRUMENT, APERTURES_PER_INSTRUMENT, DETECTOR_PER_INSTRUMENT, EXP_TYPE_PER_INSTRUMENT,
-                                  FILTERS_PER_INSTRUMENT, GENERIC_SUFFIX_TYPES, GRATING_PER_INSTRUMENT, GUIDER_FILENAME_TYPE, JWST_INSTRUMENT_NAMES_MIXEDCASE,
-                                  JWST_INSTRUMENT_NAMES_SHORTHAND, READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES, SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT,
+from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, ANOMALIES_PER_INSTRUMENT, APERTURES_PER_INSTRUMENT, DETECTOR_PER_INSTRUMENT,
+                                  EXP_TYPE_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT, GENERIC_SUFFIX_TYPES, GRATING_PER_INSTRUMENT,
+                                  GUIDER_FILENAME_TYPE, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_SHORTHAND,
+                                  READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES,  SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT,
                                   LOOK_OPTIONS, SORT_OPTIONS, PROPOSAL_CATEGORIES)
 from jwql.utils.utils import (get_config, get_rootnames_for_instrument_proposal, filename_parser, query_format)
 
@@ -144,6 +145,8 @@ class JwqlQueryForm(BaseForm):
     look_status = forms.MultipleChoiceField(
         required=False, choices=look_choices, widget=forms.CheckboxSelectMultiple)
 
+    date_range = forms.CharField(required=True)
+
     cat_choices = [(query_format(choice), query_format(choice)) for choice in PROPOSAL_CATEGORIES]
     proposal_category = forms.MultipleChoiceField(
         required=False, choices=cat_choices, widget=forms.CheckboxSelectMultiple)
@@ -304,8 +307,10 @@ def clean_search(self):
             # See if there are any matching proposals and, if so, what
             # instrument they are for
             proposal_string = '{:05d}'.format(int(search))
-            search_string_public = os.path.join(get_config()['filesystem'], 'public', 'jw{}'.format(proposal_string), '*', '*{}*.fits'.format(proposal_string))
-            search_string_proprietary = os.path.join(get_config()['filesystem'], 'proprietary', 'jw{}'.format(proposal_string), '*', '*{}*.fits'.format(proposal_string))
+            search_string_public = os.path.join(get_config()['filesystem'], 'public', 'jw{}'.format(proposal_string),
+                                                '*', '*{}*.fits'.format(proposal_string))
+            search_string_proprietary = os.path.join(get_config()['filesystem'], 'proprietary', 'jw{}'.format(proposal_string),
+                                                     '*', '*{}*.fits'.format(proposal_string))
             all_files = glob.glob(search_string_public)
             all_files.extend(glob.glob(search_string_proprietary))
 
@@ -335,9 +340,10 @@ def clean_search(self):
 
                 if len(set(all_instruments)) > 1:
                     # Technically all proposal have multiple instruments if you include guider data. Remove Guider Data
-                    instrument_routes = [format_html('<a href="/{}/archive/{}/obs{}">{}</a>', instrument, proposal_string[1:], all_observations[instrument][0], instrument) for instrument in set(all_instruments)]
+                    instrument_routes = [format_html('<a href="/{}/archive/{}/obs{}">{}</a>', instrument, proposal_string[1:],
+                                                     all_observations[instrument][0], instrument) for instrument in set(all_instruments)]
                     raise forms.ValidationError(
-                        mark_safe(('Proposal contains multiple instruments, please click instrument link to view data: {}.').format(', '.join(instrument_routes))))  # nosec
+                        mark_safe(('Proposal contains multiple instruments, please click instrument link to view data: {}.').format(', '.join(instrument_routes))))  # noqa
 
                 self.instrument = all_instruments[0]
             else:
diff --git a/jwql/website/apps/jwql/templates/base.html b/jwql/website/apps/jwql/templates/base.html
index a06f7ffe8..1d00bfaea 100644
--- a/jwql/website/apps/jwql/templates/base.html
+++ b/jwql/website/apps/jwql/templates/base.html
@@ -27,6 +27,11 @@
 		<script type="text/javascript" src="https://cdn.bokeh.org/bokeh/release/bokeh-tables-{{ bokeh_version }}.min.js"></script>
 		<script src="{{ static('js/jwql.js') }}?{% now 'U' %}"></script>
 
+		<!-- Needed for Daterange on Query Page -->
+		<script type="text/javascript" src="https://cdn.jsdelivr.net/momentjs/latest/moment.min.js"></script>
+		<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.min.js"></script>
+		<link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/daterangepicker/daterangepicker.css" />
+
 		{% block preamble %}
         {% endblock %}
 
diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index 4cd308acd..78cb34e74 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -105,6 +105,27 @@ <h1>Dynamic Query Form</h1>
                                 {% endfor %}
                             </div>
                         </div>
+                        <div class="col-md-2">
+                            <span class="help-block">Date Range</span>
+                            <div class="form-control" id='date_range'>
+                                <input type="text" name="date_range" style="cursor: pointer; padding: 5px 10px; width: 100%"/>
+                                <script>
+                                $(function() {
+                                    $('input[name="date_range"]').daterangepicker({
+                                        timePicker: true,
+                                        timePicker24Hour: true,
+                                        showDropdowns: true,
+                                        minDate: "2021/12/05",
+                                        startDate: "2021/12/05",
+                                        endDate: moment().startOf('hour'),
+                                        locale: {
+                                        format: 'YYYY/MM/DD hh:mmA'
+                                        }
+                                    });
+                                });
+                                </script>
+                            </div>
+                        </div>
                         <div class="col-md-2">
                             <span class="help-block">Sort Type</span>
                             <div class="form-control" id='sort_type'>
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index f1916dce1..00044b2d5 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -75,7 +75,6 @@
 from .data_containers import get_image_info
 from .data_containers import get_instrument_looks
 from .data_containers import get_rootnames_from_query
-from .data_containers import get_thumbnail_by_rootname
 from .data_containers import random_404_page
 from .data_containers import text_scrape
 from .data_containers import thumbnails_ajax
@@ -111,7 +110,8 @@ def jwql_query(request):
                 query_configs[instrument]['pupils'] = [query_unformat(i) for i in form.cleaned_data['{}_pupil'.format(instrument)]]
                 query_configs[instrument]['anomalies'] = [query_unformat(i) for i in form.cleaned_data['{}_anomalies'.format(instrument)]]
 
-            all_filters, all_apers, all_detectors, all_exptypes, all_readpatts, all_gratings, all_subarrays, all_pupils, all_anomalies = {}, {}, {}, {}, {}, {}, {}, {}, {}
+            all_filters, all_apers, all_detectors, all_exptypes = {}, {}, {}, {}
+            all_readpatts, all_gratings, all_subarrays, all_pupils, all_anomalies = {}, {}, {}, {}, {}
             for instrument in query_configs:
                 all_filters[instrument] = query_configs[instrument]['filters']
                 all_apers[instrument] = query_configs[instrument]['apertures']
@@ -126,6 +126,7 @@ def jwql_query(request):
             parameters = QUERY_CONFIG_TEMPLATE.copy()
             parameters[QUERY_CONFIG_KEYS.INSTRUMENTS] = form.cleaned_data['instrument']
             parameters[QUERY_CONFIG_KEYS.LOOK_STATUS] = form.cleaned_data['look_status']
+            parameters[QUERY_CONFIG_KEYS.DATE_RANGE] = form.cleaned_data['date_range']
             parameters[QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY] = form.cleaned_data['proposal_category']
             parameters[QUERY_CONFIG_KEYS.SORT_TYPE] = form.cleaned_data['sort_type']
             parameters[QUERY_CONFIG_KEYS.ANOMALIES] = all_anomalies
@@ -994,7 +995,8 @@ def explore_image(request, inst, file_root, filetype):
     return render(request, template, context)
 
 
-def explore_image_ajax(request, inst, file_root, filetype, line_plots='false', low_lim=None, high_lim=None, ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None):
+def explore_image_ajax(request, inst, file_root, filetype, line_plots='false', low_lim=None, high_lim=None,
+                       ext_name="SCI", int1_nr=None, grp1_nr=None, int2_nr=None, grp2_nr=None):
     """Generate the page listing all archived images in the database
     for a certain proposal
 

From 3ee4483060147a1ca459abdf76483497369bbfcf Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Tue, 7 Nov 2023 17:00:50 -0500
Subject: [PATCH 232/256] fix some linting errors

---
 jwql/utils/constants.py                   | 40 ++++++++++++-----------
 jwql/website/apps/jwql/data_containers.py | 39 ++++++++++++----------
 jwql/website/apps/jwql/views.py           | 36 ++++++++++----------
 3 files changed, 60 insertions(+), 55 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 92f189b5d..5c998667e 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -235,9 +235,11 @@
 FILTERS_PER_INSTRUMENT = {'fgs': [],
                           'miri': ['F560W', 'F770W', 'F1000W', 'F1065C', 'F1130W', 'F1140C', 'F1280W', 'F1500W',
                                    'F1550C', 'F1800W', 'F2100W', 'F2300C', 'F2550W', 'F2550WR', 'FLENS', 'FND', 'OPAQUE', 'P750L'],
-                          'nircam': ['F070W', 'F090W', 'F115W', 'F140M', 'F150W', 'F150W2', 'F182M', 'F187N', 'F200W', 'F210M', 'F212N',
-                                     'WLP4', 'F277W', 'F356W', 'F444W', 'F300M', 'F335M', 'F360M', 'F410M', 'F430M', 'F460M', 'F480M', 'F250M', 'F322W2'],
-                          'niriss': ['F090W', 'F115W', 'F140M', 'F150W', 'F200W', 'F277W', 'F356W', 'F380M', 'F430M', 'F444W', 'F480M', 'GR150C', 'GR150R'],
+                          'nircam': ['F070W', 'F090W', 'F115W', 'F140M', 'F150W', 'F150W2', 'F182M', 'F187N', 'F200W', 'F210M',
+                                     'F212N', 'WLP4', 'F277W', 'F356W', 'F444W', 'F300M', 'F335M', 'F360M', 'F410M', 'F430M',
+                                     'F460M', 'F480M', 'F250M', 'F322W2'],
+                          'niriss': ['F090W', 'F115W', 'F140M', 'F150W', 'F200W', 'F277W', 'F356W', 'F380M', 'F430M', 'F444W',
+                                     'F480M', 'GR150C', 'GR150R'],
                           'nirspec': ['CLEAR', 'F070LP', 'F100LP', 'F110W', 'F140X', 'F170LP', 'F290LP', 'OPAQUE', 'P750L']}
 
 FOUR_AMP_SUBARRAYS = ['WFSS128R', 'WFSS64R']
@@ -435,7 +437,7 @@
 # Keep keys defined via class as they are used many places with potential mispellings
 # Keys are in sort order from general to instrument specific, then alphabetical
 # within instrument specific fields.
-class QUERY_CONFIG_KEYS:
+class QueryConfigKeys:
     INSTRUMENTS = "INSTRUMENTS"
     PROPOSAL_CATEGORY = "PROPOSAL_CATEGORY"
     LOOK_STATUS = "LOOK_STATUS"
@@ -455,21 +457,21 @@ class QUERY_CONFIG_KEYS:
 
 # Template for parameters to be stored in "query_config" session for query_page
 QUERY_CONFIG_TEMPLATE = {
-    QUERY_CONFIG_KEYS.INSTRUMENTS: [],
-    QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY: [],
-    QUERY_CONFIG_KEYS.LOOK_STATUS: [],
-    QUERY_CONFIG_KEYS.NUM_PER_PAGE: 100,
-    QUERY_CONFIG_KEYS.SORT_TYPE: 'Recent',
-    QUERY_CONFIG_KEYS.DATE_RANGE: "",
-    QUERY_CONFIG_KEYS.ANOMALIES: {},
-    QUERY_CONFIG_KEYS.APERTURES: {},
-    QUERY_CONFIG_KEYS.DETECTORS: {},
-    QUERY_CONFIG_KEYS.EXP_TYPES: {},
-    QUERY_CONFIG_KEYS.FILTERS: {},
-    QUERY_CONFIG_KEYS.GRATINGS: {},
-    QUERY_CONFIG_KEYS.PUPILS: {},
-    QUERY_CONFIG_KEYS.READ_PATTS: {},
-    QUERY_CONFIG_KEYS.SUBARRAYS: {}
+    QueryConfigKeys.INSTRUMENTS: [],
+    QueryConfigKeys.PROPOSAL_CATEGORY: [],
+    QueryConfigKeys.LOOK_STATUS: [],
+    QueryConfigKeys.NUM_PER_PAGE: 100,
+    QueryConfigKeys.SORT_TYPE: 'Recent',
+    QueryConfigKeys.DATE_RANGE: "",
+    QueryConfigKeys.ANOMALIES: {},
+    QueryConfigKeys.APERTURES: {},
+    QueryConfigKeys.DETECTORS: {},
+    QueryConfigKeys.EXP_TYPES: {},
+    QueryConfigKeys.FILTERS: {},
+    QueryConfigKeys.GRATINGS: {},
+    QueryConfigKeys.PUPILS: {},
+    QueryConfigKeys.READ_PATTS: {},
+    QueryConfigKeys.SUBARRAYS: {}
 }
 
 # RAPID-style readout patterns for each instrument. Added so we can
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index fb64f68da..8dd38dc25 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -58,7 +58,7 @@
 from jwql.utils.constants import EXPOSURE_PAGE_SUFFIX_ORDER, IGNORED_SUFFIXES, INSTRUMENT_SERVICE_MATCH
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES
 from jwql.utils.constants import REPORT_KEYS_PER_INSTRUMENT
-from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QUERY_CONFIG_KEYS
+from jwql.utils.constants import SUFFIXES_TO_ADD_ASSOCIATION, SUFFIXES_WITH_AVERAGED_INTS, QueryConfigKeys
 from jwql.utils.credentials import get_mast_token
 from jwql.utils.permissions import set_permissions
 from jwql.utils.utils import get_rootnames_for_instrument_proposal
@@ -617,7 +617,8 @@ def get_edb_components(request):
                     mnemonic_query_result = get_mnemonic(mnemonic_identifier, start_time, end_time)
 
                     if len(mnemonic_query_result.data) == 0:
-                        mnemonic_query_status = "QUERY RESULT RETURNED NO DATA FOR {} ON DATES {} - {}".format(mnemonic_identifier, start_time, end_time)
+                        mnemonic_query_status = "QUERY RESULT RETURNED NO DATA FOR {} ON DATES {} - {}".format(mnemonic_identifier,
+                                                                                                               start_time, end_time)
                     else:
                         mnemonic_query_status = 'SUCCESS'
 
@@ -1249,7 +1250,8 @@ def get_instrument_proposals(instrument):
         List of proposals for the given instrument
     """
     tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx")
-    tap_results = tap_service.search(f"select distinct prpID from CaomObservation where collection='JWST' and maxLevel>0 and insName like '{instrument.lower()}%'")
+    tap_results = tap_service.search(f"""select distinct prpID from CaomObservation where collection='JWST'
+                                     and maxLevel>0 and insName like '{instrument.lower()}%'""")
     prop_table = tap_results.to_table()
     proposals = prop_table['prpID'].data
     inst_proposals = sorted(proposals.compressed(), reverse=True)
@@ -1479,7 +1481,8 @@ def get_rootnames_for_proposal(proposal):
         List of rootnames for the given instrument and proposal number
     """
     tap_service = vo.dal.TAPService("https://vao.stsci.edu/caomtap/tapservice.aspx")
-    tap_results = tap_service.search(f"select observationID from dbo.CaomObservation where collection='JWST' and maxLevel=2 and prpID='{int(proposal)}'")
+    tap_results = tap_service.search(f"""select observationID from dbo.CaomObservation where
+                                     collection='JWST' and maxLevel=2 and prpID='{int(proposal)}'""")
     prop_table = tap_results.to_table()
     rootnames = prop_table['observationID'].data
     return rootnames.compressed()
@@ -1504,7 +1507,7 @@ def get_rootnames_from_query(parameters):
     DATE_FORMAT = "%Y/%m/%d %I:%M%p" #noqa n806
 
     # Parse DATE_RANGE string into correct format
-    date_range = parameters[QUERY_CONFIG_KEYS.DATE_RANGE]
+    date_range = parameters[QueryConfigKeys.DATE_RANGE]
     start_date_range, stop_date_range = date_range.split(" - ")
     # Parse the strings into datetime objects
     start_datetime = datetime.strptime(start_date_range, DATE_FORMAT)
@@ -1514,14 +1517,14 @@ def get_rootnames_from_query(parameters):
     stop_time = Time(stop_datetime.isoformat(), format="isot")
 
     # Each Query Selection is Instrument specific
-    for inst in parameters[QUERY_CONFIG_KEYS.INSTRUMENTS]:
+    for inst in parameters[QueryConfigKeys.INSTRUMENTS]:
         # Make sure instruments are of the proper format for the archive query
         inst = inst.lower()
         current_ins_rootfileinfos = RootFileInfo.objects.filter(instrument=JWST_INSTRUMENT_NAMES_MIXEDCASE[inst])
 
         # General fields
-        sort_type = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
-        look_status = parameters[QUERY_CONFIG_KEYS.LOOK_STATUS]
+        sort_type = parameters[QueryConfigKeys.SORT_TYPE]
+        look_status = parameters[QueryConfigKeys.LOOK_STATUS]
 
         # Get a queryset of all observations STARTING within our date range
         current_ins_rootfileinfos = current_ins_rootfileinfos.filter(
@@ -1532,20 +1535,20 @@ def get_rootnames_from_query(parameters):
         if len(look_status) == 1:
             viewed = (look_status[0] == 'VIEWED')
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(viewed=viewed)
-        proposal_category = parameters[QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY]
+        proposal_category = parameters[QueryConfigKeys.PROPOSAL_CATEGORY]
         if len(proposal_category) > 0:
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(obsnum__proposal__category__in=proposal_category)
 
         # Instrument fields
-        inst_anomalies = parameters[QUERY_CONFIG_KEYS.ANOMALIES][inst]
-        inst_aperture = parameters[QUERY_CONFIG_KEYS.APERTURES][inst]
-        inst_detector = parameters[QUERY_CONFIG_KEYS.DETECTORS][inst]
-        inst_exp_type = parameters[QUERY_CONFIG_KEYS.EXP_TYPES][inst]
-        inst_filter = parameters[QUERY_CONFIG_KEYS.FILTERS][inst]
-        inst_grating = parameters[QUERY_CONFIG_KEYS.GRATINGS][inst]
-        inst_pupil = parameters[QUERY_CONFIG_KEYS.PUPILS][inst]
-        inst_read_patt = parameters[QUERY_CONFIG_KEYS.READ_PATTS][inst]
-        inst_subarray = parameters[QUERY_CONFIG_KEYS.SUBARRAYS][inst]
+        inst_anomalies = parameters[QueryConfigKeys.ANOMALIES][inst]
+        inst_aperture = parameters[QueryConfigKeys.APERTURES][inst]
+        inst_detector = parameters[QueryConfigKeys.DETECTORS][inst]
+        inst_exp_type = parameters[QueryConfigKeys.EXP_TYPES][inst]
+        inst_filter = parameters[QueryConfigKeys.FILTERS][inst]
+        inst_grating = parameters[QueryConfigKeys.GRATINGS][inst]
+        inst_pupil = parameters[QueryConfigKeys.PUPILS][inst]
+        inst_read_patt = parameters[QueryConfigKeys.READ_PATTS][inst]
+        inst_subarray = parameters[QueryConfigKeys.SUBARRAYS][inst]
 
         if (inst_aperture != []):
             current_ins_rootfileinfos = current_ins_rootfileinfos.filter(aperture__in=inst_aperture)
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 00044b2d5..393f386dd 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -61,7 +61,7 @@
 from jwql.database.database_interface import load_connection
 from jwql.utils import monitor_utils
 from jwql.utils.interactive_preview_image import InteractivePreviewImg
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, URL_DICT, QUERY_CONFIG_TEMPLATE, QUERY_CONFIG_KEYS
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, URL_DICT, QUERY_CONFIG_TEMPLATE, QueryConfigKeys
 from jwql.utils.utils import filename_parser, get_base_url, get_config, get_rootnames_for_instrument_proposal, query_unformat
 
 from .data_containers import build_table
@@ -124,20 +124,20 @@ def jwql_query(request):
                 all_anomalies[instrument] = query_configs[instrument]['anomalies']
 
             parameters = QUERY_CONFIG_TEMPLATE.copy()
-            parameters[QUERY_CONFIG_KEYS.INSTRUMENTS] = form.cleaned_data['instrument']
-            parameters[QUERY_CONFIG_KEYS.LOOK_STATUS] = form.cleaned_data['look_status']
-            parameters[QUERY_CONFIG_KEYS.DATE_RANGE] = form.cleaned_data['date_range']
-            parameters[QUERY_CONFIG_KEYS.PROPOSAL_CATEGORY] = form.cleaned_data['proposal_category']
-            parameters[QUERY_CONFIG_KEYS.SORT_TYPE] = form.cleaned_data['sort_type']
-            parameters[QUERY_CONFIG_KEYS.ANOMALIES] = all_anomalies
-            parameters[QUERY_CONFIG_KEYS.APERTURES] = all_apers
-            parameters[QUERY_CONFIG_KEYS.FILTERS] = all_filters
-            parameters[QUERY_CONFIG_KEYS.DETECTORS] = all_detectors
-            parameters[QUERY_CONFIG_KEYS.EXP_TYPES] = all_exptypes
-            parameters[QUERY_CONFIG_KEYS.READ_PATTS] = all_readpatts
-            parameters[QUERY_CONFIG_KEYS.GRATINGS] = all_gratings
-            parameters[QUERY_CONFIG_KEYS.SUBARRAYS] = all_subarrays
-            parameters[QUERY_CONFIG_KEYS.PUPILS] = all_pupils
+            parameters[QueryConfigKeys.INSTRUMENTS] = form.cleaned_data['instrument']
+            parameters[QueryConfigKeys.LOOK_STATUS] = form.cleaned_data['look_status']
+            parameters[QueryConfigKeys.DATE_RANGE] = form.cleaned_data['date_range']
+            parameters[QueryConfigKeys.PROPOSAL_CATEGORY] = form.cleaned_data['proposal_category']
+            parameters[QueryConfigKeys.SORT_TYPE] = form.cleaned_data['sort_type']
+            parameters[QueryConfigKeys.ANOMALIES] = all_anomalies
+            parameters[QueryConfigKeys.APERTURES] = all_apers
+            parameters[QueryConfigKeys.FILTERS] = all_filters
+            parameters[QueryConfigKeys.DETECTORS] = all_detectors
+            parameters[QueryConfigKeys.EXP_TYPES] = all_exptypes
+            parameters[QueryConfigKeys.READ_PATTS] = all_readpatts
+            parameters[QueryConfigKeys.GRATINGS] = all_gratings
+            parameters[QueryConfigKeys.SUBARRAYS] = all_subarrays
+            parameters[QueryConfigKeys.PUPILS] = all_pupils
 
             # save the query config settings to a session
             request.session['query_config'] = parameters
@@ -482,12 +482,12 @@ def archive_thumbnails_query_ajax(request):
     filtered_rootnames = get_rootnames_from_query(parameters)
 
     paginator = Paginator(filtered_rootnames,
-                          parameters[QUERY_CONFIG_KEYS.NUM_PER_PAGE])
+                          parameters[QueryConfigKeys.NUM_PER_PAGE])
     page_number = request.GET.get("page", 1)
     page_obj = paginator.get_page(page_number)
 
     data = thumbnails_query_ajax(page_obj.object_list)
-    data['thumbnail_sort'] = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
+    data['thumbnail_sort'] = parameters[QueryConfigKeys.SORT_TYPE]
     data['thumbnail_group'] = request.session.get("image_group", "Exposure")
 
     # add top level parameters for summarizing
@@ -511,7 +511,7 @@ def archive_thumbnails_query_ajax(request):
     data['total_pages'] = paginator.num_pages
     data['total_files'] = paginator.count
 
-    request.session['image_sort'] = parameters[QUERY_CONFIG_KEYS.SORT_TYPE]
+    request.session['image_sort'] = parameters[QueryConfigKeys.SORT_TYPE]
     save_page_navigation_data(request, data)
     return JsonResponse(data, json_dumps_params={'indent': 2})
 

From 5be0a6c72eae7a72dff2c8e2eddd1eedaa0215d4 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 8 Nov 2023 10:45:23 -0500
Subject: [PATCH 233/256] remove old date_range page

---
 jwql/website/apps/jwql/data_containers.py     | 126 ------------------
 jwql/website/apps/jwql/static/js/jwql.js      |  96 ++-----------
 .../jwql/templates/archive_date_range.html    |  59 --------
 jwql/website/apps/jwql/templates/base.html    |   1 -
 .../apps/jwql/templates/instrument.html       |   5 +-
 jwql/website/apps/jwql/urls.py                |  14 +-
 jwql/website/apps/jwql/views.py               | 110 +--------------
 7 files changed, 24 insertions(+), 387 deletions(-)
 delete mode 100644 jwql/website/apps/jwql/templates/archive_date_range.html

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 8dd38dc25..23862e07d 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -2002,132 +2002,6 @@ def thumbnails_ajax(inst, proposal, obs_num=None):
 
     return data_dict
 
-
-def thumbnails_date_range_ajax(inst, observations, inclusive_start_time_mjd, exclusive_stop_time_mjd):
-    """Generate a page that provides data necessary to render thumbnails for
-    ``archive_date_range`` template.
-
-    Parameters
-    ----------
-    inst : str
-        Name of JWST instrument
-    observations: list
-        observation models to use to get filenames
-    inclusive_start_time_mjd : float
-        Start time in mjd format for date range
-    exclusive_stop_time_mjd : float
-        Stop time in mjd format for date range
-
-    Returns
-    -------
-    data_dict : dict
-        Dictionary of data needed for the ``thumbnails`` template
-    """
-
-    data_dict = {'inst': inst,
-                 'file_data': dict()}
-    exp_types = set()
-    exp_groups = set()
-
-    # Get the available files for the instrument
-    for observation in observations:
-        obs_num = observation.obsnum
-        proposal = observation.proposal.prop_id
-        filenames, columns = get_filenames_by_instrument(inst, proposal, observation_id=obs_num, other_columns=['expstart', 'exp_type'])
-        # Get set of unique rootnames
-        rootnames = set(['_'.join(f.split('/')[-1].split('_')[:-1]) for f in filenames])
-        # Gather data for each rootname, and construct a list of all observations in the proposal
-        for rootname in rootnames:
-            # Parse filename
-            try:
-                filename_dict = filename_parser(rootname)
-
-                # Weed out file types that are not supported by generate_preview_images
-                if 'stage_3' in filename_dict['filename_type']:
-                    continue
-
-            except ValueError:
-                # Temporary workaround for noncompliant files in filesystem
-                filename_dict = {'activity': rootname[17:19],
-                                 'detector': rootname[26:],
-                                 'exposure_id': rootname[20:25],
-                                 'observation': rootname[7:10],
-                                 'parallel_seq_id': rootname[16],
-                                 'program_id': rootname[2:7],
-                                 'visit': rootname[10:13],
-                                 'visit_group': rootname[14:16],
-                                 'group_root': rootname[:26]}
-
-            # Get list of available filenames and exposure start times. All files with a given
-            # rootname will have the same exposure start time, so just keep the first.
-            available_files = []
-            exp_start = None
-            exp_type = None
-            for i, item in enumerate(filenames):
-                if rootname in item:
-                    available_files.append(item)
-                    if exp_start is None:
-                        exp_start = columns['expstart'][i]
-                        exp_type = columns['exp_type'][i]
-
-            if exp_start >= inclusive_start_time_mjd and exp_start < exclusive_stop_time_mjd:
-                exp_types.add(exp_type)
-                # Viewed is stored by rootname in the Model db.  Save it with the data_dict
-                # THUMBNAIL_FILTER_LOOK is boolean accessed according to a viewed flag
-                try:
-                    root_file_info = RootFileInfo.objects.get(root_name=rootname)
-                    viewed = THUMBNAIL_FILTER_LOOK[root_file_info.viewed]
-                except RootFileInfo.DoesNotExist:
-                    viewed = THUMBNAIL_FILTER_LOOK[0]
-
-                # Add to list of all exposure groups
-                exp_groups.add(filename_dict['group_root'])
-
-                # Add data to dictionary
-                data_dict['file_data'][rootname] = {}
-                data_dict['file_data'][rootname]['filename_dict'] = filename_dict
-                data_dict['file_data'][rootname]['available_files'] = available_files
-                data_dict['file_data'][rootname]["viewed"] = viewed
-                data_dict['file_data'][rootname]["exp_type"] = exp_type
-                data_dict['file_data'][rootname]['thumbnail'] = get_thumbnail_by_rootname(rootname)
-
-                try:
-                    data_dict['file_data'][rootname]['expstart'] = exp_start
-                    data_dict['file_data'][rootname]['expstart_iso'] = Time(exp_start, format='mjd').iso.split('.')[0]
-                except (ValueError, TypeError) as e:
-                    logging.warning("Unable to populate exp_start info for {}".format(rootname))
-                    logging.warning(e)
-                except KeyError:
-                    print("KeyError with get_expstart for {}".format(rootname))
-
-    # Extract information for sorting with dropdown menus
-    # (Don't include the proposal as a sorting parameter if the proposal has already been specified)
-    detectors, proposals = [], []
-    for rootname in list(data_dict['file_data'].keys()):
-        proposals.append(data_dict['file_data'][rootname]['filename_dict']['program_id'])
-        try:  # Some rootnames cannot parse out detectors
-            detectors.append(data_dict['file_data'][rootname]['filename_dict']['detector'])
-        except KeyError:
-            pass
-
-    dropdown_menus = {'detector': sorted(detectors),
-                      'proposal': sorted(proposals),
-                      'look': THUMBNAIL_FILTER_LOOK,
-                      'exp_type': sorted(set(exp_types))}
-
-    data_dict['tools'] = MONITORS
-    data_dict['dropdown_menus'] = dropdown_menus
-
-    # Order dictionary by descending expstart time.
-    sorted_file_data = OrderedDict(sorted(data_dict['file_data'].items(),
-                                   key=lambda x: getitem(x[1], 'expstart'), reverse=True))
-
-    data_dict['file_data'] = sorted_file_data
-    data_dict['exp_groups'] = sorted(exp_groups)
-
-    return data_dict
-
-
 def thumbnails_query_ajax(rootnames):
     """Generate a page that provides data necessary to render the
     ``thumbnails`` template.
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 4bca2470e..54ff39519 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -312,8 +312,8 @@ function determine_page_title_obs(instrument, proposal, observation) {
 
 /**
  * adds/removes disabled_section class and clears value
- * @param {string} element_id 
- * @param {boolean} set_disable 
+ * @param {string} element_id
+ * @param {boolean} set_disable
  */
  function set_disabled_section (element_id, set_disable) {
 
@@ -331,7 +331,7 @@ function determine_page_title_obs(instrument, proposal, observation) {
  *                              values are the number of groups for that suffix
  */
 function explore_image_update_enable_options(integrations, groups) {
-    
+
     // Check nr of integrations and groups of currently selected extension
     var ext_name = get_radio_button_value("extension");
 
@@ -343,7 +343,7 @@ function explore_image_update_enable_options(integrations, groups) {
     groups = groups.replace(/&#39;/g, '"');
     groups = groups.replace(/'/g, '"');
     groups = JSON.parse(groups)[ext_name];
-    
+
     // Zero base our calculations
     integrations -= 1
     groups -=1
@@ -353,15 +353,15 @@ function explore_image_update_enable_options(integrations, groups) {
     document.getElementById("integration2").max = integrations;
     document.getElementById("group1").max = groups;
     document.getElementById("group2").max = groups;
-    
-    
+
+
     // If multiple integrations or groups.  Allow difference calculations
     //          enable calculate_difference box
     //          enable subtrahend boxes
     if (integrations > 0 || groups > 0) {
         set_disabled_section("calcDifferenceForm", false);
         calc_difference = document.getElementById("calcDifference").checked;
-        
+
     } else {
         document.getElementById("calcDifference").checked.value = false;
         set_disabled_section("calcDifferenceForm", true);
@@ -384,7 +384,7 @@ function explore_image_update_enable_options(integrations, groups) {
     set_disabled_section("groupInput1", (groups < 1));
     set_disabled_section("integrationInput2", (!calc_difference || integrations < 1));
     set_disabled_section("groupInput2", (!calc_difference || groups < 1));
-    
+
 }
 
 
@@ -811,7 +811,7 @@ function sort_by_thumbnails(sort_type, base_url) {
     // Update dropdown menu text
     document.getElementById('sort_dropdownMenuButton').innerHTML = sort_type;
 
-    // Sort the thumbnails accordingly.  
+    // Sort the thumbnails accordingly.
     // Note: Because thumbnails will sort relating to their current order (when the exp_start is the same between thumbnails), we need to do multiple sorts to guarantee consistency.
 
     var thumbs = $('div#thumbnail-array>div')
@@ -838,9 +838,9 @@ function sort_by_thumbnails(sort_type, base_url) {
 
 
 /**
- * Toggle a viewed button when pressed.  
+ * Toggle a viewed button when pressed.
  * Ajax call to update RootFileInfo model with toggled value
- * 
+ *
  * @param {String} file_root - The rootname of the file corresponding to the thumbnail
  * @param {String} base_url - The base URL for gathering data from the AJAX view.
  */
@@ -850,7 +850,7 @@ function toggle_viewed(file_root, base_url) {
     var elem = document.getElementById("viewed");
     update_viewed_button(elem.value == "New" ? true : false);
     elem.disabled=true;
-    
+
     // Ajax Call to update RootFileInfo model with "viewed" info
     $.ajax({
         url: base_url + '/ajax/viewed/' + file_root,
@@ -1309,7 +1309,7 @@ function update_thumbnail_array(data) {
     var thumbnail_content = "";
     var image_updates = [];
     for (var i = 0; i < Object.keys(data.file_data).length; i++) {
-        
+
         // Parse out useful variables
         var rootname = Object.keys(data.file_data)[i];
         var file = data.file_data[rootname];
@@ -1361,76 +1361,6 @@ function update_thumbnail_array(data) {
     insert_thumbnail_images(image_updates);
 }
 
-/**
- * Read and submit the form for archive date ranges.
- * @param {String} inst - The instrument of interest (e.g. "FGS")
- * @param {String} base_url - The base URL for gathering data from the AJAX view.
- */
-function submit_date_range_form(inst, base_url, group) {
-
-    var start_date = document.getElementById("start_date_range").value;
-    var stop_date = document.getElementById("stop_date_range").value;
-
-    if (!start_date) {
-        alert("You must enter a Start Date/Time");
-    }  else if (!stop_date) {
-        alert("You must enter a Stop Date/Time");
-    } else if (start_date >= stop_date) {
-        alert("Start Time must be earlier than Stop Time");
-    } else {
-        document.getElementById("loading").style.display = "block";
-        document.getElementById("thumbnail-array").style.display = "none";
-        document.getElementById("no_thumbnails_msg").style.display = "none";
-        $.ajax({
-            url: base_url + '/ajax/' + inst + '/archive_date_range/start_date_' + start_date + '/stop_date_' + stop_date,
-            success: function(data){
-                var show_thumbs = true;
-                var num_thumbnails = Object.keys(data.file_data).length;
-                // verify we want to continue with results
-                if (num_thumbnails > 1000) {
-                    show_thumbs = false;
-                    alert("Returning " + num_thumbnails + " images reduce your date/time range for page to load correctly");
-                }
-                if (show_thumbs) {
-                    // Handle DIV updates
-                    // Clear our existing array
-                    $("#thumbnail-array")[0].innerHTML = "";
-                    if (num_thumbnails > 0) {
-                        update_show_count(num_thumbnails, 'activities');
-                        update_thumbnail_array(data);
-                        update_filter_options(data, base_url, 'thumbnail');
-                        update_group_options(data, base_url);
-                        update_sort_options(data, base_url);
-
-                        // Do initial sort and group to match sort button display
-                        group_by_thumbnails(group, base_url);
-                        sort_by_thumbnails(data.thumbnail_sort, base_url);
-
-                        // Replace loading screen with the proposal array div
-                        document.getElementById("loading").style.display = "none";
-                        document.getElementById("thumbnail-array").style.display = "block";
-                        document.getElementById("no_thumbnails_msg").style.display = "none";
-                    } else {
-                        show_thumbs = false;
-                    }
-                } 
-                if (!show_thumbs) {
-                    document.getElementById("loading").style.display = "none";
-                    document.getElementById("no_thumbnails_msg").style.display = "inline-block";
-                }
-
-            },
-            error : function(response) {
-                document.getElementById("loading").style.display = "none";
-                document.getElementById("thumbnail-array").style.display = "none";
-                document.getElementById("no_thumbnails_msg").style.display = "inline-block";
-    
-            }
-        });
-    }
-}
-
-
 /**
  * Updates various components on the thumbnails page
  * @param {String} inst - The instrument of interest (e.g. "FGS")
diff --git a/jwql/website/apps/jwql/templates/archive_date_range.html b/jwql/website/apps/jwql/templates/archive_date_range.html
deleted file mode 100644
index 15b7ec4af..000000000
--- a/jwql/website/apps/jwql/templates/archive_date_range.html
+++ /dev/null
@@ -1,59 +0,0 @@
-{% extends "base.html" %}
-
-{% block preamble %}
-
-	<title>Archive Date Range {{ inst }} Images - JWQL</title>
-
-{% endblock %}
-
-{% block content %}
-
-<main role="main" class="container">
-
-    <h3 id='title'>Archive Date Range {{ inst }} Images</h3>
-
-    <div id='img_show_count'></div>
-    <hr>
-
-    <!-- Filter and sort options -->
-    <div class="d-flex flex-row">
-        <div class="mr-4 mr-5" id="thumbnail-filter"></div>
-        <div class="mx-4" id="thumbnail-sort"></div>
-        <div class="mx-4 mr-5 ml-auto" id="group-by-exposure"></div>
-    </div>
-
-    <hr>
-    <div class="d-flex flex-row">
-        <div class="mr-auto">
-            Date Range:<br>
-            <input type="datetime-local" id="start_date_range" name="start_date_range">
-            <input type="datetime-local" id="stop_date_range" name="stop_date_range">
-            <button class="btn btn-primary" type="button" href='#' onclick="submit_date_range_form('{{inst}}', '{{base_url}}', '{{group}}');">Submit</button>
-        </div>
-    </div>
-    <hr>
-
-    <!-- Loading animation -->
-    <div id="loading" style='display: none'>
-        <div class="lds-css ng-scope">
-            <div style="width:100%;height:100%" class="lds-magnify">
-                <div>
-                    <div>
-                        <div>
-                        </div>
-                        <div>
-                        </div>
-                    </div>
-                </div>
-            </div>
-            Loading ...
-        </div>
-    </div>
-
-    <!-- Display the data -->
-    <div id='thumbnail-array'></div>
-    <br>
-    <a id="no_thumbnails_msg">No data match the date range.</a>
-</main>
-
-{% endblock %}
diff --git a/jwql/website/apps/jwql/templates/base.html b/jwql/website/apps/jwql/templates/base.html
index 1d00bfaea..54bb04ba3 100644
--- a/jwql/website/apps/jwql/templates/base.html
+++ b/jwql/website/apps/jwql/templates/base.html
@@ -121,7 +121,6 @@
 								<!-- <a class="dropdown-item disabled-dropdown" href='#'>Unlooked Images</a> -->
 								<!-- <a class="dropdown-item disabled-dropdown" href="#">Image Anomalies</a> -->
 								<a class="dropdown-item" href={{ url('jwql:archive', args=[inst_key]) }}>Archived Images</a>
-								<a class="dropdown-item" href={{ url('jwql:archive_date_range', args=[inst_key]) }}>Date Range Images</a>
 								<div class="dropdown-divider"></div>
 								<a class='dropdown-heading mx-3'>Monitors</a>
 									{% for monitor_name, monitor_url in inst_tools %}
diff --git a/jwql/website/apps/jwql/templates/instrument.html b/jwql/website/apps/jwql/templates/instrument.html
index 1f1b08f24..ed5ce89e9 100644
--- a/jwql/website/apps/jwql/templates/instrument.html
+++ b/jwql/website/apps/jwql/templates/instrument.html
@@ -45,7 +45,6 @@ <h2>Archive</h2>
 						  <!-- <a class="dropdown-item" href={{ url('jwql:unlooked', args=[inst]) }}>Unlooked Images (Currently Unavailable)</a> -->
 						  <!-- <a class="dropdown-item" href="#">Image Anomalies (Currently Unavailable)</a> -->
 						  <a class="dropdown-item" href={{ url('jwql:archive', args=[inst]) }}>Archived Images</a>
-						  <a class="dropdown-item" href={{ url('jwql:archive_date_range', args=[inst]) }}>Date Range Images</a>
 						</div>
 					</div>
 				</div>
@@ -76,7 +75,7 @@ <h2>Monitors</h2>
 					</div>
 				</div>
 			</div>
-			
+
 			<div class="container">
 				<div class="col-md-20 text-center">
 					<!-- Documentation -->
@@ -94,7 +93,7 @@ <h2>Documentation</h2>
 					</br>
 				</div>
 			</div>
-			
+
 		</div>
 
 	</main>
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index 0b4420080..6c054ed72 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -83,25 +83,27 @@
     path('query_submit/', views.query_submit, name='query_submit'),
     re_path(r'^(?P<inst>({}))/$'.format(instruments), views.instrument, name='instrument'),
     re_path(r'^(?P<inst>({}))/archive/$'.format(instruments), views.archived_proposals, name='archive'),
-    re_path(r'^(?P<inst>({}))/archive_date_range/$'.format(instruments), views.archive_date_range, name='archive_date_range'),
     re_path(r'^(?P<inst>({}))/unlooked/$'.format(instruments), views.unlooked_images, name='unlooked'),
     re_path(r'^(?P<inst>({}))/report/$'.format(instruments), views.download_report, name='download_report'),
     re_path(r'^(?P<inst>({}))/(?P<file_root>[\w-]+)/$'.format(instruments), views.view_image, name='view_image'),
     re_path(r'^(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/'.format(instruments), views.explore_image, name='explore_image'),
     re_path(r'^(?P<inst>({}))/(?P<filename>.+)_(?P<filetype>.+)/header/'.format(instruments), views.view_header, name='view_header'),
-    re_path(r'^(?P<inst>({}))/archive/(?P<proposal>[\d]{{1,5}})/obs(?P<observation>[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_per_observation, name='archive_thumb_per_obs'),
+    re_path(r'^(?P<inst>({}))/archive/(?P<proposal>[\d]{{1,5}})/obs(?P<observation>[\d]{{1,3}})/$'.format(instruments),
+            views.archive_thumbnails_per_observation, name='archive_thumb_per_obs'),
     re_path(r'^(?P<inst>({}))/exposure/(?P<group_root>[\w-]+)/$'.format(instruments), views.view_exposure, name='view_exposure'),
 
     # AJAX views
     re_path('ajax/query_submit/', views.archive_thumbnails_query_ajax, name='archive_thumb_query_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/archive/$'.format(instruments), views.archived_proposals_ajax, name='archive_ajax'),
-    re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'),
-    re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/plot_(?P<line_plots>(true|false))/ext_(?P<ext_name>.+)/int1_(?P<int1_nr>.+)/grp1_(?P<grp1_nr>.+)/int2_(?P<int2_nr>.+)/grp2_(?P<grp2_nr>.+)/$'.format(instruments), views.explore_image_ajax, name='explore_image_ajax'),
-    re_path(r'^ajax/(?P<inst>({}))/archive/(?P<proposal>[\d]{{1,5}})/obs(?P<observation>[\d]{{1,3}})/$'.format(instruments), views.archive_thumbnails_ajax, name='archive_thumb_ajax'),
+    re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/$'.format(instruments), views.explore_image_ajax,
+            name='explore_image_ajax'),
+    re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/plot_(?P<line_plots>(true|false))/ext_(?P<ext_name>.+)/int1_(?P<int1_nr>.+)/grp1_(?P<grp1_nr>.+)/int2_(?P<int2_nr>.+)/grp2_(?P<grp2_nr>.+)/$'.format(instruments),
+             views.explore_image_ajax, name='explore_image_ajax'),
+    re_path(r'^ajax/(?P<inst>({}))/archive/(?P<proposal>[\d]{{1,5}})/obs(?P<observation>[\d]{{1,3}})/$'.format(instruments),
+            views.archive_thumbnails_ajax, name='archive_thumb_ajax'),
     re_path(r'^ajax/viewed/(?P<file_root>.+)/$', views.toggle_viewed_ajax, name='toggle_viewed_ajax'),
     re_path(r'^ajax/viewed_group/(?P<group_root>.+)/(?P<status>(viewed|new|Viewed|New))/$',
             views.set_viewed_ajax, name='set_viewed_ajax'),
-    re_path(r'^ajax/(?P<inst>({}))/archive_date_range/start_date_(?P<start_date>.+)/stop_date_(?P<stop_date>.+)/$'.format(instruments), views.archive_date_range_ajax, name='archive_date_range_ajax'),
     re_path(r'^ajax/image_group/$', views.save_image_group_ajax, name='save_image_group_ajax'),
     re_path(r'^ajax/image_sort/$', views.save_image_sort_ajax, name='save_image_sort_ajax'),
     re_path(r'^ajax/navigate_filter/$', views.save_page_navigation_data_ajax, name='save_page_navigation_data_ajax'),
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 393f386dd..30e6a0a26 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -79,14 +79,11 @@
 from .data_containers import text_scrape
 from .data_containers import thumbnails_ajax
 from .data_containers import thumbnails_query_ajax
-from .data_containers import thumbnails_date_range_ajax
 from .forms import JwqlQueryForm
 from .forms import FileSearchForm
 if not os.environ.get("READTHEDOCS"):
-    from .models import Observation, RootFileInfo
+    from .models import RootFileInfo
 from astropy.io import fits
-from astropy.time import Time
-import astropy.units as u
 
 
 def jwql_query(request):
@@ -221,111 +218,6 @@ def save_page_navigation_data_ajax(request):
     context = {'item': request.session['navigation_data']}
     return JsonResponse(context, json_dumps_params={'indent': 2})
 
-
-def archive_date_range(request, inst):
-    """Generate the page for date range images
-
-    Parameters
-    ----------
-    request : HttpRequest object
-        Incoming request from the webpage
-    inst : str
-        Name of JWST instrument
-
-    Returns
-    -------
-    HttpResponse object
-        Outgoing response sent to the webpage
-    """
-
-    # Ensure the instrument is correctly capitalized
-    inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()]
-
-    template = 'archive_date_range.html'
-    sort_type = request.session.get('image_sort', 'Recent')
-    group_type = request.session.get('image_group', 'Exposure')
-    context = {'inst': inst,
-               'base_url': get_base_url(),
-               'sort': sort_type,
-               'group': group_type}
-
-    return render(request, template, context)
-
-
-def archive_date_range_ajax(request, inst, start_date, stop_date):
-    """Generate the page listing all archived images within the inclusive date range for all proposals
-
-    Parameters
-    ----------
-    request : HttpRequest object
-        Incoming request from the webpage
-    inst : str
-        Name of JWST instrument
-    start_date : str
-        Start date for date range
-    stop_date : str
-        stop date for date range
-
-    Returns
-    -------
-    JsonResponse object
-        Outgoing response sent to the webpage
-    """
-
-    # Ensure the instrument is correctly capitalized
-    inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()]
-
-    # Calculate start date/time in MJD format to begin our range
-    inclusive_start_time = Time(start_date)
-
-    # Add a minute to the stop time and mark it 'exclusive' for code clarity,
-    # doing this to get all seconds selected minute
-    exclusive_stop_time = Time(stop_date) + (1 * u.minute)
-
-    # Get a queryset of all observations STARTING WITHIN our date range
-    begin_after_start = Observation.objects.filter(
-        proposal__archive__instrument=inst,
-        obsstart__gte=inclusive_start_time.mjd)
-    all_entries_beginning_in_range = begin_after_start.filter(
-        obsstart__lt=exclusive_stop_time.mjd)
-
-    # Get a queryset of all observations ENDING WITHIN our date range
-    end_after_start = Observation.objects.filter(
-        proposal__archive__instrument=inst,
-        obsend__gte=inclusive_start_time.mjd)
-    all_entries_ending_in_range = end_after_start.filter(
-        obsend__lt=exclusive_stop_time.mjd)
-
-    # Get a queryset of all observations SPANNING
-    # (starting before and ending after) our date range.
-    # Bump our window out a few days to catch hypothetical
-    # observations that last over 24 hours.
-    # The larger the window the more time the query takes so keeping it tight.
-    two_days_before_start_time = Time(start_date) - (2 * u.day)
-    two_days_after_end_time = Time(stop_date) + (3 * u.day)
-    end_after_stop = Observation.objects.filter(
-        proposal__archive__instrument=inst,
-        obsend__gte=two_days_after_end_time.mjd)
-    all_entries_spanning_range = end_after_stop.filter(
-        obsstart__lt=two_days_before_start_time.mjd)
-
-    obs_beginning = [observation for observation in all_entries_beginning_in_range]
-    obs_ending = [observation for observation in all_entries_ending_in_range]
-    obs_spanning = [observation for observation in all_entries_spanning_range]
-
-    # Create a single list of all pertinent observations
-    all_observations = list(set(obs_beginning + obs_ending + obs_spanning))
-    # Get all thumbnails that occurred within the time frame for these observations
-    data = thumbnails_date_range_ajax(
-        inst, all_observations, inclusive_start_time.mjd, exclusive_stop_time.mjd)
-    data['thumbnail_sort'] = request.session.get("image_sort", "Recent")
-    data['thumbnail_group'] = request.session.get("image_group", "Exposure")
-
-    # Create Dictionary of Rootnames with expstart
-    save_page_navigation_data(request, data)
-    return JsonResponse(data, json_dumps_params={'indent': 2})
-
-
 def archived_proposals(request, inst):
     """Generate the page listing all archived proposals in the database
 

From 744d8017b8404b8abae98e60f3290c0921d2b872 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 8 Nov 2023 12:20:35 -0500
Subject: [PATCH 234/256] addressed review comments; changed output directory
 for display

---
 .../nircam_monitors/claw_monitor.py           | 75 ++++++++++---------
 jwql/website/apps/jwql/monitor_views.py       |  6 +-
 2 files changed, 42 insertions(+), 39 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 67a3bfc49..72244aad3 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -32,7 +32,6 @@
 from astropy.time import Time
 from astroquery.mast import Mast
 import matplotlib
-matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
@@ -44,6 +43,8 @@
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
+matplotlib.use('Agg')
+
 
 class ClawMonitor():
     """Class for executing the claw monitor.
@@ -62,7 +63,7 @@ class ClawMonitor():
     outfile : str
         The name of the output plot for a given claw stack combination.
 
-    output_dir : str
+    output_dir_claws : str
         Path into which claw stack plots will be placed.
 
     output_dir_bkg : str
@@ -74,7 +75,7 @@ class ClawMonitor():
     query_end : float
         MJD end date to use for querying MAST.
 
-    wv : str
+    channel : str
         NIRCam channel for a given claw stack, either ``SW`` or ``LW``.
 
     proposal : str
@@ -101,8 +102,8 @@ def __init__(self):
         """
 
         # Define and setup the output directories for the claw and background plots.
-        self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
-        ensure_dir_exists(self.output_dir)
+        self.output_dir_claws = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+        ensure_dir_exists(self.output_dir_claws)
         self.output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
         ensure_dir_exists(self.output_dir_bkg)
 
@@ -185,7 +186,7 @@ def process(self):
         """
 
         # Get detector order and plot settings, depending on the wavelength channel
-        if self.wv == 'SW':
+        if self.channel == 'SW':
             detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']  # in on-sky order, don't change order
             cols, rows = 5, 2
             grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1, 1, 1, 1, .1])
@@ -207,20 +208,20 @@ def process(self):
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files,
             # which should be plenty to see any claws.
-            files = [f for f in files if os.path.exists(f)][0:20]
+            files = [fname for fname in files if os.path.exists(fname)][0:20]
             stack = np.ma.ones((len(files), 2048, 2048))
-            for n, f in enumerate(files):
-                logging.info('Working on: {}'.format(f))
-                h = fits.open(f)
+            for n, fname in enumerate(files):
+                logging.info('Working on: {}'.format(fname))
+                hdu = fits.open(fname)
 
                 # Get plot label info from first image
                 if n == 0:
-                    obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
-                    pa_v3 = h[1].header['PA_V3']
+                    obs_start = '{}T{}'.format(hdu[0].header['DATE-OBS'], hdu[0].header['TIME-OBS'])
+                    pa_v3 = hdu[1].header['PA_V3']
 
                 # Make source segmap, add the masked data to the stack, and get background stats
-                data = h['SCI'].data
-                dq = h['DQ'].data
+                data = hdu['SCI'].data
+                dq = hdu['DQ'].data
                 threshold = detect_threshold(data, 1.0)
                 sigma = 3.0 * gaussian_fwhm_to_sigma  # FWHM = 3.
                 kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
@@ -234,18 +235,18 @@ def process(self):
 
                 # Add this file's stats to the claw database table. Can't insert values with numpy.float32
                 # datatypes into database so need to change the datatypes of these values.
-                claw_db_entry = {'filename': os.path.basename(f),
+                claw_db_entry = {'filename': os.path.basename(fname),
                                  'proposal': self.proposal,
                                  'obs': self.obs,
                                  'detector': det.upper(),
                                  'filter': self.fltr.upper(),
                                  'pupil': self.pupil.upper(),
-                                 'expstart': '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS']),
-                                 'expstart_mjd': h[0].header['EXPSTART'],
-                                 'effexptm': h[0].header['EFFEXPTM'],
-                                 'ra': h[1].header['RA_V1'],
-                                 'dec': h[1].header['DEC_V1'],
-                                 'pa_v3': h[1].header['PA_V3'],
+                                 'expstart': '{}T{}'.format(hdu[0].header['DATE-OBS'], hdu[0].header['TIME-OBS']),
+                                 'expstart_mjd': hdu[0].header['EXPSTART'],
+                                 'effexptm': hdu[0].header['EFFEXPTM'],
+                                 'ra': hdu[1].header['RA_V1'],
+                                 'dec': hdu[1].header['DEC_V1'],
+                                 'pa_v3': hdu[1].header['PA_V3'],
                                  'mean': float(mean),
                                  'median': float(med),
                                  'stddev': float(stddev),
@@ -255,7 +256,7 @@ def process(self):
                                  }
                 with engine.begin() as connection:
                     connection.execute(self.stats_table.__table__.insert(), claw_db_entry)
-                h.close()
+                hdu.close()
 
             # Make the normalized skyflat for this detector
             skyflat = np.ma.median(stack, axis=0)
@@ -264,7 +265,7 @@ def process(self):
             skyflat[~np.isfinite(skyflat)] = 1  # fill missing values
 
             # Add the skyflat for this detector to the claw stack plot
-            if (self.wv == 'SW') & (i > 3):  # skip colobar axis
+            if (self.channel == 'SW') & (i > 3):  # skip colobar axis
                 idx = i + 1
             else:
                 idx = i
@@ -274,7 +275,7 @@ def process(self):
                 ax.imshow(skyflat, cmap='coolwarm', vmin=999, vmax=999, origin='lower')
             elif (len(skyflat[skyflat != 1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
                 mean, med, stddev = sigma_clipped_stats(skyflat)
-                vmin, vmax =  med - 3 * stddev, med + 3 * stddev
+                vmin, vmax = med - 3 * stddev, med + 3 * stddev
                 found_scale = True
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
@@ -340,25 +341,25 @@ def run(self):
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
         self.query_start_mjd, self.query_end_mjd = 59715.28951771492, 59715.29771992559  # todo remove
-        t = self.query_mast()
-        logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
+        mast_table = self.query_mast()
+        logging.info('{} files found between {} and {}.'.format(len(mast_table), self.query_start_mjd, self.query_end_mjd))
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
-        combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
-        t['combos'] = combos
+        combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in mast_table])
+        mast_table['combos'] = combos
         monitor_run = False
         for combo in np.unique(combos):
-            tt = t[t['combos'] == combo]
-            if 'long' in tt['filename'][0]:
-                self.wv = 'LW'
+            mast_table_combo = mast_table[mast_table['combos'] == combo]
+            if 'long' in mast_table_combo['filename'][0]:
+                self.channel = 'LW'
             else:
-                self.wv = 'SW'
+                self.channel = 'SW'
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
-            self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
+            self.outfile = os.path.join(self.output_dir_claws, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment?
-            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in tt]) # todo remove
-            self.detectors = np.array(tt['detector'])
+            #self.files = np.array([filesystem_path(row['filename']) for row in mast_table_combo])  # todo uncomment?
+            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in mast_table_combo]) # todo remove
+            self.detectors = np.array(mast_table_combo['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
                 self.process()
@@ -367,7 +368,7 @@ def run(self):
                 logging.info('{} already exists'.format(self.outfile))
 
         # Update the background trending plots, if any new data exists
-        if len(t) > 0:
+        if len(mast_table) > 0:
             logging.info('Making background trending plots.')
             self.make_background_plots()
 
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index a724f420c..e750d461a 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -70,8 +70,9 @@ def background_monitor(request):
     template = "background_monitor.html"
 
     # Get the background trending filters to display
+    output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
     fltrs = ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']
-    bkg_plots = ['/static/outputs/claw_monitor/backgrounds/{}_backgrounds.png'.format(fltr) for fltr in fltrs]
+    bkg_plots = [os.path.join(output_dir_bkg, '{}_backgrounds.png'.format(fltr)) for fltr in fltrs]
 
     context = {
         'inst': 'NIRCam',
@@ -155,7 +156,8 @@ def claw_monitor(request):
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
     recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 1000]))  # todo change from 1000 to 10
-    claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
+    output_dir_claws = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+    claw_stacks = [os.path.join(output_dir_claws, filename) for filename in recent_files]
 
     context = {
         'inst': 'NIRCam',

From 9bf381b7e26544bb95ccf1d2c9874f7086887bf3 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 8 Nov 2023 12:24:55 -0500
Subject: [PATCH 235/256] pep8

---
 jwql/database/database_interface.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 47800f647..73b536eed 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -573,7 +573,7 @@ class : obj
             NIRSpecEDBDailyStats, NIRSpecEDBBlockStats, NIRSpecEDBTimeIntervalStats,
             NIRSpecEDBEveryChangeStats, FGSEDBDailyStats, FGSEDBBlockStats,
             FGSEDBTimeIntervalStats, FGSEDBEveryChangeStats],
-    'claw': [NIRCamClawQueryHistory, NIRCamClawStats],}
+    'claw': [NIRCamClawQueryHistory, NIRCamClawStats], }
 
 if __name__ == '__main__':
     base.metadata.create_all(engine)

From cac9cd2f4b35f7ca1fddf9d74c080028d2ca5bc9 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 8 Nov 2023 12:28:40 -0500
Subject: [PATCH 236/256] pesky pep8 fixes

---
 jwql/utils/constants.py                   | 1137 +++++++++++++++------
 jwql/website/apps/jwql/data_containers.py |   15 +-
 jwql/website/apps/jwql/forms.py           |    2 +-
 jwql/website/apps/jwql/urls.py            |    2 +-
 jwql/website/apps/jwql/views.py           |    1 +
 5 files changed, 809 insertions(+), 348 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 5c998667e..39db77e92 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -34,109 +34,159 @@
 # and the second for y coordinates. Within each tuple are value for
 # starting, ending, and step size. Step size is needed for MIRI, where
 # the pixels corresponding to the 4 amplifiers are interleaved.
-AMPLIFIER_BOUNDARIES = {'nircam': {'1': [(0, 512, 1), (0, 2048, 1)],
-                                   '2': [(512, 1024, 1), (0, 2048, 1)],
-                                   '3': [(1024, 1536, 1), (0, 2048, 1)],
-                                   '4': [(1536, 2048, 1), (0, 2048, 1)]},
-                        'niriss': {'1': [(0, 2048, 1), (0, 512, 1)],
-                                   '2': [(0, 2048, 1), (512, 1024, 1)],
-                                   '3': [(0, 2048, 1), (1024, 1536, 1)],
-                                   '4': [(0, 2048, 1), (1536, 2048, 1)]},
-                        'fgs': {'1': [(0, 512, 1), (0, 2048, 1)],
-                                '2': [(512, 1024, 1), (0, 2048, 1)],
-                                '3': [(1024, 1536, 1), (0, 2048, 1)],
-                                '4': [(1536, 2048, 1), (0, 2048, 1)]},
-                        'nirspec': {'1': [(0, 2048, 1), (0, 512, 1)],
-                                    '2': [(0, 2048, 1), (512, 1024, 1)],
-                                    '3': [(0, 2048, 1), (1024, 1536, 1)],
-                                    '4': [(0, 2048, 1), (1536, 2048, 1)]},
-                        'miri': {'1': [(0, 1032, 4), (0, 1024, 1)],
-                                 '2': [(1, 1032, 4), (0, 1024, 1)],
-                                 '3': [(2, 1032, 4), (0, 1024, 1)],
-                                 '4': [(3, 1032, 4), (0, 1024, 1)]}}
+AMPLIFIER_BOUNDARIES = {
+    "nircam": {
+        "1": [(0, 512, 1), (0, 2048, 1)],
+        "2": [(512, 1024, 1), (0, 2048, 1)],
+        "3": [(1024, 1536, 1), (0, 2048, 1)],
+        "4": [(1536, 2048, 1), (0, 2048, 1)],
+    },
+    "niriss": {
+        "1": [(0, 2048, 1), (0, 512, 1)],
+        "2": [(0, 2048, 1), (512, 1024, 1)],
+        "3": [(0, 2048, 1), (1024, 1536, 1)],
+        "4": [(0, 2048, 1), (1536, 2048, 1)],
+    },
+    "fgs": {
+        "1": [(0, 512, 1), (0, 2048, 1)],
+        "2": [(512, 1024, 1), (0, 2048, 1)],
+        "3": [(1024, 1536, 1), (0, 2048, 1)],
+        "4": [(1536, 2048, 1), (0, 2048, 1)],
+    },
+    "nirspec": {
+        "1": [(0, 2048, 1), (0, 512, 1)],
+        "2": [(0, 2048, 1), (512, 1024, 1)],
+        "3": [(0, 2048, 1), (1024, 1536, 1)],
+        "4": [(0, 2048, 1), (1536, 2048, 1)],
+    },
+    "miri": {
+        "1": [(0, 1032, 4), (0, 1024, 1)],
+        "2": [(1, 1032, 4), (0, 1024, 1)],
+        "3": [(2, 1032, 4), (0, 1024, 1)],
+        "4": [(3, 1032, 4), (0, 1024, 1)],
+    },
+}
 
 # Dictionary describing instruments to which anomalies apply
 ANOMALIES_PER_INSTRUMENT = {
     # anomalies affecting all instruments:
-    'cosmic_ray_shower': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
-    'diffraction_spike': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
-    'excessive_saturation': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
-    'guidestar_failure': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
-    'persistence': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
+    "cosmic_ray_shower": ["fgs", "miri", "nircam", "niriss", "nirspec"],
+    "diffraction_spike": ["fgs", "miri", "nircam", "niriss", "nirspec"],
+    "excessive_saturation": ["fgs", "miri", "nircam", "niriss", "nirspec"],
+    "guidestar_failure": ["fgs", "miri", "nircam", "niriss", "nirspec"],
+    "persistence": ["fgs", "miri", "nircam", "niriss", "nirspec"],
     # anomalies affecting multiple instruments:
-    'crosstalk': ['fgs', 'nircam', 'niriss', 'nirspec'],
-    'data_transfer_error': ['fgs', 'nircam', 'niriss', 'nirspec'],
-    'ghost': ['fgs', 'nircam', 'niriss', 'nirspec'],
-    'snowball': ['fgs', 'nircam', 'niriss', 'nirspec'],
+    "crosstalk": ["fgs", "nircam", "niriss", "nirspec"],
+    "data_transfer_error": ["fgs", "nircam", "niriss", "nirspec"],
+    "ghost": ["fgs", "nircam", "niriss", "nirspec"],
+    "snowball": ["fgs", "nircam", "niriss", "nirspec"],
     # instrument-specific anomalies:
-    'column_pull_up': ['miri'],
-    'column_pull_down': ['miri'],
-    'Dominant_MSA_Leakage': ['nirspec'],
-    'dragons_breath': ['nircam'],
-    'MRS_Glow': ['miri'],
-    'MRS_Zipper': ['miri'],
-    'internal_reflection': ['miri'],
-    'optical_short': ['nirspec'],  # Only for MOS observations
-    'row_pull_up': ['miri'],
-    'row_pull_down': ['miri'],
-    'LRS_Contamination': ['miri'],
-    'tree_rings': ['miri'],
-    'scattered_light': ['niriss', 'nircam'],
-    'claws': ['nircam'],
-    'wisps': ['nircam'],
-    'tilt_event': ['nircam'],
-    'light_saber': ['niriss'],
+    "column_pull_up": ["miri"],
+    "column_pull_down": ["miri"],
+    "Dominant_MSA_Leakage": ["nirspec"],
+    "dragons_breath": ["nircam"],
+    "MRS_Glow": ["miri"],
+    "MRS_Zipper": ["miri"],
+    "internal_reflection": ["miri"],
+    "optical_short": ["nirspec"],  # Only for MOS observations
+    "row_pull_up": ["miri"],
+    "row_pull_down": ["miri"],
+    "LRS_Contamination": ["miri"],
+    "tree_rings": ["miri"],
+    "scattered_light": ["niriss", "nircam"],
+    "claws": ["nircam"],
+    "wisps": ["nircam"],
+    "tilt_event": ["nircam"],
+    "light_saber": ["niriss"],
     # additional anomalies:
-    'other': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec']}
+    "other": ["fgs", "miri", "nircam", "niriss", "nirspec"],
+}
 # anomalies that shouldn't be 'titleized'
-special_cases = ['Dominant_MSA_Leakage', 'MRS_Glow', 'MRS_Zipper', 'LRS_Contamination']
+special_cases = ["Dominant_MSA_Leakage", "MRS_Glow", "MRS_Zipper", "LRS_Contamination"]
 
 # Defines the possible anomalies to flag through the web app
-ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) if anomaly not in special_cases
-                   else (anomaly, anomaly.replace('_', ' '))
-                   for anomaly in ANOMALIES_PER_INSTRUMENT]
-
-ANOMALY_CHOICES_FGS = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT
-                       if 'fgs' in ANOMALIES_PER_INSTRUMENT[anomaly]]
-
-ANOMALY_CHOICES_MIRI = [(anomaly, inflection.titleize(anomaly)) if anomaly not in special_cases
-                        else (anomaly, anomaly.replace('_', ' '))
-                        for anomaly in ANOMALIES_PER_INSTRUMENT
-                        if 'miri' in ANOMALIES_PER_INSTRUMENT[anomaly]]
-
-ANOMALY_CHOICES_NIRCAM = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT
-                          if 'nircam' in ANOMALIES_PER_INSTRUMENT[anomaly]]
-
-ANOMALY_CHOICES_NIRISS = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT
-                          if 'niriss' in ANOMALIES_PER_INSTRUMENT[anomaly]]
-
-ANOMALY_CHOICES_NIRSPEC = [(anomaly, inflection.titleize(anomaly)) if anomaly not in special_cases
-                           else (anomaly, anomaly.replace('_', ' '))
-                           for anomaly in ANOMALIES_PER_INSTRUMENT
-                           if 'nirspec' in ANOMALIES_PER_INSTRUMENT[anomaly]]
-
-ANOMALY_CHOICES_PER_INSTRUMENT = {'fgs': ANOMALY_CHOICES_FGS,
-                                  'miri': ANOMALY_CHOICES_MIRI,
-                                  'nircam': ANOMALY_CHOICES_NIRCAM,
-                                  'niriss': ANOMALY_CHOICES_NIRISS,
-                                  'nirspec': ANOMALY_CHOICES_NIRSPEC
-                                  }
-
-APERTURES_PER_INSTRUMENT = {'nircam': [],  # NIRCAM aperture redundant, can just use Subarray + Detector
-                            'niriss': [],  # NIRISS preferred subarray only
-                            'nirspec': ['NRS_FULL_MSA', 'NRS_FULL_IFU', 'NRS_S200A1_SLIT', 'NRS_S200A2_SLIT',
-                                        'NRS_S400A1_SLIT', 'NRS_S1600A1_SLIT', 'NRS_S200B1_SLIT'],
-                            'miri': [],   # MIRI preferred subarray only
-                            'fgs': ['FGS1_FULL', 'FGS2_FULL']}
+ANOMALY_CHOICES = [
+    (anomaly, inflection.titleize(anomaly))
+    if anomaly not in special_cases
+    else (anomaly, anomaly.replace("_", " "))
+    for anomaly in ANOMALIES_PER_INSTRUMENT
+]
+
+ANOMALY_CHOICES_FGS = [
+    (anomaly, inflection.titleize(anomaly))
+    for anomaly in ANOMALIES_PER_INSTRUMENT
+    if "fgs" in ANOMALIES_PER_INSTRUMENT[anomaly]
+]
+
+ANOMALY_CHOICES_MIRI = [
+    (anomaly, inflection.titleize(anomaly))
+    if anomaly not in special_cases
+    else (anomaly, anomaly.replace("_", " "))
+    for anomaly in ANOMALIES_PER_INSTRUMENT
+    if "miri" in ANOMALIES_PER_INSTRUMENT[anomaly]
+]
+
+ANOMALY_CHOICES_NIRCAM = [
+    (anomaly, inflection.titleize(anomaly))
+    for anomaly in ANOMALIES_PER_INSTRUMENT
+    if "nircam" in ANOMALIES_PER_INSTRUMENT[anomaly]
+]
+
+ANOMALY_CHOICES_NIRISS = [
+    (anomaly, inflection.titleize(anomaly))
+    for anomaly in ANOMALIES_PER_INSTRUMENT
+    if "niriss" in ANOMALIES_PER_INSTRUMENT[anomaly]
+]
+
+ANOMALY_CHOICES_NIRSPEC = [
+    (anomaly, inflection.titleize(anomaly))
+    if anomaly not in special_cases
+    else (anomaly, anomaly.replace("_", " "))
+    for anomaly in ANOMALIES_PER_INSTRUMENT
+    if "nirspec" in ANOMALIES_PER_INSTRUMENT[anomaly]
+]
+
+ANOMALY_CHOICES_PER_INSTRUMENT = {
+    "fgs": ANOMALY_CHOICES_FGS,
+    "miri": ANOMALY_CHOICES_MIRI,
+    "nircam": ANOMALY_CHOICES_NIRCAM,
+    "niriss": ANOMALY_CHOICES_NIRISS,
+    "nirspec": ANOMALY_CHOICES_NIRSPEC,
+}
+
+APERTURES_PER_INSTRUMENT = {
+    "nircam": [],  # NIRCAM aperture redundant, can just use Subarray + Detector
+    "niriss": [],  # NIRISS preferred subarray only
+    "nirspec": [
+        "NRS_FULL_MSA",
+        "NRS_FULL_IFU",
+        "NRS_S200A1_SLIT",
+        "NRS_S200A2_SLIT",
+        "NRS_S400A1_SLIT",
+        "NRS_S1600A1_SLIT",
+        "NRS_S200B1_SLIT",
+    ],
+    "miri": [],  # MIRI preferred subarray only
+    "fgs": ["FGS1_FULL", "FGS2_FULL"],
+}
 
 # Observing templates used for ASIC tuning. MAST query results that
 # have one of these templates will be ignored
-ASIC_TEMPLATES = ['ISIM ASIC Tuning']
+ASIC_TEMPLATES = ["ISIM ASIC Tuning"]
 
 # Bad pixel types by the type of data used to find them
-BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL']
-DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH']
-FLATS_BAD_PIXEL_TYPES = ['DEAD', 'OPEN', 'ADJ_OPEN', 'LOW_QE']
+BAD_PIXEL_TYPES = [
+    "DEAD",
+    "HOT",
+    "LOW_QE",
+    "RC",
+    "OPEN",
+    "ADJ_OPEN",
+    "TELEGRAPH",
+    "OTHER_BAD_PIXEL",
+]
+DARKS_BAD_PIXEL_TYPES = ["HOT", "RC", "OTHER_BAD_PIXEL", "TELEGRAPH"]
+FLATS_BAD_PIXEL_TYPES = ["DEAD", "OPEN", "ADJ_OPEN", "LOW_QE"]
 
 # The maximum number of bad pixels allowed on a bad pixel monitor plot. If there
 # are more than this number of bad pixels identified for a particular type of
@@ -145,14 +195,16 @@
 BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT = 15000
 
 # Possible exposure types for dark current data
-DARK_EXP_TYPES = {'nircam': ['NRC_DARK'],
-                  'niriss': ['NIS_DARK'],
-                  'miri': ['MIR_DARKIMG', 'MIR_DARKMRS', 'MIR_DARKALL'],
-                  'nirspec': ['NRS_DARK'],
-                  'fgs': ['FGS_DARK']}
+DARK_EXP_TYPES = {
+    "nircam": ["NRC_DARK"],
+    "niriss": ["NIS_DARK"],
+    "miri": ["MIR_DARKIMG", "MIR_DARKMRS", "MIR_DARKALL"],
+    "nirspec": ["NRS_DARK"],
+    "fgs": ["FGS_DARK"],
+}
 
 # Types of potential bad pixels identified by the dark current monitor
-DARK_MONITOR_BADPIX_TYPES = ['hot', 'dead', 'noisy']
+DARK_MONITOR_BADPIX_TYPES = ["hot", "dead", "noisy"]
 
 # Maximum number of potential new bad pixels to overplot on the dark monitor
 # mean dark image plot. Too many overplotted points starts to obscure the image
@@ -160,48 +212,144 @@
 DARK_MONITOR_MAX_BADPOINTS_TO_PLOT = 1000
 
 # Dictionary of observing modes available for each instrument
-DETECTOR_PER_INSTRUMENT = {'miri': ['MIRIFULONG', 'MIRIFUSHORT', 'MIRIMAGE'],
-                           'nircam': ['NRCB4', 'NRCA4', 'NRCA2', 'NRCALONG',
-                                      'NRCBLONG', 'NRCB2', 'NRCB3', 'NRCA1',
-                                      'NRCA3', 'NRCB1'],
-                           'niriss': ['NIS'],
-                           'nirspec': ['NRS1', 'NRS2'],
-                           'fgs': ['GUIDER1', 'GUIDER2']}
+DETECTOR_PER_INSTRUMENT = {
+    "miri": ["MIRIFULONG", "MIRIFUSHORT", "MIRIMAGE"],
+    "nircam": [
+        "NRCB4",
+        "NRCA4",
+        "NRCA2",
+        "NRCALONG",
+        "NRCBLONG",
+        "NRCB2",
+        "NRCB3",
+        "NRCA1",
+        "NRCA3",
+        "NRCB1",
+    ],
+    "niriss": ["NIS"],
+    "nirspec": ["NRS1", "NRS2"],
+    "fgs": ["GUIDER1", "GUIDER2"],
+}
 
 # Default time range to use for EDB monitor telemetry plots. The plots will
 # go from this starting time to the monitor run time, unless otherwise requested.
 EDB_DEFAULT_PLOT_RANGE = 14  # days.
 
-EXP_TYPE_PER_INSTRUMENT = {'fgs': ['FGS_FOCUS', 'FGS_IMAGE', 'FGS_INTFLAT',
-                                   'FGS_SKYFLAT', 'FGS_DARK'],
-                           'miri': ['MIR_FLATMRS', 'MIR_MRS', 'MIR_FLATIMAGE',
-                                    'MIR_DARK', 'MIR_LYOT', 'MIR_IMAGE',
-                                    'MIR_LRS-FIXEDSLIT', 'MIR_LRS-SLITLESS',
-                                    'MIR_CORONCAL', 'MIR_4QPM', 'MIR_FLATIMAGE-EXT',
-                                    'MIR_TACQ', 'MIR_DARKMRS',
-                                    'MIR_DARKIMG', 'MIR_FLATMRS-EXT', 'MIR_TACONFIRM'],
-                           'nircam': ['NRC_LED', 'NRC_DARK', 'NRC_CORON',
-                                      'NRC_IMAGE', 'NRC_FOCUS', 'NRC_TSGRISM',
-                                      'NRC_TSIMAGE', 'NRC_WFSS', 'NRC_TACQ',
-                                      'NRC_TACONFIRM', 'NRC_FLAT', 'NRC_GRISM'],
-                           'niriss': ['NIS_IMAGE', 'NIS_FOCUS', 'NIS_SOSS',
-                                      'NIS_AMI', 'NIS_LAMP', 'NIS_WFSS', 'NIS_DARK',
-                                      'NIS_EXTCAL', 'NIS_TACONFIRM', 'NIS_TACQ'],
-                           'nirspec': ['NRS_IFU', 'NRS_MSASPEC', 'NRS_BRIGHTOBJ', 'NRS_DARK',
-                                       'NRS_AUTOWAVE', 'NRS_LAMP', 'NRS_AUTOFLAT', 'NRS_IMAGE',
-                                       'NRS_CONFIRM', 'NRS_FIXEDSLIT', 'NRS_MIMF', 'NRS_FOCUS',
-                                       'NRS_TACONFIRM', 'NRS_WATA', 'NRS_MSATA']}
-
-EXPTYPES = {"nircam": {"imaging": "NRC_IMAGE", "ts_imaging": "NRC_TSIMAGE",
-                       "wfss": "NRC_WFSS", "ts_grism": "NRC_TSGRISM"},
-            "niriss": {"imaging": "NIS_IMAGE", "ami": "NIS_IMAGE",
-                       "pom": "NIS_IMAGE", "wfss": "NIS_WFSS"},
-            "fgs": {"imaging": "FGS_IMAGE"}}
-
-EXPOSURE_PAGE_SUFFIX_ORDER = ['uncal', 'dark', 'trapsfilled', 'ramp', 'rate', 'rateints', 'fitopt', 'cal', 'calints',
-                              'msa', 'crf', 'crfints', 'bsub', 'bsubints', 'i2d', 's2d', 's3d', 'x1d', 'x1dints',
-                              'cat', 'segm', 'c1d', 'psfstack', 'psfalign', 'psfsub', 'amiavg', 'aminorm', 'ami',
-                              'psf-amiavg', 'phot', 'whtlt', 'wfscmb']
+EXP_TYPE_PER_INSTRUMENT = {
+    "fgs": ["FGS_FOCUS", "FGS_IMAGE", "FGS_INTFLAT", "FGS_SKYFLAT", "FGS_DARK"],
+    "miri": [
+        "MIR_FLATMRS",
+        "MIR_MRS",
+        "MIR_FLATIMAGE",
+        "MIR_DARK",
+        "MIR_LYOT",
+        "MIR_IMAGE",
+        "MIR_LRS-FIXEDSLIT",
+        "MIR_LRS-SLITLESS",
+        "MIR_CORONCAL",
+        "MIR_4QPM",
+        "MIR_FLATIMAGE-EXT",
+        "MIR_TACQ",
+        "MIR_DARKMRS",
+        "MIR_DARKIMG",
+        "MIR_FLATMRS-EXT",
+        "MIR_TACONFIRM",
+    ],
+    "nircam": [
+        "NRC_LED",
+        "NRC_DARK",
+        "NRC_CORON",
+        "NRC_IMAGE",
+        "NRC_FOCUS",
+        "NRC_TSGRISM",
+        "NRC_TSIMAGE",
+        "NRC_WFSS",
+        "NRC_TACQ",
+        "NRC_TACONFIRM",
+        "NRC_FLAT",
+        "NRC_GRISM",
+    ],
+    "niriss": [
+        "NIS_IMAGE",
+        "NIS_FOCUS",
+        "NIS_SOSS",
+        "NIS_AMI",
+        "NIS_LAMP",
+        "NIS_WFSS",
+        "NIS_DARK",
+        "NIS_EXTCAL",
+        "NIS_TACONFIRM",
+        "NIS_TACQ",
+    ],
+    "nirspec": [
+        "NRS_IFU",
+        "NRS_MSASPEC",
+        "NRS_BRIGHTOBJ",
+        "NRS_DARK",
+        "NRS_AUTOWAVE",
+        "NRS_LAMP",
+        "NRS_AUTOFLAT",
+        "NRS_IMAGE",
+        "NRS_CONFIRM",
+        "NRS_FIXEDSLIT",
+        "NRS_MIMF",
+        "NRS_FOCUS",
+        "NRS_TACONFIRM",
+        "NRS_WATA",
+        "NRS_MSATA",
+    ],
+}
+
+EXPTYPES = {
+    "nircam": {
+        "imaging": "NRC_IMAGE",
+        "ts_imaging": "NRC_TSIMAGE",
+        "wfss": "NRC_WFSS",
+        "ts_grism": "NRC_TSGRISM",
+    },
+    "niriss": {
+        "imaging": "NIS_IMAGE",
+        "ami": "NIS_IMAGE",
+        "pom": "NIS_IMAGE",
+        "wfss": "NIS_WFSS",
+    },
+    "fgs": {"imaging": "FGS_IMAGE"},
+}
+
+EXPOSURE_PAGE_SUFFIX_ORDER = [
+    "uncal",
+    "dark",
+    "trapsfilled",
+    "ramp",
+    "rate",
+    "rateints",
+    "fitopt",
+    "cal",
+    "calints",
+    "msa",
+    "crf",
+    "crfints",
+    "bsub",
+    "bsubints",
+    "i2d",
+    "s2d",
+    "s3d",
+    "x1d",
+    "x1dints",
+    "cat",
+    "segm",
+    "c1d",
+    "psfstack",
+    "psfalign",
+    "psfsub",
+    "amiavg",
+    "aminorm",
+    "ami",
+    "psf-amiavg",
+    "phot",
+    "whtlt",
+    "wfscmb",
+]
 
 # Filename Component Lengths
 FILE_AC_CAR_ID_LEN = 4
@@ -221,109 +369,248 @@
 FILE_VISIT_LEN = 3
 
 # MSA metadata file do not have a standard suffix attached
-FILETYPE_WO_STANDARD_SUFFIX = 'msa.fits'
-
-FLAT_EXP_TYPES = {'nircam': ['NRC_FLAT'],
-                  'niriss': ['NIS_LAMP'],
-                  'miri': ['MIR_FLATIMAGE', 'MIR_FLATMRS'],
-                  'nirspec': ['NRS_AUTOFLAT', 'NRS_LAMP'],
-                  'fgs': ['FGS_INTFLAT']}
+FILETYPE_WO_STANDARD_SUFFIX = "msa.fits"
+
+FLAT_EXP_TYPES = {
+    "nircam": ["NRC_FLAT"],
+    "niriss": ["NIS_LAMP"],
+    "miri": ["MIR_FLATIMAGE", "MIR_FLATMRS"],
+    "nirspec": ["NRS_AUTOFLAT", "NRS_LAMP"],
+    "fgs": ["FGS_INTFLAT"],
+}
 
 # output subdirectories to keep track of via the filesytem monitor
-FILESYSTEM_MONITOR_SUBDIRS = ['logs', 'outputs', 'preview_images', 'thumbnails', 'all']
-
-FILTERS_PER_INSTRUMENT = {'fgs': [],
-                          'miri': ['F560W', 'F770W', 'F1000W', 'F1065C', 'F1130W', 'F1140C', 'F1280W', 'F1500W',
-                                   'F1550C', 'F1800W', 'F2100W', 'F2300C', 'F2550W', 'F2550WR', 'FLENS', 'FND', 'OPAQUE', 'P750L'],
-                          'nircam': ['F070W', 'F090W', 'F115W', 'F140M', 'F150W', 'F150W2', 'F182M', 'F187N', 'F200W', 'F210M',
-                                     'F212N', 'WLP4', 'F277W', 'F356W', 'F444W', 'F300M', 'F335M', 'F360M', 'F410M', 'F430M',
-                                     'F460M', 'F480M', 'F250M', 'F322W2'],
-                          'niriss': ['F090W', 'F115W', 'F140M', 'F150W', 'F200W', 'F277W', 'F356W', 'F380M', 'F430M', 'F444W',
-                                     'F480M', 'GR150C', 'GR150R'],
-                          'nirspec': ['CLEAR', 'F070LP', 'F100LP', 'F110W', 'F140X', 'F170LP', 'F290LP', 'OPAQUE', 'P750L']}
+FILESYSTEM_MONITOR_SUBDIRS = ["logs", "outputs", "preview_images", "thumbnails", "all"]
+
+FILTERS_PER_INSTRUMENT = {
+    "fgs": [],
+    "miri": [
+        "F560W",
+        "F770W",
+        "F1000W",
+        "F1065C",
+        "F1130W",
+        "F1140C",
+        "F1280W",
+        "F1500W",
+        "F1550C",
+        "F1800W",
+        "F2100W",
+        "F2300C",
+        "F2550W",
+        "F2550WR",
+        "FLENS",
+        "FND",
+        "OPAQUE",
+        "P750L",
+    ],
+    "nircam": [
+        "F070W",
+        "F090W",
+        "F115W",
+        "F140M",
+        "F150W",
+        "F150W2",
+        "F182M",
+        "F187N",
+        "F200W",
+        "F210M",
+        "F212N",
+        "WLP4",
+        "F277W",
+        "F356W",
+        "F444W",
+        "F300M",
+        "F335M",
+        "F360M",
+        "F410M",
+        "F430M",
+        "F460M",
+        "F480M",
+        "F250M",
+        "F322W2",
+    ],
+    "niriss": [
+        "F090W",
+        "F115W",
+        "F140M",
+        "F150W",
+        "F200W",
+        "F277W",
+        "F356W",
+        "F380M",
+        "F430M",
+        "F444W",
+        "F480M",
+        "GR150C",
+        "GR150R",
+    ],
+    "nirspec": [
+        "CLEAR",
+        "F070LP",
+        "F100LP",
+        "F110W",
+        "F140X",
+        "F170LP",
+        "F290LP",
+        "OPAQUE",
+        "P750L",
+    ],
+}
 
-FOUR_AMP_SUBARRAYS = ['WFSS128R', 'WFSS64R']
+FOUR_AMP_SUBARRAYS = ["WFSS128R", "WFSS64R"]
 
 # Names of full-frame apertures for all instruments
-FULL_FRAME_APERTURES = {'NIRCAM': ['NRCA1_FULL', 'NRCA2_FULL', 'NRCA3_FULL', 'NRCA4_FULL',
-                                   'NRCA5_FULL', 'NRCB1_FULL', 'NRCB2_FULL', 'NRCB3_FULL',
-                                   'NRCB4_FULL', 'NRCB5_FULL'],
-                        'NIRISS': ['NIS_CEN'],
-                        'NIRSPEC': ['NRS1_FULL', 'NRS2_FULL'],
-                        'MIRI': ['MIRIM_FULL'],
-                        'FGS': ['FGS1_FULL', 'FGS2_FULL']
-                        }
+FULL_FRAME_APERTURES = {
+    "NIRCAM": [
+        "NRCA1_FULL",
+        "NRCA2_FULL",
+        "NRCA3_FULL",
+        "NRCA4_FULL",
+        "NRCA5_FULL",
+        "NRCB1_FULL",
+        "NRCB2_FULL",
+        "NRCB3_FULL",
+        "NRCB4_FULL",
+        "NRCB5_FULL",
+    ],
+    "NIRISS": ["NIS_CEN"],
+    "NIRSPEC": ["NRS1_FULL", "NRS2_FULL"],
+    "MIRI": ["MIRIM_FULL"],
+    "FGS": ["FGS1_FULL", "FGS2_FULL"],
+}
 
 # Possible suffix types for nominal files
-GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d',
-                        'x1dints', 'x1d', 's2d', 's3d', 'dark', 'crfints',
-                        'crf', 'ramp', 'fitopt', 'bsubints', 'bsub', 'cat', 'segm', 'c1d']
+GENERIC_SUFFIX_TYPES = [
+    "uncal",
+    "cal",
+    "rateints",
+    "rate",
+    "trapsfilled",
+    "i2d",
+    "x1dints",
+    "x1d",
+    "s2d",
+    "s3d",
+    "dark",
+    "crfints",
+    "crf",
+    "ramp",
+    "fitopt",
+    "bsubints",
+    "bsub",
+    "cat",
+    "segm",
+    "c1d",
+]
 
 # Gratings available for each instrument
-GRATING_PER_INSTRUMENT = {'fgs': [],
-                          'miri': [],
-                          'nircam': [],
-                          'niriss': [],
-                          'nirspec': ['G140M', 'G235M', 'G395M', 'G140H',
-                                      'G235H', 'G395H', 'PRISM', 'MIRROR']
-                          }
+GRATING_PER_INSTRUMENT = {
+    "fgs": [],
+    "miri": [],
+    "nircam": [],
+    "niriss": [],
+    "nirspec": [
+        "G140M",
+        "G235M",
+        "G395M",
+        "G140H",
+        "G235H",
+        "G395H",
+        "PRISM",
+        "MIRROR",
+    ],
+}
 
 # Filename extensions for guider data
-GUIDER_FILENAME_TYPE = ['gs-fg', 'gs-track', 'gs-id', 'gs-acq1', 'gs-acq2']
+GUIDER_FILENAME_TYPE = ["gs-fg", "gs-track", "gs-id", "gs-acq1", "gs-acq2"]
 
 # Possible suffix types for guider exposures
-GUIDER_SUFFIX_TYPES = ['stream', 'stacked_uncal', 'image_uncal', 'stacked_cal', 'image_cal']
+GUIDER_SUFFIX_TYPES = [
+    "stream",
+    "stacked_uncal",
+    "image_uncal",
+    "stacked_cal",
+    "image_cal",
+]
 
 # JWQL should ignore some filetypes in the filesystem.
-IGNORED_SUFFIXES = ['original', 'stream', 'x1d', 'x1dints', 'c1d', 'pre-image']
+IGNORED_SUFFIXES = ["original", "stream", "x1d", "x1dints", "c1d", "pre-image"]
 
 # Instrument monitor database tables
 INSTRUMENT_MONITOR_DATABASE_TABLES = {
-    'dark_monitor': ['<instrument>_dark_dark_current', '<instrument>_dark_pixel_stats', '<instrument>_dark_query_history'],
-    'bad_pixel_monitor': ['<instrument>_bad_pixel_stats', '<instrument>_bad_pixel_query_history'],
-    'cosmic_ray_monitor': ['<instrument>_cosmic_ray_stats', '<instrument>_cosmic_ray_query_history'],
-    'msata_monitor': ['<instrument>_ta_stats', '<instrument>_ta_query_history'],
-    'wata_monitor': ['<instrument>_ta_stats', '<instrument>_ta_query_history']
+    "dark_monitor": [
+        "<instrument>_dark_dark_current",
+        "<instrument>_dark_pixel_stats",
+        "<instrument>_dark_query_history",
+    ],
+    "bad_pixel_monitor": [
+        "<instrument>_bad_pixel_stats",
+        "<instrument>_bad_pixel_query_history",
+    ],
+    "cosmic_ray_monitor": [
+        "<instrument>_cosmic_ray_stats",
+        "<instrument>_cosmic_ray_query_history",
+    ],
+    "msata_monitor": ["<instrument>_ta_stats", "<instrument>_ta_query_history"],
+    "wata_monitor": ["<instrument>_ta_stats", "<instrument>_ta_query_history"],
 }
 
 INSTRUMENT_SERVICE_MATCH = {
-    'FGS': 'Mast.Jwst.Filtered.Fgs',
-    'MIRI': 'Mast.Jwst.Filtered.Miri',
-    'NIRCam': 'Mast.Jwst.Filtered.Nircam',
-    'NIRISS': 'Mast.Jwst.Filtered.Niriss',
-    'NIRSpec': 'Mast.Jwst.Filtered.Nirspec'}
+    "FGS": "Mast.Jwst.Filtered.Fgs",
+    "MIRI": "Mast.Jwst.Filtered.Miri",
+    "NIRCam": "Mast.Jwst.Filtered.Nircam",
+    "NIRISS": "Mast.Jwst.Filtered.Niriss",
+    "NIRSpec": "Mast.Jwst.Filtered.Nirspec",
+}
 
 # JWST data products
-JWST_DATAPRODUCTS = ['IMAGE', 'SPECTRUM', 'SED', 'TIMESERIES', 'VISIBILITY',
-                     'EVENTLIST', 'CUBE', 'CATALOG', 'ENGINEERING', 'NULL']
+JWST_DATAPRODUCTS = [
+    "IMAGE",
+    "SPECTRUM",
+    "SED",
+    "TIMESERIES",
+    "VISIBILITY",
+    "EVENTLIST",
+    "CUBE",
+    "CATALOG",
+    "ENGINEERING",
+    "NULL",
+]
 
 # Lowercase JWST instrument names
-JWST_INSTRUMENT_NAMES = sorted(['niriss', 'nircam', 'nirspec', 'miri', 'fgs'])
+JWST_INSTRUMENT_NAMES = sorted(["niriss", "nircam", "nirspec", "miri", "fgs"])
 
 # JWST instrument names with shorthand notation
-JWST_INSTRUMENT_NAMES_SHORTHAND = {'gui': 'fgs',
-                                   'mir': 'miri',
-                                   'nis': 'niriss',
-                                   'nrc': 'nircam',
-                                   'nrs': 'nirspec'}
+JWST_INSTRUMENT_NAMES_SHORTHAND = {
+    "gui": "fgs",
+    "mir": "miri",
+    "nis": "niriss",
+    "nrc": "nircam",
+    "nrs": "nirspec",
+}
 
 # Mixed case JWST instrument names
-JWST_INSTRUMENT_NAMES_MIXEDCASE = {'fgs': 'FGS',
-                                   'miri': 'MIRI',
-                                   'nircam': 'NIRCam',
-                                   'niriss': 'NIRISS',
-                                   'nirspec': 'NIRSpec'}
+JWST_INSTRUMENT_NAMES_MIXEDCASE = {
+    "fgs": "FGS",
+    "miri": "MIRI",
+    "nircam": "NIRCam",
+    "niriss": "NIRISS",
+    "nirspec": "NIRSpec",
+}
 
 # Upper case JWST instrument names
-JWST_INSTRUMENT_NAMES_UPPERCASE = {key: value.upper() for key, value in
-                                   JWST_INSTRUMENT_NAMES_MIXEDCASE.items()}
+JWST_INSTRUMENT_NAMES_UPPERCASE = {
+    key: value.upper() for key, value in JWST_INSTRUMENT_NAMES_MIXEDCASE.items()
+}
 
 # Astoquery service string for each JWST instrument
-JWST_MAST_SERVICES = ['Mast.Jwst.Filtered.{}'.format(value.title()) for value in
-                      JWST_INSTRUMENT_NAMES]
+JWST_MAST_SERVICES = [
+    "Mast.Jwst.Filtered.{}".format(value.title()) for value in JWST_INSTRUMENT_NAMES
+]
 
 # Possible values for look status filter
-LOOK_OPTIONS = ['New', 'Viewed']
+LOOK_OPTIONS = ["New", "Viewed"]
 
 # Maximum number of records returned by MAST for a single query
 MAST_QUERY_LIMIT = 500000
@@ -331,107 +618,163 @@
 # Expected position sensor values for MIRI. Used by the EDB monitor
 # to filter out bad values. Tuple values are the expected value and
 # the standard deviation associated with the value
-MIRI_POS_RATIO_VALUES = {'FW': {'FND': (-164.8728073, 0.204655346),
-                                'OPAQUE':  (380.6122145, 0.078856646),
-                                'F1000W':  (-24.15638797, 0.182865887),
-                                'F1130W':  (137.8245397, 0.24910941),
-                                'F1280W':  (-298.7062532, 0.229963508),
-                                'P750L':   (12.39439777, 0.246932037),
-                                'F1500W':  (-377.9888235, 0.263432415),
-                                'F1800W':  (435.9046314, 0.27885876),
-                                'F2100W':  (-126.5991201, 0.197193968),
-                                'F560W':   (218.0010353, 0.282554884),
-                                'FLENS':   (-212.7978283, 0.409300208),
-                                'F2300C':  (306.0488778, 0.265448583),
-                                'F770W':   (-62.48455213, 0.340861733),
-                                'F1550C':  (188.7366748, 0.291288105),
-                                'F2550W':  (-324.2364737, 0.176262309),
-                                'F1140C':  (82.81057729, 0.169772457),
-                                'F2550WR': (-255.5816917, 0.251581688),
-                                'F1065C':  (261.4486618, 0.16177981),
-                                },
-                         'CCC': {'CLOSED': (398.0376386, 0.173703628),
-                                 'OPEN': (504.0482685, 0.328112274)
-                                 },
-                         'GW14': {'SHORT': (626.9411005, 0.116034024),
-                                  'MEDIUM': (342.8685233, 0.127123169),
-                                  'LONG': (408.8339259, 0.117079193)
-                                  },
-                         'GW23': {'SHORT': (619.7948107, 0.215417336),
-                                  'MEDIUM': (373.1697309, 0.204314122),
-                                  'LONG': (441.6632325, 0.349161169)
-                                  }
-                         }
+MIRI_POS_RATIO_VALUES = {
+    "FW": {
+        "FND": (-164.8728073, 0.204655346),
+        "OPAQUE": (380.6122145, 0.078856646),
+        "F1000W": (-24.15638797, 0.182865887),
+        "F1130W": (137.8245397, 0.24910941),
+        "F1280W": (-298.7062532, 0.229963508),
+        "P750L": (12.39439777, 0.246932037),
+        "F1500W": (-377.9888235, 0.263432415),
+        "F1800W": (435.9046314, 0.27885876),
+        "F2100W": (-126.5991201, 0.197193968),
+        "F560W": (218.0010353, 0.282554884),
+        "FLENS": (-212.7978283, 0.409300208),
+        "F2300C": (306.0488778, 0.265448583),
+        "F770W": (-62.48455213, 0.340861733),
+        "F1550C": (188.7366748, 0.291288105),
+        "F2550W": (-324.2364737, 0.176262309),
+        "F1140C": (82.81057729, 0.169772457),
+        "F2550WR": (-255.5816917, 0.251581688),
+        "F1065C": (261.4486618, 0.16177981),
+    },
+    "CCC": {"CLOSED": (398.0376386, 0.173703628), "OPEN": (504.0482685, 0.328112274)},
+    "GW14": {
+        "SHORT": (626.9411005, 0.116034024),
+        "MEDIUM": (342.8685233, 0.127123169),
+        "LONG": (408.8339259, 0.117079193),
+    },
+    "GW23": {
+        "SHORT": (619.7948107, 0.215417336),
+        "MEDIUM": (373.1697309, 0.204314122),
+        "LONG": (441.6632325, 0.349161169),
+    },
+}
 
 # Suffix for msa files
-MSA_SUFFIX = ['msa']
+MSA_SUFFIX = ["msa"]
 
 # Available monitor names and their location for each JWST instrument
 MONITORS = {
-    'fgs': [('Bad Pixel Monitor', '/fgs/bad_pixel_monitor'),
-            ('Cosmic Ray Monitor', '#'),
-            ('Dark Current Monitor', '/fgs/dark_monitor'),
-            ('EDB Telemetry Monitor', '/fgs/edb_monitor'),
-            ('Readnoise Monitor', '/fgs/readnoise_monitor')],
-    'miri': [('Bad Pixel Monitor', '/miri/bad_pixel_monitor'),
-             ('Cosmic Ray Monitor', '#'),
-             ('Dark Current Monitor', '/miri/dark_monitor'),
-             ('EDB Telemetry Monitor', '/miri/edb_monitor'),
-             ('Readnoise Monitor', '/miri/readnoise_monitor')],
-    'nircam': [('Bad Pixel Monitor', '/nircam/bad_pixel_monitor'),
-               ('Bias Monitor', '/nircam/bias_monitor'),
-               ('Cosmic Ray Monitor', '#'),
-               ('Dark Current Monitor', '/nircam/dark_monitor'),
-               ('EDB Telemetry Monitor', '/nircam/edb_monitor'),
-               ('Readnoise Monitor', '/nircam/readnoise_monitor')],
-    'niriss': [('Bad Pixel Monitor', '/niriss/bad_pixel_monitor'),
-               ('Bias Monitor', '/niriss/bias_monitor'),
-               ('Cosmic Ray Monitor', '#'),
-               ('Dark Current Monitor', '/niriss/dark_monitor'),
-               ('EDB Telemetry Monitor', '/niriss/edb_monitor'),
-               ('Readnoise Monitor', '/niriss/readnoise_monitor')],
-    'nirspec': [('Bad Pixel Monitor', '/nirspec/bad_pixel_monitor'),
-                ('Bias Monitor', '/nirspec/bias_monitor'),
-                ('Dark Monitor', '/nirspec/dark_monitor'),
-                ('Cosmic Ray Monitor', '#'),
-                ('EDB Telemetry Monitor', '/nirspec/edb_monitor'),
-                ('MSATA Monitor', '/nirspec/msata_monitor'),
-                ('Readnoise Monitor', '/nirspec/readnoise_monitor'),
-                ('WATA Monitor', '/nirspec/wata_monitor')
-                ]}
+    "fgs": [
+        ("Bad Pixel Monitor", "/fgs/bad_pixel_monitor"),
+        ("Cosmic Ray Monitor", "#"),
+        ("Dark Current Monitor", "/fgs/dark_monitor"),
+        ("EDB Telemetry Monitor", "/fgs/edb_monitor"),
+        ("Readnoise Monitor", "/fgs/readnoise_monitor"),
+    ],
+    "miri": [
+        ("Bad Pixel Monitor", "/miri/bad_pixel_monitor"),
+        ("Cosmic Ray Monitor", "#"),
+        ("Dark Current Monitor", "/miri/dark_monitor"),
+        ("EDB Telemetry Monitor", "/miri/edb_monitor"),
+        ("Readnoise Monitor", "/miri/readnoise_monitor"),
+    ],
+    "nircam": [
+        ("Bad Pixel Monitor", "/nircam/bad_pixel_monitor"),
+        ("Bias Monitor", "/nircam/bias_monitor"),
+        ("Cosmic Ray Monitor", "#"),
+        ("Dark Current Monitor", "/nircam/dark_monitor"),
+        ("EDB Telemetry Monitor", "/nircam/edb_monitor"),
+        ("Readnoise Monitor", "/nircam/readnoise_monitor"),
+    ],
+    "niriss": [
+        ("Bad Pixel Monitor", "/niriss/bad_pixel_monitor"),
+        ("Bias Monitor", "/niriss/bias_monitor"),
+        ("Cosmic Ray Monitor", "#"),
+        ("Dark Current Monitor", "/niriss/dark_monitor"),
+        ("EDB Telemetry Monitor", "/niriss/edb_monitor"),
+        ("Readnoise Monitor", "/niriss/readnoise_monitor"),
+    ],
+    "nirspec": [
+        ("Bad Pixel Monitor", "/nirspec/bad_pixel_monitor"),
+        ("Bias Monitor", "/nirspec/bias_monitor"),
+        ("Dark Monitor", "/nirspec/dark_monitor"),
+        ("Cosmic Ray Monitor", "#"),
+        ("EDB Telemetry Monitor", "/nirspec/edb_monitor"),
+        ("MSATA Monitor", "/nirspec/msata_monitor"),
+        ("Readnoise Monitor", "/nirspec/readnoise_monitor"),
+        ("WATA Monitor", "/nirspec/wata_monitor"),
+    ],
+}
 
 # Possible suffix types for coronograph exposures
-NIRCAM_CORONAGRAPHY_SUFFIX_TYPES = ['psfstack', 'psfalign', 'psfsub']
+NIRCAM_CORONAGRAPHY_SUFFIX_TYPES = ["psfstack", "psfalign", "psfsub"]
 
 # NIRCam subarrays that use four amps for readout
-NIRCAM_FOUR_AMP_SUBARRAYS = ['WFSS128R', 'WFSS64R']
+NIRCAM_FOUR_AMP_SUBARRAYS = ["WFSS128R", "WFSS64R"]
 
 # NIRCam long wavelength detector names
-NIRCAM_LONGWAVE_DETECTORS = ['NRCA5', 'NRCB5']
+NIRCAM_LONGWAVE_DETECTORS = ["NRCA5", "NRCB5"]
 
 # NIRCam short wavelength detector names
-NIRCAM_SHORTWAVE_DETECTORS = ['NRCA1', 'NRCA2', 'NRCA3', 'NRCA4',
-                              'NRCB1', 'NRCB2', 'NRCB3', 'NRCB4']
+NIRCAM_SHORTWAVE_DETECTORS = [
+    "NRCA1",
+    "NRCA2",
+    "NRCA3",
+    "NRCA4",
+    "NRCB1",
+    "NRCB2",
+    "NRCB3",
+    "NRCB4",
+]
 
 # NIRCam subarrays that use either one or four amps
-NIRCAM_SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256']
+NIRCAM_SUBARRAYS_ONE_OR_FOUR_AMPS = [
+    "SUBGRISMSTRIPE64",
+    "SUBGRISMSTRIPE128",
+    "SUBGRISMSTRIPE256",
+]
 
 # Possible suffix types for AMI files
-NIRISS_AMI_SUFFIX_TYPES = ['amiavg', 'aminorm', 'ami', 'psf-amiavg']
+NIRISS_AMI_SUFFIX_TYPES = ["amiavg", "aminorm", "ami", "psf-amiavg"]
 
 # Base name for the file listing the preview images for a given instrument.
 # The complete name will have "_{instrument.lower}.txt" added to the end of this.
-PREVIEW_IMAGE_LISTFILE = 'preview_image_inventory'
+PREVIEW_IMAGE_LISTFILE = "preview_image_inventory"
 
 # All possible proposal categories
-PROPOSAL_CATEGORIES = ['AR', 'CAL', 'COM', 'DD', 'ENG', 'GO', 'GTO', 'NASA', 'SURVEY']
-
-PUPILS_PER_INSTRUMENT = {'nircam': ['CLEAR', 'FLAT', 'F162M', 'F164N', 'GDHS0', 'GDHS60', 'MASKBAR', 'MASKIPR', 'MASKRND',
-                                    'PINHOLES', 'WLM8', 'WLP8', 'F323N', 'F405N', 'F466N', 'F470N', 'GRISMC', 'GRISMR', 'GRISMV2', 'GRISMV3'],
-                         'niriss': ['CLEARP', 'F090W', 'F115W', 'F140M', 'F150W', 'F158M', 'F200W', 'GR700XD', 'NRM'],
-                         'nirspec': [],
-                         'miri': [],
-                         'fgs': []}
+PROPOSAL_CATEGORIES = ["AR", "CAL", "COM", "DD", "ENG", "GO", "GTO", "NASA", "SURVEY"]
+
+PUPILS_PER_INSTRUMENT = {
+    "nircam": [
+        "CLEAR",
+        "FLAT",
+        "F162M",
+        "F164N",
+        "GDHS0",
+        "GDHS60",
+        "MASKBAR",
+        "MASKIPR",
+        "MASKRND",
+        "PINHOLES",
+        "WLM8",
+        "WLP8",
+        "F323N",
+        "F405N",
+        "F466N",
+        "F470N",
+        "GRISMC",
+        "GRISMR",
+        "GRISMV2",
+        "GRISMV3",
+    ],
+    "niriss": [
+        "CLEARP",
+        "F090W",
+        "F115W",
+        "F140M",
+        "F150W",
+        "F158M",
+        "F200W",
+        "GR700XD",
+        "NRM",
+    ],
+    "nirspec": [],
+    "miri": [],
+    "fgs": [],
+}
 
 
 # Keep keys defined via class as they are used many places with potential mispellings
@@ -461,7 +804,7 @@ class QueryConfigKeys:
     QueryConfigKeys.PROPOSAL_CATEGORY: [],
     QueryConfigKeys.LOOK_STATUS: [],
     QueryConfigKeys.NUM_PER_PAGE: 100,
-    QueryConfigKeys.SORT_TYPE: 'Recent',
+    QueryConfigKeys.SORT_TYPE: "Recent",
     QueryConfigKeys.DATE_RANGE: "",
     QueryConfigKeys.ANOMALIES: {},
     QueryConfigKeys.APERTURES: {},
@@ -471,89 +814,205 @@ class QueryConfigKeys:
     QueryConfigKeys.GRATINGS: {},
     QueryConfigKeys.PUPILS: {},
     QueryConfigKeys.READ_PATTS: {},
-    QueryConfigKeys.SUBARRAYS: {}
+    QueryConfigKeys.SUBARRAYS: {},
 }
 
 # RAPID-style readout patterns for each instrument. Added so we can
 # differentiate in MAST searches for e.g. the dark current monitor
-RAPID_READPATTERNS = {'fgs': ['FGSRAPID'],
-                      'miri': ['FAST', 'FASTR1', 'SLOW', 'SLOWR1', 'FASTGRPAVG',
-                               'FASTGRPAVG8', 'FASTGRPAVG16', 'FASTGRPAVG32',
-                               'FASTGRPAVG64', 'FASTR100'],
-                      'nircam': ['RAPID'],
-                      'niriss': ['NISRAPID'],
-                      'nirspec': ['NRSRAPID', 'NRSIRS2RAPID']}
-
-READPATT_PER_INSTRUMENT = {'fgs': ['FGS', 'FGSRAPID', 'FGS60', 'FGS840', 'FGS8370'],
-                           'miri': ['FAST', 'FASTR1', 'SLOW', 'SLOWR1', 'FASTGRPAVG',
-                                    'FASTGRPAVG8', 'FASTGRPAVG16', 'FASTGRPAVG32',
-                                    'FASTGRPAVG64', 'FASTR100'],
-                           'nircam': ['RAPID', 'SHALLOW2', 'BRIGHT2', 'MEDIUM2', 'SHALLOW4',
-                                      'MEDIUM8', 'BRIGHT1', 'DEEP2', 'DEEP8'],
-                           'niriss': ['NISRAPID', 'NIS'],
-                           'nirspec': ['NRS', 'NRSRAPID', 'NRSIRS2RAPID',
-                                       'NRSRAPIDD2', 'NRSRAPIDD6']}
-
-
-REPORT_KEYS_PER_INSTRUMENT = {'fgs': ['proposal', 'exp_type',
-                                      'expstart', 'filter', 'aperture',
-                                      'detector', 'subarray', 'viewed'],
-                              'miri': ['proposal', 'exp_type',
-                                       'expstart', 'filter', 'aperture',
-                                       'detector', 'subarray', 'viewed'],
-                              'nircam': ['proposal', 'exp_type', 'expstart',
-                                         'filter', 'pupil', 'aperture',
-                                         'detector', 'subarray', 'viewed'],
-                              'niriss': ['proposal', 'exp_type', 'expstart',
-                                         'filter', 'pupil', 'aperture',
-                                         'detector', 'subarray', 'viewed'],
-                              'nirspec': ['exp_type', 'filter', 'grating',
-                                          'read_patt_num', 'viewed']}
+RAPID_READPATTERNS = {
+    "fgs": ["FGSRAPID"],
+    "miri": [
+        "FAST",
+        "FASTR1",
+        "SLOW",
+        "SLOWR1",
+        "FASTGRPAVG",
+        "FASTGRPAVG8",
+        "FASTGRPAVG16",
+        "FASTGRPAVG32",
+        "FASTGRPAVG64",
+        "FASTR100",
+    ],
+    "nircam": ["RAPID"],
+    "niriss": ["NISRAPID"],
+    "nirspec": ["NRSRAPID", "NRSIRS2RAPID"],
+}
+
+READPATT_PER_INSTRUMENT = {
+    "fgs": ["FGS", "FGSRAPID", "FGS60", "FGS840", "FGS8370"],
+    "miri": [
+        "FAST",
+        "FASTR1",
+        "SLOW",
+        "SLOWR1",
+        "FASTGRPAVG",
+        "FASTGRPAVG8",
+        "FASTGRPAVG16",
+        "FASTGRPAVG32",
+        "FASTGRPAVG64",
+        "FASTR100",
+    ],
+    "nircam": [
+        "RAPID",
+        "SHALLOW2",
+        "BRIGHT2",
+        "MEDIUM2",
+        "SHALLOW4",
+        "MEDIUM8",
+        "BRIGHT1",
+        "DEEP2",
+        "DEEP8",
+    ],
+    "niriss": ["NISRAPID", "NIS"],
+    "nirspec": ["NRS", "NRSRAPID", "NRSIRS2RAPID", "NRSRAPIDD2", "NRSRAPIDD6"],
+}
 
-# Possible values for sort order
-SORT_OPTIONS = ['Ascending', 'Descending', 'Recent', 'Oldest']
 
-SUBARRAYS_ONE_OR_FOUR_AMPS = ['SUBGRISMSTRIPE64', 'SUBGRISMSTRIPE128', 'SUBGRISMSTRIPE256']
+REPORT_KEYS_PER_INSTRUMENT = {
+    "fgs": [
+        "proposal",
+        "exp_type",
+        "expstart",
+        "filter",
+        "aperture",
+        "detector",
+        "subarray",
+        "viewed",
+    ],
+    "miri": [
+        "proposal",
+        "exp_type",
+        "expstart",
+        "filter",
+        "aperture",
+        "detector",
+        "subarray",
+        "viewed",
+    ],
+    "nircam": [
+        "proposal",
+        "exp_type",
+        "expstart",
+        "filter",
+        "pupil",
+        "aperture",
+        "detector",
+        "subarray",
+        "viewed",
+    ],
+    "niriss": [
+        "proposal",
+        "exp_type",
+        "expstart",
+        "filter",
+        "pupil",
+        "aperture",
+        "detector",
+        "subarray",
+        "viewed",
+    ],
+    "nirspec": ["exp_type", "filter", "grating", "read_patt_num", "viewed"],
+}
 
-SUBARRAYS_PER_INSTRUMENT = {'nircam': ['FULL', 'FULLP', 'SUB640', 'SUB320', 'SUB160', 'SUB400P', 'SUB160P', 'SUB64P',
-                                       'SUB32TATS', 'SUB640A210R', 'SUB640ASWB', 'SUB320A335R', 'SUB320A430R', 'SUB320ALWB',
-                                       'SUBGRISM256', 'SUBGRISM128', 'SUBGRISM64', 'SUB32TATSGRISM'],
-                            'niriss': ['FULL', 'SUBSTRIP96', 'SUBSTRIP256', 'SUB80', 'SUB64', 'SUB128', 'SUB256',
-                                       'WFSS64R', 'WFSS128R', 'WFSS64C', 'WFSS128C', 'SUBAMPCAL', 'SUBTAAMI', 'SUBTASOSS'],
-                            'nirspec': [],
-                            'miri': ['BRIGHTSKY', 'FULL', 'MASK1065', 'MASK1140', 'MASK1550', 'MASKLYOT', 'SLITLESSPRISM',
-                                     'SUB64', 'SUB128', 'SUB256'],
-                            'fgs': []}
+# Possible values for sort order
+SORT_OPTIONS = ["Ascending", "Descending", "Recent", "Oldest"]
+
+SUBARRAYS_ONE_OR_FOUR_AMPS = [
+    "SUBGRISMSTRIPE64",
+    "SUBGRISMSTRIPE128",
+    "SUBGRISMSTRIPE256",
+]
+
+SUBARRAYS_PER_INSTRUMENT = {
+    "nircam": [
+        "FULL",
+        "FULLP",
+        "SUB640",
+        "SUB320",
+        "SUB160",
+        "SUB400P",
+        "SUB160P",
+        "SUB64P",
+        "SUB32TATS",
+        "SUB640A210R",
+        "SUB640ASWB",
+        "SUB320A335R",
+        "SUB320A430R",
+        "SUB320ALWB",
+        "SUBGRISM256",
+        "SUBGRISM128",
+        "SUBGRISM64",
+        "SUB32TATSGRISM",
+    ],
+    "niriss": [
+        "FULL",
+        "SUBSTRIP96",
+        "SUBSTRIP256",
+        "SUB80",
+        "SUB64",
+        "SUB128",
+        "SUB256",
+        "WFSS64R",
+        "WFSS128R",
+        "WFSS64C",
+        "WFSS128C",
+        "SUBAMPCAL",
+        "SUBTAAMI",
+        "SUBTASOSS",
+    ],
+    "nirspec": [],
+    "miri": [
+        "BRIGHTSKY",
+        "FULL",
+        "MASK1065",
+        "MASK1140",
+        "MASK1550",
+        "MASKLYOT",
+        "SLITLESSPRISM",
+        "SUB64",
+        "SUB128",
+        "SUB256",
+    ],
+    "fgs": [],
+}
 
 # Filename suffixes that need to include the association value in the suffix in
 # order to identify the preview image file. This should only be crf and crfints,
 # since those are essentially level 2 files that are output by the level 3 pipeline.
-SUFFIXES_TO_ADD_ASSOCIATION = ['crf', 'crfints']
+SUFFIXES_TO_ADD_ASSOCIATION = ["crf", "crfints"]
 
 # Filename suffixes where data have been averaged over integrations
-SUFFIXES_WITH_AVERAGED_INTS = ['rate', 'cal', 'crf', 'i2d', 'bsub']
+SUFFIXES_WITH_AVERAGED_INTS = ["rate", "cal", "crf", "i2d", "bsub"]
 
 # boolean accessed according to a viewed flag
-THUMBNAIL_FILTER_LOOK = ['New', 'Viewed']
+THUMBNAIL_FILTER_LOOK = ["New", "Viewed"]
 
 # Base name for the file listing the thumbnail images for a given instrument.
 # The complete name will have "_{instrument.lower}.txt" added to the end of this.
-THUMBNAIL_LISTFILE = 'thumbnail_inventory'
+THUMBNAIL_LISTFILE = "thumbnail_inventory"
 
 # Possible suffix types for time-series exposures
-TIME_SERIES_SUFFIX_TYPES = ['phot', 'whtlt']
+TIME_SERIES_SUFFIX_TYPES = ["phot", "whtlt"]
 
 # Possible suffix types for WFS&C files
-WFSC_SUFFIX_TYPES = ['wfscmb']
+WFSC_SUFFIX_TYPES = ["wfscmb"]
 
 # Concatenate all suffix types (ordered to ensure successful matching)
-FILE_SUFFIX_TYPES = GUIDER_SUFFIX_TYPES + GENERIC_SUFFIX_TYPES + \
-    TIME_SERIES_SUFFIX_TYPES + NIRCAM_CORONAGRAPHY_SUFFIX_TYPES + \
-    NIRISS_AMI_SUFFIX_TYPES + WFSC_SUFFIX_TYPES + MSA_SUFFIX
+FILE_SUFFIX_TYPES = (
+    GUIDER_SUFFIX_TYPES
+    + GENERIC_SUFFIX_TYPES
+    + TIME_SERIES_SUFFIX_TYPES
+    + NIRCAM_CORONAGRAPHY_SUFFIX_TYPES
+    + NIRISS_AMI_SUFFIX_TYPES
+    + WFSC_SUFFIX_TYPES
+    + MSA_SUFFIX
+)
 
 # Instrument Documentation Links
-URL_DICT = {'fgs': 'https://jwst-docs.stsci.edu/jwst-observatory-hardware/jwst-fine-guidance-sensor',
-            'miri': 'https://jwst-docs.stsci.edu/jwst-mid-infrared-instrument',
-            'niriss': 'https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph',
-            'nirspec': 'https://jwst-docs.stsci.edu/jwst-near-infrared-spectrograph',
-            'nircam': 'https://jwst-docs.stsci.edu/jwst-near-infrared-camera'}
+URL_DICT = {
+    "fgs": "https://jwst-docs.stsci.edu/jwst-observatory-hardware/jwst-fine-guidance-sensor",
+    "miri": "https://jwst-docs.stsci.edu/jwst-mid-infrared-instrument",
+    "niriss": "https://jwst-docs.stsci.edu/jwst-near-infrared-imager-and-slitless-spectrograph",
+    "nirspec": "https://jwst-docs.stsci.edu/jwst-near-infrared-spectrograph",
+    "nircam": "https://jwst-docs.stsci.edu/jwst-near-infrared-camera",
+}
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 23862e07d..170390673 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -406,8 +406,8 @@ def get_available_suffixes(all_suffixes, return_untracked=True):
     untracked_suffixes = set(all_suffixes)
     for poss_suffix in EXPOSURE_PAGE_SUFFIX_ORDER:
         if 'crf' not in poss_suffix:
-            if (poss_suffix in all_suffixes
-                    and poss_suffix not in suffixes):
+            if (poss_suffix in all_suffixes and
+                poss_suffix not in suffixes):
                 suffixes.append(poss_suffix)
                 untracked_suffixes.remove(poss_suffix)
         else:
@@ -417,8 +417,8 @@ def get_available_suffixes(all_suffixes, return_untracked=True):
             # So in this case, we strip the e.g. o001 from the
             # suffixes and check which list elements match.
             for image_suffix in all_suffixes:
-                if (image_suffix.endswith(poss_suffix)
-                        and image_suffix not in suffixes):
+                if (image_suffix.endswith(poss_suffix) and
+                    image_suffix not in suffixes):
                     suffixes.append(image_suffix)
                     untracked_suffixes.remove(image_suffix)
 
@@ -1407,7 +1407,7 @@ def get_proposals_by_category(instrument):
     unique_results = list(map(dict, set(tuple(sorted(sub.items())) for sub in results)))
 
     # Make a dictionary of {program: category} to pull from
-    proposals_by_category = {d['program']:d['category'] for d in unique_results}
+    proposals_by_category = {d['program']: d['category'] for d in unique_results}
 
     return proposals_by_category
 
@@ -1504,7 +1504,7 @@ def get_rootnames_from_query(parameters):
     """
 
     filtered_rootnames = []
-    DATE_FORMAT = "%Y/%m/%d %I:%M%p" #noqa n806
+    DATE_FORMAT = "%Y/%m/%d %I:%M%p"  # noqa n806
 
     # Parse DATE_RANGE string into correct format
     date_range = parameters[QueryConfigKeys.DATE_RANGE]
@@ -1513,7 +1513,7 @@ def get_rootnames_from_query(parameters):
     start_datetime = datetime.strptime(start_date_range, DATE_FORMAT)
     stop_datetime = datetime.strptime(stop_date_range, DATE_FORMAT)
     # store as astroquery Time objects in isot format to be used in filter (with mjd format)
-    start_time  = Time(start_datetime.isoformat(), format="isot")
+    start_time = Time(start_datetime.isoformat(), format="isot")
     stop_time = Time(stop_datetime.isoformat(), format="isot")
 
     # Each Query Selection is Instrument specific
@@ -2002,6 +2002,7 @@ def thumbnails_ajax(inst, proposal, obs_num=None):
 
     return data_dict
 
+
 def thumbnails_query_ajax(rootnames):
     """Generate a page that provides data necessary to render the
     ``thumbnails`` template.
diff --git a/jwql/website/apps/jwql/forms.py b/jwql/website/apps/jwql/forms.py
index f156350b8..5b7d7ac49 100644
--- a/jwql/website/apps/jwql/forms.py
+++ b/jwql/website/apps/jwql/forms.py
@@ -62,7 +62,7 @@ def view_function(request):
 from jwql.utils.constants import (ANOMALY_CHOICES_PER_INSTRUMENT, ANOMALIES_PER_INSTRUMENT, APERTURES_PER_INSTRUMENT, DETECTOR_PER_INSTRUMENT,
                                   EXP_TYPE_PER_INSTRUMENT, FILTERS_PER_INSTRUMENT, GENERIC_SUFFIX_TYPES, GRATING_PER_INSTRUMENT,
                                   GUIDER_FILENAME_TYPE, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_INSTRUMENT_NAMES_SHORTHAND,
-                                  READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES,  SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT,
+                                  READPATT_PER_INSTRUMENT, IGNORED_SUFFIXES, SUBARRAYS_PER_INSTRUMENT, PUPILS_PER_INSTRUMENT,
                                   LOOK_OPTIONS, SORT_OPTIONS, PROPOSAL_CATEGORIES)
 from jwql.utils.utils import (get_config, get_rootnames_for_instrument_proposal, filename_parser, query_format)
 
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index 6c054ed72..afe91f441 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -98,7 +98,7 @@
     re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/$'.format(instruments), views.explore_image_ajax,
             name='explore_image_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/(?P<file_root>.+)_(?P<filetype>.+)/explore_image/plot_(?P<line_plots>(true|false))/ext_(?P<ext_name>.+)/int1_(?P<int1_nr>.+)/grp1_(?P<grp1_nr>.+)/int2_(?P<int2_nr>.+)/grp2_(?P<grp2_nr>.+)/$'.format(instruments),
-             views.explore_image_ajax, name='explore_image_ajax'),
+            views.explore_image_ajax, name='explore_image_ajax'),
     re_path(r'^ajax/(?P<inst>({}))/archive/(?P<proposal>[\d]{{1,5}})/obs(?P<observation>[\d]{{1,3}})/$'.format(instruments),
             views.archive_thumbnails_ajax, name='archive_thumb_ajax'),
     re_path(r'^ajax/viewed/(?P<file_root>.+)/$', views.toggle_viewed_ajax, name='toggle_viewed_ajax'),
diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py
index 30e6a0a26..3fc36eaa7 100644
--- a/jwql/website/apps/jwql/views.py
+++ b/jwql/website/apps/jwql/views.py
@@ -218,6 +218,7 @@ def save_page_navigation_data_ajax(request):
     context = {'item': request.session['navigation_data']}
     return JsonResponse(context, json_dumps_params={'indent': 2})
 
+
 def archived_proposals(request, inst):
     """Generate the page listing all archived proposals in the database
 

From ea4ae52c2a4a1b80fb3834972455f2eb22dd0d0b Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 8 Nov 2023 12:33:34 -0500
Subject: [PATCH 237/256] more pep8

---
 jwql/utils/constants.py                   | 14 +++++++-------
 jwql/website/apps/jwql/data_containers.py |  8 ++++----
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 39db77e92..499921019 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -999,13 +999,13 @@ class QueryConfigKeys:
 
 # Concatenate all suffix types (ordered to ensure successful matching)
 FILE_SUFFIX_TYPES = (
-    GUIDER_SUFFIX_TYPES
-    + GENERIC_SUFFIX_TYPES
-    + TIME_SERIES_SUFFIX_TYPES
-    + NIRCAM_CORONAGRAPHY_SUFFIX_TYPES
-    + NIRISS_AMI_SUFFIX_TYPES
-    + WFSC_SUFFIX_TYPES
-    + MSA_SUFFIX
+    GUIDER_SUFFIX_TYPES +
+    GENERIC_SUFFIX_TYPES +
+    TIME_SERIES_SUFFIX_TYPES +
+    NIRCAM_CORONAGRAPHY_SUFFIX_TYPES +
+    NIRISS_AMI_SUFFIX_TYPES +
+    WFSC_SUFFIX_TYPES +
+    MSA_SUFFIX
 )
 
 # Instrument Documentation Links
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 170390673..5f8033e04 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -406,10 +406,10 @@ def get_available_suffixes(all_suffixes, return_untracked=True):
     untracked_suffixes = set(all_suffixes)
     for poss_suffix in EXPOSURE_PAGE_SUFFIX_ORDER:
         if 'crf' not in poss_suffix:
-            if (poss_suffix in all_suffixes and
-                poss_suffix not in suffixes):
-                suffixes.append(poss_suffix)
-                untracked_suffixes.remove(poss_suffix)
+            if (poss_suffix in all_suffixes
+                and poss_suffix not in suffixes):
+                    suffixes.append(poss_suffix)
+                    untracked_suffixes.remove(poss_suffix)
         else:
             # EXPOSURE_PAGE_SUFFIX_ORDER contains crf and crfints,
             # but the actual suffixes in the data will be e.g. o001_crf,

From 445b84ba96806daa62c195e608a62889578761a6 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 8 Nov 2023 12:43:04 -0500
Subject: [PATCH 238/256] fix anti-pattern

---
 jwql/utils/constants.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index 499921019..39db77e92 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -999,13 +999,13 @@ class QueryConfigKeys:
 
 # Concatenate all suffix types (ordered to ensure successful matching)
 FILE_SUFFIX_TYPES = (
-    GUIDER_SUFFIX_TYPES +
-    GENERIC_SUFFIX_TYPES +
-    TIME_SERIES_SUFFIX_TYPES +
-    NIRCAM_CORONAGRAPHY_SUFFIX_TYPES +
-    NIRISS_AMI_SUFFIX_TYPES +
-    WFSC_SUFFIX_TYPES +
-    MSA_SUFFIX
+    GUIDER_SUFFIX_TYPES
+    + GENERIC_SUFFIX_TYPES
+    + TIME_SERIES_SUFFIX_TYPES
+    + NIRCAM_CORONAGRAPHY_SUFFIX_TYPES
+    + NIRISS_AMI_SUFFIX_TYPES
+    + WFSC_SUFFIX_TYPES
+    + MSA_SUFFIX
 )
 
 # Instrument Documentation Links

From bafa6c2a5d700945e9591d15c0f5dd2bd6b1e2a5 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Wed, 8 Nov 2023 14:18:36 -0500
Subject: [PATCH 239/256] put DateRange above GENERAL options row

---
 .../apps/jwql/templates/jwql_query.html       | 44 ++++++++++---------
 1 file changed, 23 insertions(+), 21 deletions(-)

diff --git a/jwql/website/apps/jwql/templates/jwql_query.html b/jwql/website/apps/jwql/templates/jwql_query.html
index 78cb34e74..6274c5a83 100644
--- a/jwql/website/apps/jwql/templates/jwql_query.html
+++ b/jwql/website/apps/jwql/templates/jwql_query.html
@@ -73,6 +73,29 @@ <h1>Dynamic Query Form</h1>
             <form class='form-horizontal' id='searchform' method='post'>
                 <hr class="col-md-12">
                 <div class='instrument-form-group' id='instrument_form_group'>
+                    <div style="display: flex" id="DATERANGE">
+                        <div class="col-md-3">
+                            <span class="help-block">Date Range</span>
+                            <div class="form-control" id='date_range'>
+                                <input type="text" name="date_range" style="cursor: pointer; padding: 5px 10px; width: 100%"/>
+                                <script>
+                                $(function() {
+                                    $('input[name="date_range"]').daterangepicker({
+                                        timePicker: true,
+                                        timePicker24Hour: true,
+                                        showDropdowns: true,
+                                        minDate: "2021/12/05",
+                                        startDate: "2021/12/05",
+                                        endDate: moment().startOf('hour'),
+                                        locale: {
+                                        format: 'YYYY/MM/DD hh:mmA'
+                                        }
+                                    });
+                                });
+                                </script>
+                            </div>
+                        </div>
+                    </div>
                     <!-- Initial INSTRUMENT Checkbox -->
                     <div style="display: flex" id="GENERAL">
                         <div class="col-md-2">
@@ -105,27 +128,6 @@ <h1>Dynamic Query Form</h1>
                                 {% endfor %}
                             </div>
                         </div>
-                        <div class="col-md-2">
-                            <span class="help-block">Date Range</span>
-                            <div class="form-control" id='date_range'>
-                                <input type="text" name="date_range" style="cursor: pointer; padding: 5px 10px; width: 100%"/>
-                                <script>
-                                $(function() {
-                                    $('input[name="date_range"]').daterangepicker({
-                                        timePicker: true,
-                                        timePicker24Hour: true,
-                                        showDropdowns: true,
-                                        minDate: "2021/12/05",
-                                        startDate: "2021/12/05",
-                                        endDate: moment().startOf('hour'),
-                                        locale: {
-                                        format: 'YYYY/MM/DD hh:mmA'
-                                        }
-                                    });
-                                });
-                                </script>
-                            </div>
-                        </div>
                         <div class="col-md-2">
                             <span class="help-block">Sort Type</span>
                             <div class="form-control" id='sort_type'>

From 690725967083bbdace7797579c343538ad6053db Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 13 Nov 2023 14:46:47 -0500
Subject: [PATCH 240/256] update filters key value pair in params for get
 proposals by category

---
 jwql/website/apps/jwql/data_containers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index f4ecb6aa7..95b4a41be 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1396,7 +1396,7 @@ def get_proposals_by_category(instrument):
 
     service = "Mast.Jwst.Filtered.{}".format(instrument)
     params = {"columns": "program, category",
-              "filters": []}
+              "filters": [{'paramName':'instrume', 'values':[instrument]}]}
     response = Mast.service_request_async(service, params)
     results = response[0].json()['data']
 

From c5b286e97748699c8d2f7f60cf3b753894c89ae0 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 13 Nov 2023 15:17:53 -0500
Subject: [PATCH 241/256] Updating filters key in params for
 test_astroquery_mast

---
 jwql/tests/test_mast_utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/tests/test_mast_utils.py b/jwql/tests/test_mast_utils.py
index 6267be9bc..74c3f0328 100755
--- a/jwql/tests/test_mast_utils.py
+++ b/jwql/tests/test_mast_utils.py
@@ -34,8 +34,10 @@
 def test_astroquery_mast():
     """Test if the astroquery.mast service can complete a request"""
     service = 'Mast.Caom.Filtered'
-    params = {'columns': 'COUNT_BIG(*)', 'filters': [], 'pagesize': 1,
-              'page': 1}
+    params = {'columns': 'COUNT_BIG(*)', 
+              'filters': [{"paramName": "obs_collection",
+                           "values": ["JWST"]},], 
+              'pagesize': 1, 'page': 1}
     response = Mast.service_request_async(service, params)
     result = response[0].json()
 

From d62aab527a4812ccc7966cab16efd2aeca64c77a Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Wed, 22 Nov 2023 12:21:35 -0500
Subject: [PATCH 242/256] Update CHANGES.rst

---
 CHANGES.rst | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index 2452df232..6280f83ca 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,3 +1,79 @@
+## What's Changed
+
+1.2.0 (2023-11-21)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Hover over proposal thumbnails by @shanosborne in https://github.com/spacetelescope/jwql/pull/927
+- Not found images replaced with default image by @BradleySappington in https://github.com/spacetelescope/jwql/pull/949
+- Remove second set of colorbar tick labels from preview images by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/963
+- Split program page by obsnum by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/948
+- Interactive preview images by @BradleySappington in https://github.com/spacetelescope/jwql/pull/965
+- Alert user to non-existant proposal number by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/994
+- Fix bad thumbnails by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/997
+- Explore by group type by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1000
+- Default scaling params for preview images where all pix are NaN by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1006
+- Explore int grp difference by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1020
+- Add inst select home search by @mengesser in https://github.com/spacetelescope/jwql/pull/1010
+- Allow proposal thumbnails to come from dark thumbnails by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1001
+- Adding Next and Previous Buttons by @mfixstsci in https://github.com/spacetelescope/jwql/pull/978
+- Build archive page from database by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1030
+- Select anomaly name formatting by @rcooper295 in https://github.com/spacetelescope/jwql/pull/1053
+- Implement "Viewed" Button and Filtering by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1049
+- fixes for no data found on server by @penaguerrero in https://github.com/spacetelescope/jwql/pull/1042
+- Filter archive page by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1065
+- Exposure page: Default to show rate img, keep order of radio buttons constant by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1111
+- Filter type by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1092
+- Sort obs date by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1113
+- added scattered light, light saber anomalies for niriss by @rcooper295 in https://github.com/spacetelescope/jwql/pull/1133
+- Add new NIRCam anomalies by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1136
+- Add Logging View by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1080
+- Remove bokeh templating from Dark monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1153
+- Fix API view for filenames per proposal and preview images by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1046
+- Create archive page asynchronously by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1203
+- Adding sort by category by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1155
+- Improve bad pixel monitor plots by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1202
+- List viewed/new status by instrument by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1197
+- Display all detectors by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1213
+- Remove bokeh templating from Bias Monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1230
+- Reformat query page functionality by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1240
+- Add download links to image pages by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1283
+- Improve explore image by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1282
+- Remove Bokeh Templating from Readnoise Monitor by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1238
+- Move Date Range selection to Query Page by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1396
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Make Readnoise Monitor less memory-intensive  by @bsunnquist in https://github.com/spacetelescope/jwql/pull/922
+- Speed up MAST queries by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/942
+- Lock python by @BradleySappington in https://github.com/spacetelescope/jwql/pull/955
+- Reduce memory usage by the dark monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/908
+- Fix clipping logic in preview image scaling by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1011
+- Cosmic ray monitor (via mengesser:develop) by @bourque in https://github.com/spacetelescope/jwql/pull/700
+- nirspec TA monitors: wata and msata by @penaguerrero in https://github.com/spacetelescope/jwql/pull/888
+- allow per-instrument database table reset by @york-stsci in https://github.com/spacetelescope/jwql/pull/1031
+- Celery monitors by @york-stsci in https://github.com/spacetelescope/jwql/pull/921
+- Raise the upper limit on number of returned Mast entries by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1048
+- Added chunked iteration to CR monitor by @york-stsci in https://github.com/spacetelescope/jwql/pull/1066
+- Created a clone_tables script by @york-stsci in https://github.com/spacetelescope/jwql/pull/1059
+- Add EDB telemetry monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/832
+- Tweaks to mnemonics queried by the EDB monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1115
+- Fix failing MIRI dark montior by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1129
+- Unit test cleanup by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1167
+- Catch missing obs in navigation data by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1206
+- Update preview image generator to handle new NaN pixels by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1212
+- drop support for Python 3.8 by @zacharyburnett in https://github.com/spacetelescope/jwql/pull/1249
+- Adding Python 3.10 Support to Testing Matrix by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1257
+- Remove mast monitor by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1256
+- Improve general JWQL queries by @melanieclarke in https://github.com/spacetelescope/jwql/pull/1281
+- Prepare models for external postgres db by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1287
+- Pipeline skip already run steps by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1262
+- Include migrations in source control by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1364
+- Add step_args to calwebb_detector1_save_jump by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1358
+
+--Full Changelog--: https://github.com/spacetelescope/jwql/compare/1.1.0...1.2.0
+
 Unreleased
 ==========
 

From e7af577b3689c7941dba060d7dac4d89e860b514 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 29 Nov 2023 00:13:39 +0000
Subject: [PATCH 243/256] Bump cryptography from 41.0.4 to 41.0.6

Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.4 to 41.0.6.
- [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pyca/cryptography/compare/41.0.4...41.0.6)

---
updated-dependencies:
- dependency-name: cryptography
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 6a7f670cd..7ec4993b5 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ bandit==1.7.5
 beautifulsoup4==4.12.2
 bokeh==2.4.3
 celery==5.3.4
-cryptography==41.0.4
+cryptography==41.0.6
 django==4.2.5
 inflection==0.5.1
 ipython==8.16.1

From a78500998b83fd77519c0b5e264e4d1d1ef0d31f Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Thu, 14 Dec 2023 11:58:02 -0500
Subject: [PATCH 244/256] filter thumbnails on visit

---
 jwql/website/apps/jwql/data_containers.py | 20 +++++++++++++++-----
 jwql/website/apps/jwql/static/js/jwql.js  |  3 ++-
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 296e29249..6dad35bc8 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1968,23 +1968,29 @@ def thumbnails_ajax(inst, proposal, obs_num=None):
 
     # Extract information for sorting with dropdown menus
     # (Don't include the proposal as a sorting parameter if the proposal has already been specified)
-    detectors, proposals = [], []
+    detectors, proposals, visits = [], [], []
     for rootname in list(data_dict['file_data'].keys()):
         proposals.append(data_dict['file_data'][rootname]['filename_dict']['program_id'])
         try:  # Some rootnames cannot parse out detectors
             detectors.append(data_dict['file_data'][rootname]['filename_dict']['detector'])
         except KeyError:
             pass
+        try:  # Some rootnames cannot parse out visit
+            visits.append(data_dict['file_data'][rootname]['filename_dict']['visit'])
+        except KeyError:
+            pass
 
     if proposal is not None:
         dropdown_menus = {'detector': sorted(detectors),
                           'look': THUMBNAIL_FILTER_LOOK,
-                          'exp_type': sorted(exp_types)}
+                          'exp_type': sorted(exp_types),
+                          'visit': sorted(visits)}
     else:
         dropdown_menus = {'detector': sorted(detectors),
                           'proposal': sorted(proposals),
                           'look': THUMBNAIL_FILTER_LOOK,
-                          'exp_type': sorted(exp_types)}
+                          'exp_type': sorted(exp_types),
+                          'visit': sorted(visits)}
 
     data_dict['tools'] = MONITORS
     data_dict['dropdown_menus'] = dropdown_menus
@@ -2079,10 +2085,14 @@ def thumbnails_query_ajax(rootnames):
                    rootname in list(data_dict['file_data'].keys())]
     proposals = [data_dict['file_data'][rootname]['filename_dict']['program_id'] for
                  rootname in list(data_dict['file_data'].keys())]
+    visits = [data_dict['file_data'][rootname]['filename_dict']['visit'] for
+                 rootname in list(data_dict['file_data'].keys())]
 
+    #SAPP TODO ADD VISIT HERE IN DROPDOWN MENUS
     dropdown_menus = {'instrument': instruments,
-                      'detector': detectors,
-                      'proposal': proposals}
+                      'detector': sorted(detectors),
+                      'proposal': sorted(proposals),
+                      'visit': sorted(visits)}
 
     data_dict['tools'] = MONITORS
     data_dict['dropdown_menus'] = dropdown_menus
diff --git a/jwql/website/apps/jwql/static/js/jwql.js b/jwql/website/apps/jwql/static/js/jwql.js
index 54ff39519..0f4ee63ba 100644
--- a/jwql/website/apps/jwql/static/js/jwql.js
+++ b/jwql/website/apps/jwql/static/js/jwql.js
@@ -1327,7 +1327,8 @@ function update_thumbnail_array(data) {
         var content = '<div class="thumbnail" data-instrument="' + instrument +
                       '" data-detector="' + filename_dict.detector + '" data-proposal="' + filename_dict.program_id +
                       '" data-file_root="' + rootname + '" data-group_root="' + filename_dict.group_root +
-                      '" data-exp_start="' + file.expstart + '" data-look="' + viewed + '" data-exp_type="' + exp_type + '">';
+                      '" data-exp_start="' + file.expstart + '" data-look="' + viewed + '" data-exp_type="' + exp_type +
+                      '" data-visit="' + filename_dict.visit + '">';
         content += '<div class="thumbnail-group">'
         content += '<a class="thumbnail-link" href="#" data-image-href="/' +
                    instrument + '/' + rootname + '/" data-group-href="/' +

From 6f8054b9a1afb51de06e75fe8a7026d441456946 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Thu, 14 Dec 2023 12:12:08 -0500
Subject: [PATCH 245/256] remove personal TODO

---
 jwql/website/apps/jwql/data_containers.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 6dad35bc8..4958451c9 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -2088,7 +2088,6 @@ def thumbnails_query_ajax(rootnames):
     visits = [data_dict['file_data'][rootname]['filename_dict']['visit'] for
                  rootname in list(data_dict['file_data'].keys())]
 
-    #SAPP TODO ADD VISIT HERE IN DROPDOWN MENUS
     dropdown_menus = {'instrument': instruments,
                       'detector': sorted(detectors),
                       'proposal': sorted(proposals),

From 71ecba104999f3f69d9fb8543cc60d09327eec6f Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Fri, 15 Dec 2023 10:20:06 -0500
Subject: [PATCH 246/256] file path needs to be set before calling monitor
 methods

---
 .../apps/jwql/monitor_pages/monitor_readnoise_bokeh.py        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 543dcac9a..5f296474e 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -114,12 +114,12 @@ def __init__(self, instrument, aperture):
 
         self.db = ReadnoiseMonitorData(self.instrument, self.aperture)
 
+        self.file_path = os.path.join(OUTPUTS_DIR, "readnoise_monitor", "data", self.ins_ap)
+
         self.plot_readnoise_amplifers()
         self.plot_readnoise_difference_image()
         self.plot_readnoise_histogram()
 
-        self.file_path = os.path.join(OUTPUT_DIR, "readnoise_monitor", "data", self.ins_ap)
-
         self.tab = Panel(child=column(row(*self.amp_plots),
                                       self.diff_image_plot,
                                       self.readnoise_histogram),

From 25490ec87f1110401d25315448f4f38ab1570639 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Fri, 15 Dec 2023 14:16:35 -0500
Subject: [PATCH 247/256] Updating static path for files

---
 .../website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 5f296474e..043f164d4 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -26,6 +26,7 @@
 from bokeh.models import ColumnDataSource, HoverTool
 # from bokeh.models import TabPanel, Tabs  # bokeh >= 3.0
 from bokeh.plotting import figure
+from django.templatetags.static import static
 import numpy as np
 
 from jwql.database.database_interface import session
@@ -114,7 +115,7 @@ def __init__(self, instrument, aperture):
 
         self.db = ReadnoiseMonitorData(self.instrument, self.aperture)
 
-        self.file_path = os.path.join(OUTPUTS_DIR, "readnoise_monitor", "data", self.ins_ap)
+        self.file_path = static(os.path.join("outputs", "readnoise_monitor", "data", self.ins_ap))
 
         self.plot_readnoise_amplifers()
         self.plot_readnoise_difference_image()

From 7f296852c104485cbd1cc7f7619e898ad5216c35 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 18 Dec 2023 10:09:39 -0500
Subject: [PATCH 248/256] Adding server name to path for data access

---
 .../apps/jwql/monitor_pages/monitor_readnoise_bokeh.py        | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 043f164d4..814ff7239 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -115,7 +115,9 @@ def __init__(self, instrument, aperture):
 
         self.db = ReadnoiseMonitorData(self.instrument, self.aperture)
 
-        self.file_path = static(os.path.join("outputs", "readnoise_monitor", "data", self.ins_ap))
+        # Use outputs directory to obtain server name in path.
+        server_path = OUTPUTS_DIR.split("outputs/", 1)[1]
+        self.file_path = static(os.path.join("outputs", server_path, "readnoise_monitor", "data", self.ins_ap))
 
         self.plot_readnoise_amplifers()
         self.plot_readnoise_difference_image()

From 75de14516a67c42adb4984957631736317ee4521 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 18 Dec 2023 10:18:46 -0500
Subject: [PATCH 249/256] Changing variable name to make it more descriptive

---
 .../apps/jwql/monitor_pages/monitor_readnoise_bokeh.py        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
index 814ff7239..41ca929b9 100644
--- a/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
+++ b/jwql/website/apps/jwql/monitor_pages/monitor_readnoise_bokeh.py
@@ -116,8 +116,8 @@ def __init__(self, instrument, aperture):
         self.db = ReadnoiseMonitorData(self.instrument, self.aperture)
 
         # Use outputs directory to obtain server name in path.
-        server_path = OUTPUTS_DIR.split("outputs/", 1)[1]
-        self.file_path = static(os.path.join("outputs", server_path, "readnoise_monitor", "data", self.ins_ap))
+        server_name = OUTPUTS_DIR.split("outputs/", 1)[1]
+        self.file_path = static(os.path.join("outputs", server_name, "readnoise_monitor", "data", self.ins_ap))
 
         self.plot_readnoise_amplifers()
         self.plot_readnoise_difference_image()

From 8087142e4cf4e2591ff04a6187d0834145492528 Mon Sep 17 00:00:00 2001
From: Bradley Sappington <bsappington@stsci.edu>
Date: Mon, 18 Dec 2023 10:56:23 -0500
Subject: [PATCH 250/256] pep8 fixes

---
 jwql/website/apps/jwql/data_containers.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
index 4958451c9..fcd5db35b 100644
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -407,9 +407,9 @@ def get_available_suffixes(all_suffixes, return_untracked=True):
     for poss_suffix in EXPOSURE_PAGE_SUFFIX_ORDER:
         if 'crf' not in poss_suffix:
             if (poss_suffix in all_suffixes
-                and poss_suffix not in suffixes):
-                    suffixes.append(poss_suffix)
-                    untracked_suffixes.remove(poss_suffix)
+            and poss_suffix not in suffixes):
+                suffixes.append(poss_suffix)
+                untracked_suffixes.remove(poss_suffix)
         else:
             # EXPOSURE_PAGE_SUFFIX_ORDER contains crf and crfints,
             # but the actual suffixes in the data will be e.g. o001_crf,
@@ -417,8 +417,8 @@ def get_available_suffixes(all_suffixes, return_untracked=True):
             # So in this case, we strip the e.g. o001 from the
             # suffixes and check which list elements match.
             for image_suffix in all_suffixes:
-                if (image_suffix.endswith(poss_suffix) and
-                    image_suffix not in suffixes):
+                if (image_suffix.endswith(poss_suffix)
+                and image_suffix not in suffixes):
                     suffixes.append(image_suffix)
                     untracked_suffixes.remove(image_suffix)
 
@@ -1399,7 +1399,7 @@ def get_proposals_by_category(instrument):
 
     service = "Mast.Jwst.Filtered.{}".format(instrument)
     params = {"columns": "program, category",
-              "filters": [{'paramName':'instrume', 'values':[instrument]}]}
+              "filters": [{'paramName': 'instrume', 'values': [instrument]}]}
     response = Mast.service_request_async(service, params)
     results = response[0].json()['data']
 
@@ -2086,7 +2086,7 @@ def thumbnails_query_ajax(rootnames):
     proposals = [data_dict['file_data'][rootname]['filename_dict']['program_id'] for
                  rootname in list(data_dict['file_data'].keys())]
     visits = [data_dict['file_data'][rootname]['filename_dict']['visit'] for
-                 rootname in list(data_dict['file_data'].keys())]
+              rootname in list(data_dict['file_data'].keys())]
 
     dropdown_menus = {'instrument': instruments,
                       'detector': sorted(detectors),

From 4c57c27ca0465f25ec84da790aea20facfc508d5 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 18 Dec 2023 11:28:06 -0500
Subject: [PATCH 251/256] updating static file path for claw monitor

---
 jwql/website/apps/jwql/monitor_views.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index e750d461a..49d1f257a 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -34,6 +34,7 @@
 from bokeh.resources import CDN, INLINE
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
+from django.templatetags.static import static
 import json
 import pandas as pd
 
@@ -70,7 +71,8 @@ def background_monitor(request):
     template = "background_monitor.html"
 
     # Get the background trending filters to display
-    output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
+    server_name = get_config()['outputs'].split("outputs/", 1)[1]
+    output_dir_bkg = static(os.path.join("outputs", server_name, "claw_monitor", "backgrounds"))
     fltrs = ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']
     bkg_plots = [os.path.join(output_dir_bkg, '{}_backgrounds.png'.format(fltr)) for fltr in fltrs]
 
@@ -156,7 +158,10 @@ def claw_monitor(request):
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
     recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 1000]))  # todo change from 1000 to 10
-    output_dir_claws = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+    
+    server_name = get_config()['outputs'].split("outputs/", 1)[1]
+    output_dir_claws = static(os.path.join("outputs", server_name, "claw_monitor", "claw_stacks"))
+
     claw_stacks = [os.path.join(output_dir_claws, filename) for filename in recent_files]
 
     context = {

From 04ba4ba518c311ceb6cbdeeb4e3c0089535a89b3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 18 Dec 2023 16:39:59 +0000
Subject: [PATCH 252/256] Bump scipy from 1.9.3 to 1.11.4

Bumps [scipy](https://github.com/scipy/scipy) from 1.9.3 to 1.11.4.
- [Release notes](https://github.com/scipy/scipy/releases)
- [Commits](https://github.com/scipy/scipy/compare/v1.9.3...v1.11.4)

---
updated-dependencies:
- dependency-name: scipy
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 7ec4993b5..55003018b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -26,7 +26,7 @@ pyvo==1.4.2
 pyyaml==6.0.1
 redis==5.0.0
 ruff==0.0.292
-scipy==1.9.3
+scipy==1.11.4
 selenium==4.13.0
 setuptools==68.2.2
 sphinx==7.2.6

From 31a3416bfc4b8fd8ef2f01e89025bd6b0da8923e Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Mon, 18 Dec 2023 11:45:48 -0500
Subject: [PATCH 253/256] removed testing statements

---
 jwql/instrument_monitors/nircam_monitors/claw_monitor.py | 4 +---
 jwql/website/apps/jwql/monitor_views.py                  | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 72244aad3..72f45021f 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -340,7 +340,6 @@ def run(self):
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
-        self.query_start_mjd, self.query_end_mjd = 59715.28951771492, 59715.29771992559  # todo remove
         mast_table = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(mast_table), self.query_start_mjd, self.query_end_mjd))
 
@@ -357,8 +356,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir_claws, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            #self.files = np.array([filesystem_path(row['filename']) for row in mast_table_combo])  # todo uncomment?
-            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in mast_table_combo]) # todo remove
+            self.files = np.array([filesystem_path(row['filename']) for row in mast_table_combo])
             self.detectors = np.array(mast_table_combo['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index 49d1f257a..5b8607ac1 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -157,7 +157,7 @@ def claw_monitor(request):
     # Get all recent claw stack images from the last 10 days
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
-    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 1000]))  # todo change from 1000 to 10
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 10]))
     
     server_name = get_config()['outputs'].split("outputs/", 1)[1]
     output_dir_claws = static(os.path.join("outputs", server_name, "claw_monitor", "claw_stacks"))

From f5e3af1b623ac29fe4a979621f91dfa979251172 Mon Sep 17 00:00:00 2001
From: Bradley Sappington
 <101193271+BradleySappington@users.noreply.github.com>
Date: Mon, 18 Dec 2023 12:06:12 -0500
Subject: [PATCH 254/256] Update requirements.txt

Include all current dependabot updates
---
 requirements.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 55003018b..5f601fddc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ bandit==1.7.5
 beautifulsoup4==4.12.2
 bokeh==2.4.3
 celery==5.3.4
-cryptography==41.0.6
+cryptography==41.0.7
 django==4.2.5
 inflection==0.5.1
 ipython==8.16.1
@@ -25,16 +25,16 @@ pytest-mock==3.11.1
 pyvo==1.4.2
 pyyaml==6.0.1
 redis==5.0.0
-ruff==0.0.292
+ruff==0.1.6
 scipy==1.11.4
 selenium==4.13.0
 setuptools==68.2.2
 sphinx==7.2.6
 sphinx_rtd_theme==1.3.0
-sqlalchemy==2.0.21
+sqlalchemy==2.0.23
 stdatamodels==1.8.3
 stsci_rtd_theme==1.0.0
 twine==4.0.2
-vine==5.0.0
+vine==5.1.0
 wtforms==3.0.1
 git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles

From bc497280c460835b67c3734735fc2f38166fd834 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Mon, 18 Dec 2023 15:36:30 -0500
Subject: [PATCH 255/256] skipping claw stacks that are missing cal files

---
 .../nircam_monitors/claw_monitor.py                  | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 72f45021f..c92dde3e2 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -356,14 +356,20 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir_claws, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([filesystem_path(row['filename']) for row in mast_table_combo])
+            existing_files = []
+            for row in mast_table_combo:
+                try:
+                    existing_files.append(filesystem_path(row['filename']))
+                except:
+                    pass
+            self.files = np.array(existing_files)
             self.detectors = np.array(mast_table_combo['detector'])
-            if not os.path.exists(self.outfile):
+            if (not os.path.exists(self.outfile)) & (len(existing_files) == len(mast_table_combo)):
                 logging.info('Working on {}'.format(self.outfile))
                 self.process()
                 monitor_run = True
             else:
-                logging.info('{} already exists'.format(self.outfile))
+                logging.info('{} already exists or is missing cal files ({}/{} files found).'.format(self.outfile, len(existing_files), len(mast_table_combo)))
 
         # Update the background trending plots, if any new data exists
         if len(mast_table) > 0:

From 3c27420a1dcc501efbdecee46788f5313c4886c5 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Wed, 20 Dec 2023 10:15:49 -0500
Subject: [PATCH 256/256] Updating CHANGES

---
 CHANGES.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/CHANGES.rst b/CHANGES.rst
index 6280f83ca..1c7b7a96b 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -1,5 +1,18 @@
 ## What's Changed
 
+1.2.1 (2023-12-20)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Visit Filter on query and archive thumbnails pages by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1412
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- NIRCam Claw Monitor by @bsunnquist in https://github.com/spacetelescope/jwql/pull/1152
+- Stop storing absolute paths in the database by @york-stsci in https://github.com/spacetelescope/jwql/pull/1394
+
+
 1.2.0 (2023-11-21)
 ==================