From 952ad17e3e9a72e8802973f82ce770ede7c923e0 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Fri, 6 Jan 2023 10:38:22 -0600
Subject: [PATCH 01/22] initial commit of claw monitor

---
 .../nircam_monitors/claw_monitor.py           | 64 +++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100644 jwql/instrument_monitors/nircam_monitors/claw_monitor.py

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
new file mode 100644
index 000000000..c11ca57f9
--- /dev/null
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -0,0 +1,64 @@
+#! /usr/bin/env python
+
+"""This module contains code for the claw monitor.
+
+Author
+------
+    - Ben Sunnquist
+
+Use
+---
+    This module can be used from the command line as such:
+
+    ::
+
+        python claw_monitor.py
+"""
+
+import logging
+import os
+
+from astropy.io import fits
+from astropy.time import Time
+from astropy.visualization import ZScaleInterval
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import numpy as np
+
+from jwql.utils import monitor_utils
+from jwql.utils.logging_functions import log_info, log_fail
+
+
+class ClawMonitor():
+    """Class for executing the claw monitor.
+    """
+
+    def __init__(self):
+        """Initialize an instance of the ``ClawMonitor`` class.
+        """
+
+    def process(self, file_list):
+        """The main method for processing.  See module docstrings for further details.
+        """
+
+    #@log_fail  # ben uncomment
+    #@log_info  # ben uncomment
+    def run(self):
+        """The main method.  See module docstrings for further details."""
+
+        logging.info('Begin logging for claw_monitor')
+
+
+        logging.info('Claw Monitor completed successfully.')
+
+
+if __name__ == '__main__':
+
+    module = os.path.basename(__file__).strip('.py')
+    #start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # ben uncomment
+
+    monitor = ClawMonitor()
+    monitor.run()
+
+    #monitor_utils.update_monitor_table(module, start_time, log_file)   # ben uncomment

From 402d7638651cb12ffda85680e55d649d78f02bcf Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Fri, 6 Jan 2023 15:41:52 -0600
Subject: [PATCH 02/22] added mast query, initial stacking and plotting

---
 .../nircam_monitors/claw_monitor.py           | 97 +++++++++++++++++--
 1 file changed, 91 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index c11ca57f9..db829c14f 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -21,6 +21,7 @@
 from astropy.io import fits
 from astropy.time import Time
 from astropy.visualization import ZScaleInterval
+from astroquery.mast import Mast
 import matplotlib
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
@@ -38,17 +39,101 @@ def __init__(self):
         """Initialize an instance of the ``ClawMonitor`` class.
         """
 
-    def process(self, file_list):
+    def process(self):
         """The main method for processing.  See module docstrings for further details.
         """
 
-    #@log_fail  # ben uncomment
-    #@log_info  # ben uncomment
+        # Get detector order and plot settings, depending on the wavelength channel
+        if self.wv == 'SW':
+            detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']  # in on-sky order, don't change order
+            cols, rows = 5, 2
+            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,1,1,.1])
+            fig = plt.figure(figsize=(40, 20))
+            cbar_fs = 20
+            fs = 30
+        else:
+            detectors_to_run = ['NRCALONG', 'NRCBLONG']
+            cols, rows = 3, 1
+            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,.1])
+            fig = plt.figure(figsize=(20, 10))
+            cbar_fs = 15
+            fs = 25
+        
+        # Make source-masked, median-stack of each detector's images, and add them to the plot
+        for det in detectors_to_run:
+            files = self.files[self.detectors == det]
+            print(det)
+            print(files)
+            print('------')
+
+    def query_mast(self):
+        """Query MAST for new nircam full-frame imaging data.
+
+        Returns
+        -------
+        t : astropy.table.table.Table
+            A table summarizing the new nircam imaging data.
+        """
+
+        server = "https://mast.stsci.edu"
+        JwstObs = Mast()
+        JwstObs._portal_api_connection.MAST_REQUEST_URL = server + "/portal_jwst/Mashup/Mashup.asmx/invoke"
+        JwstObs._portal_api_connection.MAST_DOWNLOAD_URL = server + "/jwst/api/v0.1/download/file"
+        JwstObs._portal_api_connection.COLUMNS_CONFIG_URL = server + "/portal_jwst/Mashup/Mashup.asmx/columnsconfig"
+        JwstObs._portal_api_connection.MAST_BUNDLE_URL = server + "/jwst/api/v0.1/download/bundle"
+        service = 'Mast.Jwst.Filtered.Nircam'
+        FIELDS = ['filename','program', 'observtn','category','instrume', 'productLevel', 'filter', 
+                  'pupil', 'subarray', 'detector','datamodl','date_beg_mjd', 'effexptm']
+        params = {"columns":",".join(FIELDS),
+                "filters":[
+                {"paramName":"pupil","values":['CLEAR','F162M','F164N','F323N','F405N','F466N','F470N']},
+                {"paramName":"exp_type","values":['NRC_IMAGE']},
+                {"paramName":"datamodl", "values":['ImageModel']},  # exclude calints, which are cubemodel
+                {"paramName":"productLevel", "values":['2b']},  # i.e. cal.fits
+                {"paramName":"subarray", "values":['FULL']},
+                ]
+                }
+        t = JwstObs.service_request(service, params)
+        t = t[t['date_beg_mjd']>self.query_start_mjd]
+        t.sort('date_beg_mjd')
+        filetypes = np.array([row['filename'].split('_')[-1].replace('.fits','') for row in t])
+        t = t[filetypes=='cal']  # only want cal.fits files, no e.g. i2d.fits
+
+        return t
+
+    #@log_fail  # todo uncomment
+    #@log_info  # todo uncomment
     def run(self):
         """The main method.  See module docstrings for further details."""
 
         logging.info('Begin logging for claw_monitor')
-
+        self.output_dir = '/Users/bsunnquist/Downloads/'  # todo change this to os.path.join(get_config()['outputs'], 'claw_monitor')
+        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
+
+        # Query MAST for new imaging data from the last 3 days
+        self.query_end_mjd = Time.now().mjd
+        self.query_start_mjd = self.query_end_mjd - 3
+        print(self.query_end_mjd, self.query_start_mjd)
+        t = self.query_mast()
+        print(t)
+
+        # Create observation-level median stacks for each filter/pupil combo, in pixel-space
+        combos = np.array(sorted(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t]))
+        t['combos'] = combos
+        for combo in np.unique(combos)[0:2]:  # todo take off 0:2
+            tt = t[t['combos']==combo]
+            if 'long' in tt['filename'][0]:
+                self.wv = 'LW'
+            else:
+                self.wv = 'SW'
+            self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
+            self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
+            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            self.detectors = np.array(tt['detector'])
+            if not os.path.exists(self.outfile):
+                self.process()
+            else:
+                print('{} already exists'.format(self.outfile))
 
         logging.info('Claw Monitor completed successfully.')
 
@@ -56,9 +141,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    #start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # ben uncomment
+    #start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # todo uncomment
 
     monitor = ClawMonitor()
     monitor.run()
 
-    #monitor_utils.update_monitor_table(module, start_time, log_file)   # ben uncomment
+    #monitor_utils.update_monitor_table(module, start_time, log_file)   # todo uncomment

From 0ab25da8e3d4ed49205e334d2cf6d7b98e291af5 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Tue, 10 Jan 2023 13:30:13 -0600
Subject: [PATCH 03/22] stacks and plots now working on test case

---
 .../nircam_monitors/claw_monitor.py           | 80 +++++++++++++++++--
 1 file changed, 74 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index db829c14f..eb0d993a0 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -18,7 +18,9 @@
 import logging
 import os
 
+from astropy.convolution import Gaussian2DKernel, convolve
 from astropy.io import fits
+from astropy.stats import gaussian_fwhm_to_sigma
 from astropy.time import Time
 from astropy.visualization import ZScaleInterval
 from astroquery.mast import Mast
@@ -26,6 +28,7 @@
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import numpy as np
+from photutils import detect_sources, detect_threshold
 
 from jwql.utils import monitor_utils
 from jwql.utils.logging_functions import log_info, log_fail
@@ -59,12 +62,73 @@ def process(self):
             cbar_fs = 15
             fs = 25
         
-        # Make source-masked, median-stack of each detector's images, and add them to the plot
-        for det in detectors_to_run:
+        # Make source-masked, median-stack of each detector's images
+        print(self.outfile)
+        print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
+        found_scale = False
+        for i,det in enumerate(detectors_to_run):
             files = self.files[self.detectors == det]
+            # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
+            files = [f for f in files if os.path.exists(f)][0:2]  # todo change index value?
+            stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
             print(files)
             print('------')
+            for n,f in enumerate(files):
+                # Get pointing and other info from first image
+                if n == 0:
+                    h = fits.open(f)
+                    obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
+                    obs_start_mjd = h[0].header['EXPSTART']
+                    targname, ra_v1, dec_v1, pa_v3 = h[0].header['TARGPROP'], h[1].header['RA_V1'], h[1].header['DEC_V1'], h[1].header['PA_V3']
+                    h.close()
+
+                # Make source segmap, and add the masked data to the stack
+                data = fits.getdata(f, 'SCI')
+                threshold = detect_threshold(data, 1.25)
+                sigma = 3.0 * gaussian_fwhm_to_sigma  # FWHM = 3.
+                kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
+                kernel.normalize()
+                data_conv = convolve(data, kernel)
+                segmap = detect_sources(data_conv, threshold, npixels=3)
+                segmap = segmap.data
+                stack[n] = np.ma.masked_array(data, mask=segmap!=0)
+
+            # Make the normalized skyflat for this detector
+            skyflat = np.ma.median(stack, axis=0)
+            skyflat = skyflat.filled(fill_value=np.nan)
+            skyflat = skyflat / np.nanmedian(skyflat)
+            skyflat[~np.isfinite(skyflat)] = 1  # fill missing values
+
+            # Add the skyflat for this detector to the plot
+            if (self.wv=='SW') & (i>3):  # skip colobar axis
+                idx = i+1
+            else:
+                idx = i
+            ax = fig.add_subplot(grid[idx])
+            if len(skyflat[skyflat!=1])==0:
+                ax.set_title('N/A', fontsize=fs)
+                ax.imshow(skyflat, cmap='coolwarm', vmin=999, vmax=999, origin='lower')
+            elif (len(skyflat[skyflat!=1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
+                z = ZScaleInterval()
+                vmin, vmax = z.get_limits(skyflat)
+                found_scale = True
+                ax.set_title(det, fontsize=fs)
+                im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
+            else:
+                ax.set_title(det, fontsize=fs)
+                im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
+            ax.axes.get_xaxis().set_ticks([])
+            ax.axes.get_yaxis().set_ticks([])
+        
+        # Add colobar, save figure if any detector stacks exist
+        if found_scale:
+            cax = fig.add_subplot(grid[0:rows, cols-1:cols])
+            cbar = fig.colorbar(im, cax=cax, orientation='vertical')
+            cbar.ax.tick_params(labelsize=cbar_fs)
+            fig.savefig(self.outfile, dpi=100, bbox_inches='tight')
+        fig.clf()
+        plt.close()
 
     def query_mast(self):
         """Query MAST for new nircam full-frame imaging data.
@@ -94,7 +158,7 @@ def query_mast(self):
                 ]
                 }
         t = JwstObs.service_request(service, params)
-        t = t[t['date_beg_mjd']>self.query_start_mjd]
+        t = t[(t['date_beg_mjd']>self.query_start_mjd) & (t['date_beg_mjd']<self.query_end_mjd)]
         t.sort('date_beg_mjd')
         filetypes = np.array([row['filename'].split('_')[-1].replace('.fits','') for row in t])
         t = t[filetypes=='cal']  # only want cal.fits files, no e.g. i2d.fits
@@ -107,21 +171,24 @@ def run(self):
         """The main method.  See module docstrings for further details."""
 
         logging.info('Begin logging for claw_monitor')
-        self.output_dir = '/Users/bsunnquist/Downloads/'  # todo change this to os.path.join(get_config()['outputs'], 'claw_monitor')
+        self.output_dir = '/Users/bsunnquist/Documents/nircam/claw_monitor_testing/'  # todo change this to os.path.join(get_config()['outputs'], 'claw_monitor')
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
 
         # Query MAST for new imaging data from the last 3 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 3
-        print(self.query_end_mjd, self.query_start_mjd)
+        self.query_end_mjd, self.query_start_mjd = 59878.986, 59878.934  # todo remove these test datess
+        print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
         print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
-        combos = np.array(sorted(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t]))
+        combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
+        print(np.unique(combos))
         t['combos'] = combos
         for combo in np.unique(combos)[0:2]:  # todo take off 0:2
             tt = t[t['combos']==combo]
+            print(tt)
             if 'long' in tt['filename'][0]:
                 self.wv = 'LW'
             else:
@@ -129,6 +196,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
             self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 self.process()

From 1a87dd260766ddfa36be39323984e300b878e8ab Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 11 Jan 2023 16:22:30 -0600
Subject: [PATCH 04/22] adding pointing and other info to plots

---
 jwql/instrument_monitors/nircam_monitors/claw_monitor.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index eb0d993a0..ab1c10bfc 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -30,8 +30,8 @@
 import numpy as np
 from photutils import detect_sources, detect_threshold
 
-from jwql.utils import monitor_utils
-from jwql.utils.logging_functions import log_info, log_fail
+#from jwql.utils import monitor_utils  # todo uncomment
+#from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
 
 
 class ClawMonitor():
@@ -59,8 +59,8 @@ def process(self):
             cols, rows = 3, 1
             grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,.1])
             fig = plt.figure(figsize=(20, 10))
-            cbar_fs = 15
-            fs = 25
+            cbar_fs = 10
+            fs = 20
         
         # Make source-masked, median-stack of each detector's images
         print(self.outfile)
@@ -123,6 +123,7 @@ def process(self):
         
         # Add colobar, save figure if any detector stacks exist
         if found_scale:
+            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
             cax = fig.add_subplot(grid[0:rows, cols-1:cols])
             cbar = fig.colorbar(im, cax=cax, orientation='vertical')
             cbar.ax.tick_params(labelsize=cbar_fs)

From 9af57328c240ecec34e836ac12b0bf38cc3ce102 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 12 Oct 2023 11:59:54 -0400
Subject: [PATCH 05/22] Adding nircam claw databases

---
 jwql/database/database_interface.py             |  7 +++++--
 .../nircam/nircam_claw_query_history.txt        |  5 +++++
 .../nircam/nircam_claw_stats.txt                | 17 +++++++++++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)
 create mode 100644 jwql/database/monitor_table_definitions/nircam/nircam_claw_query_history.txt
 create mode 100644 jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt

diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 8f660209c..47800f647 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -509,6 +509,8 @@ class : obj
 NIRSpecCosmicRayStats = monitor_orm_factory('nirspec_cosmic_ray_stats')
 NIRSpecTAQueryHistory = monitor_orm_factory('nirspec_ta_query_history')
 NIRSpecTAStats = monitor_orm_factory('nirspec_ta_stats')
+NIRCamClawQueryHistory = monitor_orm_factory('nircam_claw_query_history')
+NIRCamClawStats = monitor_orm_factory('nircam_claw_stats')
 
 INSTRUMENT_TABLES = {
     'nircam': [NIRCamDarkQueryHistory, NIRCamDarkPixelStats, NIRCamDarkDarkCurrent,
@@ -516,7 +518,7 @@ class : obj
                NIRCamBadPixelStats, NIRCamReadnoiseQueryHistory, NIRCamReadnoiseStats,
                NIRCamAnomaly, NIRCamCosmicRayQueryHistory, NIRCamCosmicRayStats,
                NIRCamEDBDailyStats, NIRCamEDBBlockStats, NIRCamEDBTimeIntervalStats,
-               NIRCamEDBEveryChangeStats],
+               NIRCamEDBEveryChangeStats, NIRCamClawQueryHistory, NIRCamClawStats],
     'niriss': [NIRISSDarkQueryHistory, NIRISSDarkPixelStats, NIRISSDarkDarkCurrent,
                NIRISSBiasQueryHistory, NIRISSBiasStats, NIRISSBadPixelQueryHistory,
                NIRISSBadPixelStats, NIRISSReadnoiseQueryHistory, NIRISSReadnoiseStats,
@@ -570,7 +572,8 @@ class : obj
             MIRIEDBBlockStats, MIRIEDBTimeIntervalStats, MIRIEDBEveryChangeStats,
             NIRSpecEDBDailyStats, NIRSpecEDBBlockStats, NIRSpecEDBTimeIntervalStats,
             NIRSpecEDBEveryChangeStats, FGSEDBDailyStats, FGSEDBBlockStats,
-            FGSEDBTimeIntervalStats, FGSEDBEveryChangeStats]}
+            FGSEDBTimeIntervalStats, FGSEDBEveryChangeStats],
+    'claw': [NIRCamClawQueryHistory, NIRCamClawStats],}
 
 if __name__ == '__main__':
     base.metadata.create_all(engine)
diff --git a/jwql/database/monitor_table_definitions/nircam/nircam_claw_query_history.txt b/jwql/database/monitor_table_definitions/nircam/nircam_claw_query_history.txt
new file mode 100644
index 000000000..d116f3c2f
--- /dev/null
+++ b/jwql/database/monitor_table_definitions/nircam/nircam_claw_query_history.txt
@@ -0,0 +1,5 @@
+INSTRUMENT, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+RUN_MONITOR, bool
+ENTRY_DATE, datetime
\ No newline at end of file
diff --git a/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt b/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
new file mode 100644
index 000000000..f6fc1131a
--- /dev/null
+++ b/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
@@ -0,0 +1,17 @@
+FILENAME, string
+PROPOSAL, string
+OBS, string
+DETECTOR, string
+FILTER, string
+PUPIL, string
+EXPSTART, string
+EXPSTART_MJD, float
+EFFEXPTM, float
+RA, float
+DEC, float
+PA_V3, float
+MEAN, float
+MEDIAN, float
+STDDEV, float
+FRAC_MASKED, float
+SKYFLAT_FILENAME, string
\ No newline at end of file

From d6be2ccc12f04d884a0e76010ff49ab523be37c3 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Mon, 16 Oct 2023 17:00:38 -0400
Subject: [PATCH 06/22] Updates after spot checking new data run

---
 .../nircam_monitors/claw_monitor.py           | 58 +++++++++++++------
 1 file changed, 39 insertions(+), 19 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index ab1c10bfc..ef1c508ca 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -20,7 +20,7 @@
 
 from astropy.convolution import Gaussian2DKernel, convolve
 from astropy.io import fits
-from astropy.stats import gaussian_fwhm_to_sigma
+from astropy.stats import gaussian_fwhm_to_sigma, sigma_clipped_stats
 from astropy.time import Time
 from astropy.visualization import ZScaleInterval
 from astroquery.mast import Mast
@@ -28,10 +28,13 @@
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import numpy as np
-from photutils import detect_sources, detect_threshold
+from photutils.segmentation import detect_sources, detect_threshold
 
-#from jwql.utils import monitor_utils  # todo uncomment
-#from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
+from jwql.database.database_interface import session
+from jwql.database.database_interface import NIRCamClawQueryHistory, NIRCamClawStats
+from jwql.utils import monitor_utils  # todo uncomment
+from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
 
 class ClawMonitor():
@@ -63,17 +66,20 @@ def process(self):
             fs = 20
         
         # Make source-masked, median-stack of each detector's images
+        logging.info('Working on claw stack: {}'.format(self.outfile))
         print(self.outfile)
         print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
         found_scale = False
         for i,det in enumerate(detectors_to_run):
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
-            files = [f for f in files if os.path.exists(f)][0:2]  # todo change index value?
+            files = [f for f in files if os.path.exists(f)][0:5]  # todo change index value?
             stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
             print(files)
             print('------')
+            backgrounds = []
+            fraction_masked = []
             for n,f in enumerate(files):
                 # Get pointing and other info from first image
                 if n == 0:
@@ -81,18 +87,23 @@ def process(self):
                     obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
                     obs_start_mjd = h[0].header['EXPSTART']
                     targname, ra_v1, dec_v1, pa_v3 = h[0].header['TARGPROP'], h[1].header['RA_V1'], h[1].header['DEC_V1'], h[1].header['PA_V3']
+                    print(obs_start, obs_start_mjd)
+                    print(targname, ra_v1, dec_v1, pa_v3)
                     h.close()
 
-                # Make source segmap, and add the masked data to the stack
+                # Make source segmap, add the masked data to the stack, and get background stats
                 data = fits.getdata(f, 'SCI')
-                threshold = detect_threshold(data, 1.25)
+                threshold = detect_threshold(data, 1.0)
                 sigma = 3.0 * gaussian_fwhm_to_sigma  # FWHM = 3.
                 kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
                 kernel.normalize()
                 data_conv = convolve(data, kernel)
-                segmap = detect_sources(data_conv, threshold, npixels=3)
+                segmap = detect_sources(data_conv, threshold, npixels=6)
                 segmap = segmap.data
                 stack[n] = np.ma.masked_array(data, mask=segmap!=0)
+                fraction_masked.append(len(segmap[segmap!=0]) / (segmap.shape[0]*segmap.shape[1]))
+                mean, med, stddev = sigma_clipped_stats(data[segmap!=0])
+                backgrounds.append(med)
 
             # Make the normalized skyflat for this detector
             skyflat = np.ma.median(stack, axis=0)
@@ -130,6 +141,7 @@ def process(self):
             fig.savefig(self.outfile, dpi=100, bbox_inches='tight')
         fig.clf()
         plt.close()
+        logging.info('Claw stack complete: {}'.format(self.outfile))
 
     def query_mast(self):
         """Query MAST for new nircam full-frame imaging data.
@@ -166,30 +178,38 @@ def query_mast(self):
 
         return t
 
-    #@log_fail  # todo uncomment
-    #@log_info  # todo uncomment
+    @log_fail  # todo uncomment
+    @log_info  # todo uncomment
     def run(self):
         """The main method.  See module docstrings for further details."""
 
         logging.info('Begin logging for claw_monitor')
-        self.output_dir = '/Users/bsunnquist/Documents/nircam/claw_monitor_testing/'  # todo change this to os.path.join(get_config()['outputs'], 'claw_monitor')
+        self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+        ensure_dir_exists(self.output_dir)
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
+        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
 
         # Query MAST for new imaging data from the last 3 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 3
-        self.query_end_mjd, self.query_start_mjd = 59878.986, 59878.934  # todo remove these test datess
+        #self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
+        self.query_start_mjd, self.query_end_mjd = 60150, 60152  # todo remove
+        #self.query_start_mjd = 59985  # last run was may 25; todo remove
+
         print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
-        print(t)
+        #print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
+        n_combos = len(np.unique(combos))
+        print('unique combos:')
         print(np.unique(combos))
         t['combos'] = combos
-        for combo in np.unique(combos)[0:2]:  # todo take off 0:2
+        for nnn,combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
+            print(combo, '{}/{}'.format(nnn,n_combos))
             tt = t[t['combos']==combo]
-            print(tt)
+            #print(tt)
             if 'long' in tt['filename'][0]:
                 self.wv = 'LW'
             else:
@@ -197,12 +217,12 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
             self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
-            print(self.files)
+            #print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 self.process()
             else:
-                print('{} already exists'.format(self.outfile))
+                logging.info('{} already exists'.format(self.outfile))
 
         logging.info('Claw Monitor completed successfully.')
 
@@ -210,9 +230,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    #start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # todo uncomment
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # todo uncomment
 
     monitor = ClawMonitor()
     monitor.run()
 
-    #monitor_utils.update_monitor_table(module, start_time, log_file)   # todo uncomment
+    monitor_utils.update_monitor_table(module, start_time, log_file)   # todo uncomment

From ab2d8036a9a0f42426d07e120d781eff1a16f5da Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Tue, 17 Oct 2023 12:22:41 -0400
Subject: [PATCH 07/22] adding new columns and other database issues

---
 .../nircam/nircam_claw_stats.txt              |  3 +-
 .../nircam_monitors/claw_monitor.py           | 67 +++++++++++++++----
 2 files changed, 55 insertions(+), 15 deletions(-)

diff --git a/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt b/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
index f6fc1131a..0730d1e3f 100644
--- a/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
+++ b/jwql/database/monitor_table_definitions/nircam/nircam_claw_stats.txt
@@ -14,4 +14,5 @@ MEAN, float
 MEDIAN, float
 STDDEV, float
 FRAC_MASKED, float
-SKYFLAT_FILENAME, string
\ No newline at end of file
+SKYFLAT_FILENAME, string
+ENTRY_DATE, datetime
\ No newline at end of file
diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index ef1c508ca..a6957d370 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -15,6 +15,7 @@
         python claw_monitor.py
 """
 
+import datetime
 import logging
 import os
 
@@ -30,7 +31,7 @@
 import numpy as np
 from photutils.segmentation import detect_sources, detect_threshold
 
-from jwql.database.database_interface import session
+from jwql.database.database_interface import session, engine
 from jwql.database.database_interface import NIRCamClawQueryHistory, NIRCamClawStats
 from jwql.utils import monitor_utils  # todo uncomment
 from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
@@ -73,26 +74,22 @@ def process(self):
         for i,det in enumerate(detectors_to_run):
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
-            files = [f for f in files if os.path.exists(f)][0:5]  # todo change index value?
+            files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
             stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
             print(files)
             print('------')
-            backgrounds = []
-            fraction_masked = []
             for n,f in enumerate(files):
-                # Get pointing and other info from first image
+                h = fits.open(f)
+                
+                # Get plot label info from first image
                 if n == 0:
-                    h = fits.open(f)
                     obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
-                    obs_start_mjd = h[0].header['EXPSTART']
-                    targname, ra_v1, dec_v1, pa_v3 = h[0].header['TARGPROP'], h[1].header['RA_V1'], h[1].header['DEC_V1'], h[1].header['PA_V3']
-                    print(obs_start, obs_start_mjd)
-                    print(targname, ra_v1, dec_v1, pa_v3)
-                    h.close()
+                    pa_v3 = h[1].header['PA_V3']
 
                 # Make source segmap, add the masked data to the stack, and get background stats
-                data = fits.getdata(f, 'SCI')
+                data = h['SCI'].data
+                dq = h['DQ'].data
                 threshold = detect_threshold(data, 1.0)
                 sigma = 3.0 * gaussian_fwhm_to_sigma  # FWHM = 3.
                 kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
@@ -100,10 +97,35 @@ def process(self):
                 data_conv = convolve(data, kernel)
                 segmap = detect_sources(data_conv, threshold, npixels=6)
                 segmap = segmap.data
+                segmap[dq&1!=0] = 1  # flag DO_NOT_USE pixels
                 stack[n] = np.ma.masked_array(data, mask=segmap!=0)
-                fraction_masked.append(len(segmap[segmap!=0]) / (segmap.shape[0]*segmap.shape[1]))
                 mean, med, stddev = sigma_clipped_stats(data[segmap!=0])
-                backgrounds.append(med)
+                
+                # Add this file's stats to the claw database table.
+                # Can't insert values with numpy.float32 datatypes into database
+                # so need to change the datatypes of these values.
+                claw_db_entry = {'filename': os.path.basename(f),
+                                 'proposal': self.proposal,
+                                 'obs': self.obs,
+                                 'detector': det,
+                                 'filter': self.fltr,
+                                 'pupil': self.pupil,
+                                 'expstart': '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS']),
+                                 'expstart_mjd': h[0].header['EXPSTART'],
+                                 'effexptm': h[0].header['EFFEXPTM'],
+                                 'ra': h[1].header['RA_V1'],
+                                 'dec': h[1].header['DEC_V1'],
+                                 'pa_v3': h[1].header['PA_V3'],
+                                 'mean': float(mean),
+                                 'median': float(med),
+                                 'stddev': float(stddev),
+                                 'frac_masked': len(segmap[segmap!=0]) / (segmap.shape[0]*segmap.shape[1]),
+                                 'skyflat_filename': os.path.basename(self.outfile),
+                                 'entry_date': datetime.datetime.now()
+                                }
+                with engine.begin() as connection:
+                    connection.execute(self.stats_table.__table__.insert(), claw_db_entry)
+                h.close()
 
             # Make the normalized skyflat for this detector
             skyflat = np.ma.median(stack, axis=0)
@@ -126,9 +148,11 @@ def process(self):
                 found_scale = True
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
+                print(det, vmin, vmax)
             else:
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
+                print(det, vmin, vmax)
             ax.axes.get_xaxis().set_ticks([])
             ax.axes.get_yaxis().set_ticks([])
         
@@ -189,6 +213,10 @@ def run(self):
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
         self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
 
+        # Get the claw monitor database tables
+        self.query_table = eval('NIRCamClawQueryHistory')
+        self.stats_table = eval('NIRCamClawStats')
+
         # Query MAST for new imaging data from the last 3 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 3
@@ -206,6 +234,7 @@ def run(self):
         print('unique combos:')
         print(np.unique(combos))
         t['combos'] = combos
+        monitor_run = False
         for nnn,combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
             print(combo, '{}/{}'.format(nnn,n_combos))
             tt = t[t['combos']==combo]
@@ -221,9 +250,19 @@ def run(self):
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 self.process()
+                monitor_run = True
             else:
                 logging.info('{} already exists'.format(self.outfile))
 
+        # Update the query history
+        new_entry = {'instrument': 'nircam',
+                     'start_time_mjd': self.query_start_mjd,
+                     'end_time_mjd': self.query_end_mjd,
+                     'run_monitor': monitor_run,
+                     'entry_date': datetime.datetime.now()}
+        with engine.begin() as connection:
+            connection.execute(self.query_table.__table__.insert(), new_entry)
+
         logging.info('Claw Monitor completed successfully.')
 
 

From d8219c73f7d10c35d0a9f3e2847bb3bd41963bd5 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Tue, 17 Oct 2023 16:36:11 -0400
Subject: [PATCH 08/22] claw monitor webpages and databases working

---
 .../nircam_monitors/claw_monitor.py           | 11 +++---
 jwql/utils/constants.py                       |  1 +
 jwql/website/apps/jwql/monitor_views.py       | 35 +++++++++++++++++++
 .../apps/jwql/templates/claw_monitor.html     | 25 +++++++++++++
 jwql/website/apps/jwql/urls.py                |  3 ++
 5 files changed, 70 insertions(+), 5 deletions(-)
 create mode 100644 jwql/website/apps/jwql/templates/claw_monitor.html

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index a6957d370..6edcafd38 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -67,11 +67,11 @@ def process(self):
             fs = 20
         
         # Make source-masked, median-stack of each detector's images
-        logging.info('Working on claw stack: {}'.format(self.outfile))
         print(self.outfile)
         print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
         found_scale = False
         for i,det in enumerate(detectors_to_run):
+            logging.info('Working on {}'.format(det))
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
             files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
@@ -80,6 +80,7 @@ def process(self):
             print(files)
             print('------')
             for n,f in enumerate(files):
+                logging.info('Working on: {}'.format(f))
                 h = fits.open(f)
                 
                 # Get plot label info from first image
@@ -99,7 +100,7 @@ def process(self):
                 segmap = segmap.data
                 segmap[dq&1!=0] = 1  # flag DO_NOT_USE pixels
                 stack[n] = np.ma.masked_array(data, mask=segmap!=0)
-                mean, med, stddev = sigma_clipped_stats(data[segmap!=0])
+                mean, med, stddev = sigma_clipped_stats(data[segmap==0])
                 
                 # Add this file's stats to the claw database table.
                 # Can't insert values with numpy.float32 datatypes into database
@@ -148,11 +149,9 @@ def process(self):
                 found_scale = True
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
-                print(det, vmin, vmax)
             else:
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
-                print(det, vmin, vmax)
             ax.axes.get_xaxis().set_ticks([])
             ax.axes.get_yaxis().set_ticks([])
         
@@ -165,7 +164,7 @@ def process(self):
             fig.savefig(self.outfile, dpi=100, bbox_inches='tight')
         fig.clf()
         plt.close()
-        logging.info('Claw stack complete: {}'.format(self.outfile))
+        logging.info('Claw stacks complete: {}'.format(self.outfile))
 
     def query_mast(self):
         """Query MAST for new nircam full-frame imaging data.
@@ -226,6 +225,7 @@ def run(self):
 
         print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
+        logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
         #print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
@@ -249,6 +249,7 @@ def run(self):
             #print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
+                logging.info('Working on {}'.format(self.outfile))
                 self.process()
                 monitor_run = True
             else:
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index dee4123d2..b5728a8c4 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -378,6 +378,7 @@
              ('Readnoise Monitor', '/miri/readnoise_monitor')],
     'nircam': [('Bad Pixel Monitor', '/nircam/bad_pixel_monitor'),
                ('Bias Monitor', '/nircam/bias_monitor'),
+               ('Claw Monitor', '/nircam/claw_monitor'),
                ('Cosmic Ray Monitor', '#'),
                ('Dark Current Monitor', '/nircam/dark_monitor'),
                ('EDB Telemetry Monitor', '/nircam/edb_monitor'),
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index b62f339de..f271e69b8 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -30,12 +30,16 @@
 
 import os
 
+from astropy.time import Time
 from bokeh.resources import CDN, INLINE
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
 import json
+import pandas as pd
 
 from . import bokeh_containers
+from jwql.database.database_interface import session
+from jwql.database.database_interface import NIRCamClawStats
 from jwql.website.apps.jwql import bokeh_containers
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.utils import get_config, get_base_url
@@ -100,6 +104,37 @@ def bias_monitor(request, inst):
     return render(request, template, context)
 
 
+def claw_monitor(request):
+    """Generate the NIRCam claw monitor page
+
+    Parameters
+    ----------
+    request : HttpRequest object
+        Incoming request from the webpage
+
+    Returns
+    -------
+    HttpResponse object
+        Outgoing response sent to the webpage
+    """
+
+    template = "claw_monitor.html"
+
+    # Get all recent claw stack images
+    query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
+    df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd']>Time.now().mjd-100]))  # todo change 100 to 10 days back?
+    claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
+
+    context = {
+        'inst': 'NIRCam',
+        'claw_stacks': claw_stacks
+        }
+
+    # Return a HTTP response with the template and dictionary of variables
+    return render(request, template, context)
+
+
 def cosmic_ray_monitor(request, inst):
     """Generate the cosmic ray monitor page for a given instrument
 
diff --git a/jwql/website/apps/jwql/templates/claw_monitor.html b/jwql/website/apps/jwql/templates/claw_monitor.html
new file mode 100644
index 000000000..f4af8d8be
--- /dev/null
+++ b/jwql/website/apps/jwql/templates/claw_monitor.html
@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+
+{% block preamble %}
+
+<title>{{ inst }} Claw Monitor - JWQL</title>
+
+{% endblock %}
+
+{% block content %}
+
+    <main role="main" class="container">
+
+		<h1>{{ inst }} Claw Monitor</h1>
+		<hr>
+
+		<div width=100%>
+			{% for claw_stack in claw_stacks %}
+				<img src={{ claw_stack | safe }} width="100%">
+				<p></p>
+			{% endfor %}
+		</div>
+
+	</main>
+
+{% endblock %}
\ No newline at end of file
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index 0b4420080..e483cff9d 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -58,6 +58,9 @@
     # Home
     path('', views.home, name='home'),
 
+    # NIRCam-specific views
+    path('nircam/claw_monitor/', monitor_views.claw_monitor, name='claw_monitor'), 
+
     # NIRSpec-specific views
     path('nirspec/msata_monitor/', monitor_views.msata_monitoring, name='msata_monitor'),
     path('nirspec/wata_monitor/', monitor_views.wata_monitoring, name='wata_monitor'),

From 8e9a1dc83c2cacac4cc3b40799d77b5f365e8622 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 18 Oct 2023 13:26:33 -0400
Subject: [PATCH 09/22] added background monitoring plots and webpages

---
 .../nircam_monitors/claw_monitor.py           | 148 ++++++++++++++++--
 jwql/utils/constants.py                       |   3 +-
 jwql/website/apps/jwql/monitor_views.py       |  28 ++++
 .../jwql/templates/background_monitor.html    |  25 +++
 jwql/website/apps/jwql/urls.py                |   1 +
 5 files changed, 188 insertions(+), 17 deletions(-)
 create mode 100644 jwql/website/apps/jwql/templates/background_monitor.html

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 6edcafd38..9e3e2c2fa 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -29,6 +29,7 @@
 matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import numpy as np
+import pandas as pd
 from photutils.segmentation import detect_sources, detect_threshold
 
 from jwql.database.database_interface import session, engine
@@ -40,12 +41,118 @@
 
 class ClawMonitor():
     """Class for executing the claw monitor.
+
+    This class searches for all new NIRCam full-frame imaging data
+    and creates observation-level, source-masked, median stacks
+    for each filter/pupil combination. These stacks are then plotted
+    in on-sky orientation and the results are used to identify new
+    instances of claws - a scattered light effect seen in NIRCam
+    data. Background statistics are also stored for each individual
+    image, which are then plotted to track the NIRCam background
+    levels over time. Results are all saved to database tables.
+
+    Attributes
+    ----------
+    outfile : str
+        The name of the output plot for a given claw stack combination.
+
+    output_dir : str
+        Path into which claw stack plots will be placed.
+
+    output_dir_bkg : str
+        Path into which background trending plots will be placed.
+
+    query_start : float
+        MJD start date to use for querying MAST.
+
+    query_end : float
+        MJD end date to use for querying MAST.
+
+    wv : str
+        NIRCam channel for a given claw stack, either ``SW`` or ``LW``.
+
+    proposal : str
+        NIRCam proposal number for a given claw stack.
+
+    obs : str
+        NIRCam observation number for a given claw stack.
+
+    fltr : str
+        NIRCam filter used for a given claw stack.
+
+    pupil : str
+        NIRCam pupil used for a given claw stack.
+
+    detectors : str
+        The detectors used for a given claw stack combination.
+
+    files : numpy.ndarray
+        The names of the individual files belonging to a given claw stack combination.
     """
 
     def __init__(self):
         """Initialize an instance of the ``ClawMonitor`` class.
         """
 
+    def make_background_plots(self):
+        """Makes plots of the background levels over time in NIRCam data.
+        """
+
+        # Get all of the background data.
+        query = session.query(NIRCamClawStats.filename, NIRCamClawStats.filter, NIRCamClawStats.pupil, NIRCamClawStats.detector,
+                              NIRCamClawStats.effexptm, NIRCamClawStats.expstart_mjd, NIRCamClawStats.entry_date, NIRCamClawStats.mean, 
+                              NIRCamClawStats.median, NIRCamClawStats.frac_masked).all()
+        df_orig = pd.DataFrame(query, columns=['filename', 'filter', 'pupil', 'detector', 'effexptm', 'expstart_mjd', 
+                                               'entry_date', 'mean', 'median', 'frac_masked'])
+        df_orig = df_orig.drop_duplicates(subset='filename', keep="last")  # remove any duplicate filename entries, keep the most recent
+
+        # Make backgroud trending plots for all SW wide filters
+        for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W']:
+            logging.info('Working on background trending plots for {}'.format(fltr))
+            grid = plt.GridSpec(2, 4, hspace=.2, wspace=.2, width_ratios=[1,1,1,1])
+            fig = plt.figure(figsize=(40, 20))
+            fig.suptitle(fltr, fontsize=45)
+            for i,det in enumerate(['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']):  # in on-sky order, don't change order
+                logging.info('Working on {}'.format(det))
+                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
+                # extended objects, nebulas, short exposures.
+                df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
+                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<0.075) & (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
+                
+                # Plot the background levels over time
+                ax = fig.add_subplot(grid[i])
+                ax.scatter(df['expstart_mjd'], df['median'])
+                ax.set_title(det, fontsize=30)
+                ax.set_ylabel('Background Level [MJy/sr]')
+                ax.set_xlabel('Date [MJD]')
+            fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
+            fig.clf()
+            plt.close()
+
+
+        # Make backgroud trending plots for all LW wide filters
+        for fltr in ['F277W', 'F356W', 'F444W']:
+            logging.info('Working on background trending plots for {}'.format(fltr))
+            grid = plt.GridSpec(1, 2, hspace=.2, wspace=.2, width_ratios=[1,1])
+            fig = plt.figure(figsize=(20, 10))
+            fig.suptitle(fltr, fontsize=30)
+            for i,det in enumerate(['NRCALONG', 'NRCBLONG']):
+                logging.info('Working on {}'.format(det))
+                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
+                # extended objects, nebulas, short exposures.
+                df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
+                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<0.15) & (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
+                
+                # Plot the background levels over time
+                ax = fig.add_subplot(grid[i])
+                ax.scatter(df['expstart_mjd'], df['median'])
+                ax.set_title(det, fontsize=20)
+                ax.set_ylabel('Background Level [MJy/sr]')
+                ax.set_xlabel('Date [MJD]')
+            fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
+            fig.clf()
+            plt.close()
+
     def process(self):
         """The main method for processing.  See module docstrings for further details.
         """
@@ -73,7 +180,8 @@ def process(self):
         for i,det in enumerate(detectors_to_run):
             logging.info('Working on {}'.format(det))
             files = self.files[self.detectors == det]
-            # Remove missing files; to avoid memory/speed issues, only use the first 20 files, which should be plenty to see any claws todo change value?
+            # Remove missing files; to avoid memory/speed issues, only use the first 20 files, 
+            # which should be plenty to see any claws todo change value?
             files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
             stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
@@ -102,15 +210,14 @@ def process(self):
                 stack[n] = np.ma.masked_array(data, mask=segmap!=0)
                 mean, med, stddev = sigma_clipped_stats(data[segmap==0])
                 
-                # Add this file's stats to the claw database table.
-                # Can't insert values with numpy.float32 datatypes into database
-                # so need to change the datatypes of these values.
+                # Add this file's stats to the claw database table. Can't insert values with numpy.float32
+                # datatypes into database so need to change the datatypes of these values.
                 claw_db_entry = {'filename': os.path.basename(f),
                                  'proposal': self.proposal,
                                  'obs': self.obs,
-                                 'detector': det,
-                                 'filter': self.fltr,
-                                 'pupil': self.pupil,
+                                 'detector': det.upper(),
+                                 'filter': self.fltr.upper(),
+                                 'pupil': self.pupil.upper(),
                                  'expstart': '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS']),
                                  'expstart_mjd': h[0].header['EXPSTART'],
                                  'effexptm': h[0].header['EFFEXPTM'],
@@ -134,7 +241,7 @@ def process(self):
             skyflat = skyflat / np.nanmedian(skyflat)
             skyflat[~np.isfinite(skyflat)] = 1  # fill missing values
 
-            # Add the skyflat for this detector to the plot
+            # Add the skyflat for this detector to the claw stack plot
             if (self.wv=='SW') & (i>3):  # skip colobar axis
                 idx = i+1
             else:
@@ -155,7 +262,7 @@ def process(self):
             ax.axes.get_xaxis().set_ticks([])
             ax.axes.get_yaxis().set_ticks([])
         
-        # Add colobar, save figure if any detector stacks exist
+        # Add colobar, save figure if any claw stacks exist
         if found_scale:
             fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
             cax = fig.add_subplot(grid[0:rows, cols-1:cols])
@@ -167,7 +274,7 @@ def process(self):
         logging.info('Claw stacks complete: {}'.format(self.outfile))
 
     def query_mast(self):
-        """Query MAST for new nircam full-frame imaging data.
+        """Query MAST for new NIRCam full-frame imaging data.
 
         Returns
         -------
@@ -207,22 +314,25 @@ def run(self):
         """The main method.  See module docstrings for further details."""
 
         logging.info('Begin logging for claw_monitor')
+
+        # Define and setup the output directories for the claw and background plots.
         self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
         ensure_dir_exists(self.output_dir)
-        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files
+        self.output_dir_bkg =  os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
+        ensure_dir_exists(self.output_dir_bkg)
+        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
         self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
 
         # Get the claw monitor database tables
         self.query_table = eval('NIRCamClawQueryHistory')
         self.stats_table = eval('NIRCamClawStats')
 
-        # Query MAST for new imaging data from the last 3 days
+        # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
-        self.query_start_mjd = self.query_end_mjd - 3
+        self.query_start_mjd = self.query_end_mjd - 2
         #self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
         self.query_start_mjd, self.query_end_mjd = 60150, 60152  # todo remove
         #self.query_start_mjd = 59985  # last run was may 25; todo remove
-
         print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
@@ -246,6 +356,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
             self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
             #print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
@@ -255,6 +366,11 @@ def run(self):
             else:
                 logging.info('{} already exists'.format(self.outfile))
 
+        # Update the background trending plots, if any new data exists
+        if len(t)>0:
+            logging.info('Making background trending plots.')
+            self.make_background_plots()
+
         # Update the query history
         new_entry = {'instrument': 'nircam',
                      'start_time_mjd': self.query_start_mjd,
@@ -270,9 +386,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)   # todo uncomment
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
 
     monitor = ClawMonitor()
     monitor.run()
 
-    monitor_utils.update_monitor_table(module, start_time, log_file)   # todo uncomment
+    monitor_utils.update_monitor_table(module, start_time, log_file)
diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
index b5728a8c4..52679037b 100644
--- a/jwql/utils/constants.py
+++ b/jwql/utils/constants.py
@@ -376,7 +376,8 @@
              ('Dark Current Monitor', '/miri/dark_monitor'),
              ('EDB Telemetry Monitor', '/miri/edb_monitor'),
              ('Readnoise Monitor', '/miri/readnoise_monitor')],
-    'nircam': [('Bad Pixel Monitor', '/nircam/bad_pixel_monitor'),
+    'nircam': [('Background Monitor', '/nircam/background_monitor'),
+               ('Bad Pixel Monitor', '/nircam/bad_pixel_monitor'),
                ('Bias Monitor', '/nircam/bias_monitor'),
                ('Claw Monitor', '/nircam/claw_monitor'),
                ('Cosmic Ray Monitor', '#'),
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index f271e69b8..d36c5ad07 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -51,6 +51,34 @@
 FILESYSTEM_DIR = os.path.join(CONFIG['jwql_dir'], 'filesystem')
 
 
+def background_monitor(request):
+    """Generate the NIRCam background monitor page
+
+    Parameters
+    ----------
+    request : HttpRequest object
+        Incoming request from the webpage
+
+    Returns
+    -------
+    HttpResponse object
+        Outgoing response sent to the webpage
+    """
+
+    template = "background_monitor.html"
+
+    # Get the background trending filters to display
+    fltrs = ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']
+    bkg_plots = ['/static/outputs/claw_monitor/backgrounds/{}_backgrounds.png'.format(fltr) for fltr in fltrs]
+
+    context = {
+        'inst': 'NIRCam',
+        'bkg_plots': bkg_plots
+        }
+
+    # Return a HTTP response with the template and dictionary of variables
+    return render(request, template, context)
+
 def bad_pixel_monitor(request, inst):
     """Generate the dark monitor page for a given instrument
 
diff --git a/jwql/website/apps/jwql/templates/background_monitor.html b/jwql/website/apps/jwql/templates/background_monitor.html
new file mode 100644
index 000000000..efe0c122b
--- /dev/null
+++ b/jwql/website/apps/jwql/templates/background_monitor.html
@@ -0,0 +1,25 @@
+{% extends "base.html" %}
+
+{% block preamble %}
+
+<title>{{ inst }} Background Monitor - JWQL</title>
+
+{% endblock %}
+
+{% block content %}
+
+    <main role="main" class="container">
+
+		<h1>{{ inst }} Background Monitor</h1>
+		<hr>
+
+		<div width=100%>
+			{% for bkg_plot in bkg_plots %}
+				<img src={{ bkg_plot | safe }} width="100%">
+				<p></p>
+			{% endfor %}
+		</div>
+
+	</main>
+
+{% endblock %}
\ No newline at end of file
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index e483cff9d..638e7501f 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -59,6 +59,7 @@
     path('', views.home, name='home'),
 
     # NIRCam-specific views
+    path('nircam/background_monitor/', monitor_views.background_monitor, name='background_monitor'), 
     path('nircam/claw_monitor/', monitor_views.claw_monitor, name='claw_monitor'), 
 
     # NIRSpec-specific views

From b2bd048679a43dde9ab91cc82d83e62eff0342cf Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 18 Oct 2023 17:09:07 -0400
Subject: [PATCH 10/22] updates to background trending plots

---
 .../nircam_monitors/claw_monitor.py           | 58 +++++++++++++++----
 .../jwql/templates/background_monitor.html    |  2 +-
 .../apps/jwql/templates/claw_monitor.html     |  2 +-
 3 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 9e3e2c2fa..379f6e35f 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -106,12 +106,18 @@ def make_background_plots(self):
                                                'entry_date', 'mean', 'median', 'frac_masked'])
         df_orig = df_orig.drop_duplicates(subset='filename', keep="last")  # remove any duplicate filename entries, keep the most recent
 
+        # Use the same time xlimits/xticks for all plots
+        start_mjd = 59650  # March 2022, middle of commissioning
+        end_mjd = Time.now().mjd + 0.05*(Time.now().mjd - start_mjd)
+        time_tick_vals = np.linspace(start_mjd, end_mjd, 5)  # Make April 2022 the first tick
+        time_tick_labels = [Time(m, format='mjd').isot.split('T')[0] for m in time_tick_vals]
+
         # Make backgroud trending plots for all SW wide filters
         for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
-            grid = plt.GridSpec(2, 4, hspace=.2, wspace=.2, width_ratios=[1,1,1,1])
+            grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1,1,1,1])
             fig = plt.figure(figsize=(40, 20))
-            fig.suptitle(fltr, fontsize=45)
+            fig.suptitle(fltr, fontsize=70)
             for i,det in enumerate(['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']):  # in on-sky order, don't change order
                 logging.info('Working on {}'.format(det))
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
@@ -122,9 +128,25 @@ def make_background_plots(self):
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
-                ax.set_title(det, fontsize=30)
-                ax.set_ylabel('Background Level [MJy/sr]')
-                ax.set_xlabel('Date [MJD]')
+
+                # Match scaling in all plots to the first detector. Shade median+/-10% region.
+                if len(df)>0:
+                    if i==0:
+                        first_med = np.nanmedian(df['median'])
+                    ax.set_ylim(first_med-first_med*0.5, first_med+first_med*0.5)
+                    med = np.nanmedian(df['median'])
+                    ax.axhline(med, ls='-', color='black')
+                    ax.axhspan(med-med*0.1, med+med*0.1, color='gray', alpha=0.4, lw=0)
+
+                # Axis formatting
+                ax.set_title(det, fontsize=40)
+                ax.set_xlim(start_mjd, end_mjd)
+                ax.set_xticks(time_tick_vals)
+                ax.set_xticklabels(time_tick_labels, fontsize=20, rotation=45)
+                ax.yaxis.set_tick_params(labelsize=20)
+                ax.set_ylabel('Background [MJy/sr]', fontsize=30)
+                #ax.set_xlabel('Date [YYYY-MM-DD]')
+                ax.grid(ls='--', color='gray')
             fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
             fig.clf()
             plt.close()
@@ -133,9 +155,9 @@ def make_background_plots(self):
         # Make backgroud trending plots for all LW wide filters
         for fltr in ['F277W', 'F356W', 'F444W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
-            grid = plt.GridSpec(1, 2, hspace=.2, wspace=.2, width_ratios=[1,1])
+            grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1,1])
             fig = plt.figure(figsize=(20, 10))
-            fig.suptitle(fltr, fontsize=30)
+            fig.suptitle(fltr, fontsize=70, y=1.05)
             for i,det in enumerate(['NRCALONG', 'NRCBLONG']):
                 logging.info('Working on {}'.format(det))
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
@@ -146,9 +168,25 @@ def make_background_plots(self):
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
-                ax.set_title(det, fontsize=20)
-                ax.set_ylabel('Background Level [MJy/sr]')
-                ax.set_xlabel('Date [MJD]')
+
+                # Match y scaling in all plots to the first detector. Shade median+/-10% region.
+                if len(df)>0:
+                    if i==0:
+                        first_med = np.nanmedian(df['median'])
+                    ax.set_ylim(first_med-first_med*0.5, first_med+first_med*0.5)
+                    med = np.nanmedian(df['median'])
+                    ax.axhline(med, ls='-', color='black')
+                    ax.axhspan(med-med*0.1, med+med*0.1, color='gray', alpha=0.4, lw=0)
+
+                # Axis formatting
+                ax.set_title(det, fontsize=40)
+                ax.set_xlim(start_mjd, end_mjd)
+                ax.set_xticks(time_tick_vals)
+                ax.set_xticklabels(time_tick_labels, fontsize=20, rotation=45)
+                ax.yaxis.set_tick_params(labelsize=20)
+                ax.set_ylabel('Background [MJy/sr]', fontsize=30)
+                #ax.set_xlabel('Date [YYYY-MM-DD]')
+                ax.grid(ls='--', color='gray')
             fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
             fig.clf()
             plt.close()
diff --git a/jwql/website/apps/jwql/templates/background_monitor.html b/jwql/website/apps/jwql/templates/background_monitor.html
index efe0c122b..ddbf53c6f 100644
--- a/jwql/website/apps/jwql/templates/background_monitor.html
+++ b/jwql/website/apps/jwql/templates/background_monitor.html
@@ -16,7 +16,7 @@ <h1>{{ inst }} Background Monitor</h1>
 		<div width=100%>
 			{% for bkg_plot in bkg_plots %}
 				<img src={{ bkg_plot | safe }} width="100%">
-				<p></p>
+				<br><br><hr>
 			{% endfor %}
 		</div>
 
diff --git a/jwql/website/apps/jwql/templates/claw_monitor.html b/jwql/website/apps/jwql/templates/claw_monitor.html
index f4af8d8be..0bae20e65 100644
--- a/jwql/website/apps/jwql/templates/claw_monitor.html
+++ b/jwql/website/apps/jwql/templates/claw_monitor.html
@@ -16,7 +16,7 @@ <h1>{{ inst }} Claw Monitor</h1>
 		<div width=100%>
 			{% for claw_stack in claw_stacks %}
 				<img src={{ claw_stack | safe }} width="100%">
-				<p></p>
+				<br><br><hr>
 			{% endfor %}
 		</div>
 

From b47dadb434b6d03d30833907e415b89b2beb8a53 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 10:33:32 -0400
Subject: [PATCH 11/22] combined sw and lw plotting in single loop

---
 .../nircam_monitors/claw_monitor.py           | 107 +++++++-----------
 1 file changed, 44 insertions(+), 63 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 379f6e35f..b912534ff 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -13,6 +13,13 @@
     ::
 
         python claw_monitor.py
+
+    To only update the background trending plots:
+
+    ::
+
+        m = ClawMonitor()
+        m.make_background_plots()
 """
 
 import datetime
@@ -94,6 +101,18 @@ def __init__(self):
         """Initialize an instance of the ``ClawMonitor`` class.
         """
 
+        # Define and setup the output directories for the claw and background plots.
+        self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+        ensure_dir_exists(self.output_dir)
+        self.output_dir_bkg =  os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
+        ensure_dir_exists(self.output_dir_bkg)
+        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
+        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
+
+        # Get the claw monitor database tables
+        self.query_table = eval('NIRCamClawQueryHistory')
+        self.stats_table = eval('NIRCamClawStats')
+
     def make_background_plots(self):
         """Makes plots of the background levels over time in NIRCam data.
         """
@@ -109,67 +128,38 @@ def make_background_plots(self):
         # Use the same time xlimits/xticks for all plots
         start_mjd = 59650  # March 2022, middle of commissioning
         end_mjd = Time.now().mjd + 0.05*(Time.now().mjd - start_mjd)
-        time_tick_vals = np.linspace(start_mjd, end_mjd, 5)  # Make April 2022 the first tick
+        time_tick_vals = np.linspace(start_mjd, end_mjd, 5)
         time_tick_labels = [Time(m, format='mjd').isot.split('T')[0] for m in time_tick_vals]
 
-        # Make backgroud trending plots for all SW wide filters
-        for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W']:
+        # Make backgroud trending plots for all wide filters
+        for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
-            grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1,1,1,1])
-            fig = plt.figure(figsize=(40, 20))
-            fig.suptitle(fltr, fontsize=70)
-            for i,det in enumerate(['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']):  # in on-sky order, don't change order
+            if int(fltr[1:4])<250:  # i.e. SW
+                detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']   # in on-sky order, don't change order
+                grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1,1,1,1])
+                fig = plt.figure(figsize=(40, 20))
+                fig.suptitle(fltr, fontsize=70)
+                frack_masked_thresh = 0.075
+            else:  # i.e. LW
+                detectors_to_run = ['NRCALONG', 'NRCBLONG']
+                grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1,1])
+                fig = plt.figure(figsize=(20, 10))
+                fig.suptitle(fltr, fontsize=70, y=1.05)
+                frack_masked_thresh = 0.15
+            for i,det in enumerate(detectors_to_run):
                 logging.info('Working on {}'.format(det))
-                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
-                # extended objects, nebulas, short exposures.
-                df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
-                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<0.075) & (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
-                
-                # Plot the background levels over time
-                ax = fig.add_subplot(grid[i])
-                ax.scatter(df['expstart_mjd'], df['median'])
 
-                # Match scaling in all plots to the first detector. Shade median+/-10% region.
-                if len(df)>0:
-                    if i==0:
-                        first_med = np.nanmedian(df['median'])
-                    ax.set_ylim(first_med-first_med*0.5, first_med+first_med*0.5)
-                    med = np.nanmedian(df['median'])
-                    ax.axhline(med, ls='-', color='black')
-                    ax.axhspan(med-med*0.1, med+med*0.1, color='gray', alpha=0.4, lw=0)
-
-                # Axis formatting
-                ax.set_title(det, fontsize=40)
-                ax.set_xlim(start_mjd, end_mjd)
-                ax.set_xticks(time_tick_vals)
-                ax.set_xticklabels(time_tick_labels, fontsize=20, rotation=45)
-                ax.yaxis.set_tick_params(labelsize=20)
-                ax.set_ylabel('Background [MJy/sr]', fontsize=30)
-                #ax.set_xlabel('Date [YYYY-MM-DD]')
-                ax.grid(ls='--', color='gray')
-            fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
-            fig.clf()
-            plt.close()
-
-
-        # Make backgroud trending plots for all LW wide filters
-        for fltr in ['F277W', 'F356W', 'F444W']:
-            logging.info('Working on background trending plots for {}'.format(fltr))
-            grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1,1])
-            fig = plt.figure(figsize=(20, 10))
-            fig.suptitle(fltr, fontsize=70, y=1.05)
-            for i,det in enumerate(['NRCALONG', 'NRCBLONG']):
-                logging.info('Working on {}'.format(det))
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
                 # extended objects, nebulas, short exposures.
                 df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
-                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<0.15) & (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
+                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<frack_masked_thresh) & 
+                             (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
                 
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
 
-                # Match y scaling in all plots to the first detector. Shade median+/-10% region.
+                # Match scaling in all plots to the first detector. Shade median+/-10% region.
                 if len(df)>0:
                     if i==0:
                         first_med = np.nanmedian(df['median'])
@@ -302,7 +292,8 @@ def process(self):
         
         # Add colobar, save figure if any claw stacks exist
         if found_scale:
-            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
+            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), 
+                         self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
             cax = fig.add_subplot(grid[0:rows, cols-1:cols])
             cbar = fig.colorbar(im, cax=cax, orientation='vertical')
             cbar.ax.tick_params(labelsize=cbar_fs)
@@ -353,18 +344,6 @@ def run(self):
 
         logging.info('Begin logging for claw_monitor')
 
-        # Define and setup the output directories for the claw and background plots.
-        self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
-        ensure_dir_exists(self.output_dir)
-        self.output_dir_bkg =  os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
-        ensure_dir_exists(self.output_dir_bkg)
-        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
-        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
-
-        # Get the claw monitor database tables
-        self.query_table = eval('NIRCamClawQueryHistory')
-        self.stats_table = eval('NIRCamClawStats')
-
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
@@ -392,8 +371,10 @@ def run(self):
             else:
                 self.wv = 'SW'
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
-            self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5), self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
+                                        self.obs, self.fltr, self.pupil).lower())
+            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 
+                            'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
             #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
             #print(self.files)
             self.detectors = np.array(tt['detector'])

From a5ac0352621c0fe009eb35ec5afaf1b1a7aa76be Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 10:59:33 -0400
Subject: [PATCH 12/22] pep8

---
 .../nircam_monitors/claw_monitor.py           | 126 +++++++++---------
 jwql/website/apps/jwql/monitor_views.py       |   7 +-
 jwql/website/apps/jwql/urls.py                |   4 +-
 3 files changed, 68 insertions(+), 69 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index b912534ff..e71ec52fa 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -104,7 +104,7 @@ def __init__(self):
         # Define and setup the output directories for the claw and background plots.
         self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
         ensure_dir_exists(self.output_dir)
-        self.output_dir_bkg =  os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
+        self.output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
         ensure_dir_exists(self.output_dir_bkg)
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
         self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
@@ -119,54 +119,54 @@ def make_background_plots(self):
 
         # Get all of the background data.
         query = session.query(NIRCamClawStats.filename, NIRCamClawStats.filter, NIRCamClawStats.pupil, NIRCamClawStats.detector,
-                              NIRCamClawStats.effexptm, NIRCamClawStats.expstart_mjd, NIRCamClawStats.entry_date, NIRCamClawStats.mean, 
+                              NIRCamClawStats.effexptm, NIRCamClawStats.expstart_mjd, NIRCamClawStats.entry_date, NIRCamClawStats.mean,
                               NIRCamClawStats.median, NIRCamClawStats.frac_masked).all()
-        df_orig = pd.DataFrame(query, columns=['filename', 'filter', 'pupil', 'detector', 'effexptm', 'expstart_mjd', 
+        df_orig = pd.DataFrame(query, columns=['filename', 'filter', 'pupil', 'detector', 'effexptm', 'expstart_mjd',
                                                'entry_date', 'mean', 'median', 'frac_masked'])
         df_orig = df_orig.drop_duplicates(subset='filename', keep="last")  # remove any duplicate filename entries, keep the most recent
 
         # Use the same time xlimits/xticks for all plots
         start_mjd = 59650  # March 2022, middle of commissioning
-        end_mjd = Time.now().mjd + 0.05*(Time.now().mjd - start_mjd)
+        end_mjd = Time.now().mjd + 0.05 * (Time.now().mjd - start_mjd)
         time_tick_vals = np.linspace(start_mjd, end_mjd, 5)
         time_tick_labels = [Time(m, format='mjd').isot.split('T')[0] for m in time_tick_vals]
 
         # Make backgroud trending plots for all wide filters
         for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
-            if int(fltr[1:4])<250:  # i.e. SW
+            if int(fltr[1:4]) < 250:  # i.e. SW
                 detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']   # in on-sky order, don't change order
-                grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1,1,1,1])
+                grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1, 1, 1, 1])
                 fig = plt.figure(figsize=(40, 20))
                 fig.suptitle(fltr, fontsize=70)
                 frack_masked_thresh = 0.075
             else:  # i.e. LW
                 detectors_to_run = ['NRCALONG', 'NRCBLONG']
-                grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1,1])
+                grid = plt.GridSpec(1, 2, hspace=.2, wspace=.4, width_ratios=[1, 1])
                 fig = plt.figure(figsize=(20, 10))
                 fig.suptitle(fltr, fontsize=70, y=1.05)
                 frack_masked_thresh = 0.15
-            for i,det in enumerate(detectors_to_run):
+            for i, det in enumerate(detectors_to_run):
                 logging.info('Working on {}'.format(det))
 
-                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields, 
+                # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields,
                 # extended objects, nebulas, short exposures.
-                df = df_orig[(df_orig['filter']==fltr) & (df_orig['pupil']=='CLEAR') & (df_orig['detector']==det) &
-                             (df_orig['effexptm']>300) & (df_orig['frac_masked']<frack_masked_thresh) & 
-                             (abs(1-(df_orig['mean']/df_orig['median']))<0.05)]
-                
+                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det) &
+                             (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh) &
+                             (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
+
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
 
                 # Match scaling in all plots to the first detector. Shade median+/-10% region.
-                if len(df)>0:
-                    if i==0:
+                if len(df) > 0:
+                    if i == 0:
                         first_med = np.nanmedian(df['median'])
-                    ax.set_ylim(first_med-first_med*0.5, first_med+first_med*0.5)
+                    ax.set_ylim(first_med - first_med * 0.5, first_med + first_med * 0.5)
                     med = np.nanmedian(df['median'])
                     ax.axhline(med, ls='-', color='black')
-                    ax.axhspan(med-med*0.1, med+med*0.1, color='gray', alpha=0.4, lw=0)
+                    ax.axhspan(med - med * 0.1, med + med * 0.1, color='gray', alpha=0.4, lw=0)
 
                 # Axis formatting
                 ax.set_title(det, fontsize=40)
@@ -175,7 +175,7 @@ def make_background_plots(self):
                 ax.set_xticklabels(time_tick_labels, fontsize=20, rotation=45)
                 ax.yaxis.set_tick_params(labelsize=20)
                 ax.set_ylabel('Background [MJy/sr]', fontsize=30)
-                #ax.set_xlabel('Date [YYYY-MM-DD]')
+                # ax.set_xlabel('Date [YYYY-MM-DD]')
                 ax.grid(ls='--', color='gray')
             fig.savefig(os.path.join(self.output_dir_bkg, '{}_backgrounds.png'.format(fltr)), dpi=180, bbox_inches='tight')
             fig.clf()
@@ -189,36 +189,36 @@ def process(self):
         if self.wv == 'SW':
             detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']  # in on-sky order, don't change order
             cols, rows = 5, 2
-            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,1,1,.1])
+            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1, 1, 1, 1, .1])
             fig = plt.figure(figsize=(40, 20))
             cbar_fs = 20
             fs = 30
         else:
             detectors_to_run = ['NRCALONG', 'NRCBLONG']
             cols, rows = 3, 1
-            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1,1,.1])
+            grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1, 1, .1])
             fig = plt.figure(figsize=(20, 10))
             cbar_fs = 10
             fs = 20
-        
+
         # Make source-masked, median-stack of each detector's images
         print(self.outfile)
         print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
         found_scale = False
-        for i,det in enumerate(detectors_to_run):
+        for i, det in enumerate(detectors_to_run):
             logging.info('Working on {}'.format(det))
             files = self.files[self.detectors == det]
-            # Remove missing files; to avoid memory/speed issues, only use the first 20 files, 
+            # Remove missing files; to avoid memory/speed issues, only use the first 20 files,
             # which should be plenty to see any claws todo change value?
             files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
             stack = np.ma.ones((len(files), 2048, 2048))
             print(det)
             print(files)
             print('------')
-            for n,f in enumerate(files):
+            for n, f in enumerate(files):
                 logging.info('Working on: {}'.format(f))
                 h = fits.open(f)
-                
+
                 # Get plot label info from first image
                 if n == 0:
                     obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
@@ -234,10 +234,10 @@ def process(self):
                 data_conv = convolve(data, kernel)
                 segmap = detect_sources(data_conv, threshold, npixels=6)
                 segmap = segmap.data
-                segmap[dq&1!=0] = 1  # flag DO_NOT_USE pixels
-                stack[n] = np.ma.masked_array(data, mask=segmap!=0)
-                mean, med, stddev = sigma_clipped_stats(data[segmap==0])
-                
+                segmap[dq & 1 != 0] = 1  # flag DO_NOT_USE pixels
+                stack[n] = np.ma.masked_array(data, mask=segmap != 0)
+                mean, med, stddev = sigma_clipped_stats(data[segmap == 0])
+
                 # Add this file's stats to the claw database table. Can't insert values with numpy.float32
                 # datatypes into database so need to change the datatypes of these values.
                 claw_db_entry = {'filename': os.path.basename(f),
@@ -255,10 +255,10 @@ def process(self):
                                  'mean': float(mean),
                                  'median': float(med),
                                  'stddev': float(stddev),
-                                 'frac_masked': len(segmap[segmap!=0]) / (segmap.shape[0]*segmap.shape[1]),
+                                 'frac_masked': len(segmap[segmap != 0]) / (segmap.shape[0] * segmap.shape[1]),
                                  'skyflat_filename': os.path.basename(self.outfile),
                                  'entry_date': datetime.datetime.now()
-                                }
+                                 }
                 with engine.begin() as connection:
                     connection.execute(self.stats_table.__table__.insert(), claw_db_entry)
                 h.close()
@@ -270,15 +270,15 @@ def process(self):
             skyflat[~np.isfinite(skyflat)] = 1  # fill missing values
 
             # Add the skyflat for this detector to the claw stack plot
-            if (self.wv=='SW') & (i>3):  # skip colobar axis
-                idx = i+1
+            if (self.wv == 'SW') & (i > 3):  # skip colobar axis
+                idx = i + 1
             else:
                 idx = i
             ax = fig.add_subplot(grid[idx])
-            if len(skyflat[skyflat!=1])==0:
+            if len(skyflat[skyflat != 1]) == 0:
                 ax.set_title('N/A', fontsize=fs)
                 ax.imshow(skyflat, cmap='coolwarm', vmin=999, vmax=999, origin='lower')
-            elif (len(skyflat[skyflat!=1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
+            elif (len(skyflat[skyflat != 1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
                 z = ZScaleInterval()
                 vmin, vmax = z.get_limits(skyflat)
                 found_scale = True
@@ -289,12 +289,12 @@ def process(self):
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
             ax.axes.get_xaxis().set_ticks([])
             ax.axes.get_yaxis().set_ticks([])
-        
+
         # Add colobar, save figure if any claw stacks exist
         if found_scale:
-            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(), 
-                         self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs*1.5)
-            cax = fig.add_subplot(grid[0:rows, cols-1:cols])
+            fig.suptitle('PID-{} OBS-{} {} {}\n{}  pa_v3={}\n'.format(self.proposal, self.obs, self.fltr.upper(),
+                         self.pupil.upper(), obs_start.split('.')[0], pa_v3), fontsize=fs * 1.5)
+            cax = fig.add_subplot(grid[0:rows, cols - 1:cols])
             cbar = fig.colorbar(im, cax=cax, orientation='vertical')
             cbar.ax.tick_params(labelsize=cbar_fs)
             fig.savefig(self.outfile, dpi=100, bbox_inches='tight')
@@ -318,22 +318,20 @@ def query_mast(self):
         JwstObs._portal_api_connection.COLUMNS_CONFIG_URL = server + "/portal_jwst/Mashup/Mashup.asmx/columnsconfig"
         JwstObs._portal_api_connection.MAST_BUNDLE_URL = server + "/jwst/api/v0.1/download/bundle"
         service = 'Mast.Jwst.Filtered.Nircam'
-        FIELDS = ['filename','program', 'observtn','category','instrume', 'productLevel', 'filter', 
-                  'pupil', 'subarray', 'detector','datamodl','date_beg_mjd', 'effexptm']
-        params = {"columns":",".join(FIELDS),
-                "filters":[
-                {"paramName":"pupil","values":['CLEAR','F162M','F164N','F323N','F405N','F466N','F470N']},
-                {"paramName":"exp_type","values":['NRC_IMAGE']},
-                {"paramName":"datamodl", "values":['ImageModel']},  # exclude calints, which are cubemodel
-                {"paramName":"productLevel", "values":['2b']},  # i.e. cal.fits
-                {"paramName":"subarray", "values":['FULL']},
-                ]
-                }
+        FIELDS = ['filename', 'program', 'observtn', 'category', 'instrume', 'productLevel', 'filter',
+                  'pupil', 'subarray', 'detector', 'datamodl', 'date_beg_mjd', 'effexptm']
+        params = {"columns" : ",".join(FIELDS),
+                "filters":[{"paramName" : "pupil","values" : ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
+                           {"paramName" : "exp_type","values" : ['NRC_IMAGE']},
+                           {"paramName" : "datamodl", "values" : ['ImageModel']},  # exclude calints, which are cubemodel
+                           {"paramName" : "productLevel", "values" : ['2b']},  # i.e. cal.fits
+                           {"paramName" : "subarray", "values" : ['FULL']},]
+                 }
         t = JwstObs.service_request(service, params)
-        t = t[(t['date_beg_mjd']>self.query_start_mjd) & (t['date_beg_mjd']<self.query_end_mjd)]
+        t = t[(t['date_beg_mjd'] > self.query_start_mjd) & (t['date_beg_mjd'] < self.query_end_mjd)]
         t.sort('date_beg_mjd')
-        filetypes = np.array([row['filename'].split('_')[-1].replace('.fits','') for row in t])
-        t = t[filetypes=='cal']  # only want cal.fits files, no e.g. i2d.fits
+        filetypes = np.array([row['filename'].split('_')[-1].replace('.fits', '') for row in t])
+        t = t[filetypes == 'cal']  # only want cal.fits files, no e.g. i2d.fits
 
         return t
 
@@ -347,13 +345,13 @@ def run(self):
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
-        #self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
+        # self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
         self.query_start_mjd, self.query_end_mjd = 60150, 60152  # todo remove
-        #self.query_start_mjd = 59985  # last run was may 25; todo remove
+        # self.query_start_mjd = 59985  # last run was may 25; todo remove
         print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
-        #print(t)
+        # print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
@@ -362,10 +360,10 @@ def run(self):
         print(np.unique(combos))
         t['combos'] = combos
         monitor_run = False
-        for nnn,combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
-            print(combo, '{}/{}'.format(nnn,n_combos))
-            tt = t[t['combos']==combo]
-            #print(tt)
+        for nnn, combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
+            print(combo, '{}/{}'.format(nnn, n_combos))
+            tt = t[t['combos'] == combo]
+            # print(tt)
             if 'long' in tt['filename'][0]:
                 self.wv = 'LW'
             else:
@@ -374,9 +372,9 @@ def run(self):
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
             self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 
-                            'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
-            #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
-            #print(self.files)
+                                  'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
+            # self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
+            # print(self.files)
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
@@ -386,7 +384,7 @@ def run(self):
                 logging.info('{} already exists'.format(self.outfile))
 
         # Update the background trending plots, if any new data exists
-        if len(t)>0:
+        if len(t) > 0:
             logging.info('Making background trending plots.')
             self.make_background_plots()
 
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index d36c5ad07..86be368f6 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -74,11 +74,12 @@ def background_monitor(request):
     context = {
         'inst': 'NIRCam',
         'bkg_plots': bkg_plots
-        }
+    }
 
     # Return a HTTP response with the template and dictionary of variables
     return render(request, template, context)
 
+
 def bad_pixel_monitor(request, inst):
     """Generate the dark monitor page for a given instrument
 
@@ -151,13 +152,13 @@ def claw_monitor(request):
     # Get all recent claw stack images
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
-    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd']>Time.now().mjd-100]))  # todo change 100 to 10 days back?
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 100]))  # todo change 100 to 10 days back?
     claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
 
     context = {
         'inst': 'NIRCam',
         'claw_stacks': claw_stacks
-        }
+    }
 
     # Return a HTTP response with the template and dictionary of variables
     return render(request, template, context)
diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py
index 638e7501f..64c00c53a 100644
--- a/jwql/website/apps/jwql/urls.py
+++ b/jwql/website/apps/jwql/urls.py
@@ -59,8 +59,8 @@
     path('', views.home, name='home'),
 
     # NIRCam-specific views
-    path('nircam/background_monitor/', monitor_views.background_monitor, name='background_monitor'), 
-    path('nircam/claw_monitor/', monitor_views.claw_monitor, name='claw_monitor'), 
+    path('nircam/background_monitor/', monitor_views.background_monitor, name='background_monitor'),
+    path('nircam/claw_monitor/', monitor_views.claw_monitor, name='claw_monitor'),
 
     # NIRSpec-specific views
     path('nirspec/msata_monitor/', monitor_views.msata_monitoring, name='msata_monitor'),

From 5b53632df4266fca18dee844acd432dc93a27b70 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 11:05:29 -0400
Subject: [PATCH 13/22] pep8

---
 .../nircam_monitors/claw_monitor.py           | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index e71ec52fa..a2c7ea8f7 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -107,7 +107,7 @@ def __init__(self):
         self.output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
         ensure_dir_exists(self.output_dir_bkg)
         self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
-        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  #todo remove
+        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  # todo remove
 
         # Get the claw monitor database tables
         self.query_table = eval('NIRCamClawQueryHistory')
@@ -151,9 +151,9 @@ def make_background_plots(self):
 
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields,
                 # extended objects, nebulas, short exposures.
-                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det) &
-                             (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh) &
-                             (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
+                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det)
+                             & (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh)
+                             & (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
 
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
@@ -320,13 +320,13 @@ def query_mast(self):
         service = 'Mast.Jwst.Filtered.Nircam'
         FIELDS = ['filename', 'program', 'observtn', 'category', 'instrume', 'productLevel', 'filter',
                   'pupil', 'subarray', 'detector', 'datamodl', 'date_beg_mjd', 'effexptm']
-        params = {"columns" : ",".join(FIELDS),
-                "filters":[{"paramName" : "pupil","values" : ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
-                           {"paramName" : "exp_type","values" : ['NRC_IMAGE']},
-                           {"paramName" : "datamodl", "values" : ['ImageModel']},  # exclude calints, which are cubemodel
-                           {"paramName" : "productLevel", "values" : ['2b']},  # i.e. cal.fits
-                           {"paramName" : "subarray", "values" : ['FULL']},]
-                 }
+        params = {"columns": ",".join(FIELDS),
+                  "filters":[{"paramName": "pupil", "values": ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
+                             {"paramName": "exp_type", "values": ['NRC_IMAGE']},
+                             {"paramName": "datamodl", "values": ['ImageModel']},  # exclude calints, which are cubemodel
+                             {"paramName": "productLevel", "values": ['2b']},  # i.e. cal.fits
+                             {"paramName": "subarray", "values": ['FULL']}, ]
+                  }
         t = JwstObs.service_request(service, params)
         t = t[(t['date_beg_mjd'] > self.query_start_mjd) & (t['date_beg_mjd'] < self.query_end_mjd)]
         t.sort('date_beg_mjd')
@@ -371,7 +371,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)), 
+            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)),
                                   'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
             # self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
             # print(self.files)

From 7e3bc889a55f5e7b2192fec91b46a38cef8d2054 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 11:07:15 -0400
Subject: [PATCH 14/22] pep8

---
 .../nircam_monitors/claw_monitor.py              | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index a2c7ea8f7..1a874b27b 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -151,9 +151,9 @@ def make_background_plots(self):
 
                 # Get relevant data for this filter/detector and remove bad datasets, e.g. crowded fields,
                 # extended objects, nebulas, short exposures.
-                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det)
-                             & (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh)
-                             & (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
+                df = df_orig[(df_orig['filter'] == fltr) & (df_orig['pupil'] == 'CLEAR') & (df_orig['detector'] == det) &
+                             (df_orig['effexptm'] > 300) & (df_orig['frac_masked'] < frack_masked_thresh) &
+                             (abs(1 - (df_orig['mean'] / df_orig['median'])) < 0.05)]
 
                 # Plot the background levels over time
                 ax = fig.add_subplot(grid[i])
@@ -321,11 +321,11 @@ def query_mast(self):
         FIELDS = ['filename', 'program', 'observtn', 'category', 'instrume', 'productLevel', 'filter',
                   'pupil', 'subarray', 'detector', 'datamodl', 'date_beg_mjd', 'effexptm']
         params = {"columns": ",".join(FIELDS),
-                  "filters":[{"paramName": "pupil", "values": ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
-                             {"paramName": "exp_type", "values": ['NRC_IMAGE']},
-                             {"paramName": "datamodl", "values": ['ImageModel']},  # exclude calints, which are cubemodel
-                             {"paramName": "productLevel", "values": ['2b']},  # i.e. cal.fits
-                             {"paramName": "subarray", "values": ['FULL']}, ]
+                  "filters": [{"paramName": "pupil", "values": ['CLEAR', 'F162M', 'F164N', 'F323N', 'F405N', 'F466N', 'F470N']},
+                              {"paramName": "exp_type", "values": ['NRC_IMAGE']},
+                              {"paramName": "datamodl", "values": ['ImageModel']},  # exclude calints, which are cubemodel
+                              {"paramName": "productLevel", "values": ['2b']},  # i.e. cal.fits
+                              {"paramName": "subarray", "values": ['FULL']}, ]
                   }
         t = JwstObs.service_request(service, params)
         t = t[(t['date_beg_mjd'] > self.query_start_mjd) & (t['date_beg_mjd'] < self.query_end_mjd)]

From 582e2e832556278364f6d578a7903306890618a6 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Thu, 19 Oct 2023 14:40:35 -0400
Subject: [PATCH 15/22] removed testing statements

---
 .../nircam_monitors/claw_monitor.py           | 36 +++++--------------
 jwql/website/apps/jwql/monitor_views.py       |  4 +--
 2 files changed, 10 insertions(+), 30 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 1a874b27b..30c2895b2 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -41,8 +41,8 @@
 
 from jwql.database.database_interface import session, engine
 from jwql.database.database_interface import NIRCamClawQueryHistory, NIRCamClawStats
-from jwql.utils import monitor_utils  # todo uncomment
-from jwql.utils.logging_functions import log_info, log_fail  # todo uncomment
+from jwql.utils import monitor_utils
+from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
 
@@ -106,8 +106,6 @@ def __init__(self):
         ensure_dir_exists(self.output_dir)
         self.output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
         ensure_dir_exists(self.output_dir_bkg)
-        self.data_dir = '/ifs/jwst/wit/nircam/commissioning/'  # todo change this to path of cal.fits files REMOVE
-        self.data_dir = '/ifs/jwst/wit/witserv/data7/nrc/bsunnquist/'  # todo remove
 
         # Get the claw monitor database tables
         self.query_table = eval('NIRCamClawQueryHistory')
@@ -202,19 +200,14 @@ def process(self):
             fs = 20
 
         # Make source-masked, median-stack of each detector's images
-        print(self.outfile)
-        print(self.proposal, self.obs, self.fltr, self.pupil, self.wv, detectors_to_run)
         found_scale = False
         for i, det in enumerate(detectors_to_run):
             logging.info('Working on {}'.format(det))
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files,
-            # which should be plenty to see any claws todo change value?
-            files = [f for f in files if os.path.exists(f)][0:3]  # todo change index value?
+            # which should be plenty to see any claws.
+            files = [f for f in files if os.path.exists(f)][0:20]
             stack = np.ma.ones((len(files), 2048, 2048))
-            print(det)
-            print(files)
-            print('------')
             for n, f in enumerate(files):
                 logging.info('Working on: {}'.format(f))
                 h = fits.open(f)
@@ -335,8 +328,8 @@ def query_mast(self):
 
         return t
 
-    @log_fail  # todo uncomment
-    @log_info  # todo uncomment
+    @log_fail
+    @log_info
     def run(self):
         """The main method.  See module docstrings for further details."""
 
@@ -345,25 +338,15 @@ def run(self):
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
-        # self.query_start_mjd, self.query_end_mjd = 59878.934, 59878.986  # todo remove these test datess test case
-        self.query_start_mjd, self.query_end_mjd = 60150, 60152  # todo remove
-        # self.query_start_mjd = 59985  # last run was may 25; todo remove
-        print(self.query_start_mjd, self.query_end_mjd)
         t = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
-        # print(t)
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
-        n_combos = len(np.unique(combos))
-        print('unique combos:')
-        print(np.unique(combos))
         t['combos'] = combos
         monitor_run = False
-        for nnn, combo in enumerate(np.unique(combos)[0:]):  # todo take off 0:2
-            print(combo, '{}/{}'.format(nnn, n_combos))
+        for combo in np.unique(combos):
             tt = t[t['combos'] == combo]
-            # print(tt)
             if 'long' in tt['filename'][0]:
                 self.wv = 'LW'
             else:
@@ -371,10 +354,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([os.path.join(self.data_dir, '{}'.format(str(self.proposal).zfill(5)),
-                                  'obsnum{}'.format(self.obs), row['filename']) for row in tt])  # todo change to server filepath
-            # self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment
-            # print(self.files)
+            self.files = np.array([filesystem_path(row['filename']) for row in tt])
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index 86be368f6..dc070b9fc 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -149,10 +149,10 @@ def claw_monitor(request):
 
     template = "claw_monitor.html"
 
-    # Get all recent claw stack images
+    # Get all recent claw stack images from the last 10 days
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
-    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 100]))  # todo change 100 to 10 days back?
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 10]))
     claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
 
     context = {

From 056295b2351d8f614aab0afdcc564220c8605fda Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Fri, 20 Oct 2023 10:16:58 -0400
Subject: [PATCH 16/22] a couple changes for testing purposes

---
 jwql/instrument_monitors/nircam_monitors/claw_monitor.py | 4 +++-
 jwql/website/apps/jwql/monitor_views.py                  | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 30c2895b2..f9826cf81 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -338,6 +338,7 @@ def run(self):
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
+        self.query_start_mjd, self.query_end_mjd = 59715.28951771492, 59715.29771992559  # todo remove
         t = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
 
@@ -354,7 +355,8 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([filesystem_path(row['filename']) for row in tt])
+            #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment?
+            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in tt])
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index dc070b9fc..d8ff30f3f 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -152,7 +152,7 @@ def claw_monitor(request):
     # Get all recent claw stack images from the last 10 days
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
-    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 10]))
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 1000]))  # todo change from 1000 to 10
     claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
 
     context = {

From 82109f5ed1b32d26c5a4813485f0c208ca3b39b2 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Fri, 20 Oct 2023 16:22:18 -0400
Subject: [PATCH 17/22] Removed zscaling; fixed potential future plot limit
 issue

---
 .../nircam_monitors/claw_monitor.py                 | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index f9826cf81..67a3bfc49 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -30,7 +30,6 @@
 from astropy.io import fits
 from astropy.stats import gaussian_fwhm_to_sigma, sigma_clipped_stats
 from astropy.time import Time
-from astropy.visualization import ZScaleInterval
 from astroquery.mast import Mast
 import matplotlib
 matplotlib.use('Agg')
@@ -132,6 +131,7 @@ def make_background_plots(self):
         # Make backgroud trending plots for all wide filters
         for fltr in ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']:
             logging.info('Working on background trending plots for {}'.format(fltr))
+            found_limits = False
             if int(fltr[1:4]) < 250:  # i.e. SW
                 detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']   # in on-sky order, don't change order
                 grid = plt.GridSpec(2, 4, hspace=.4, wspace=.4, width_ratios=[1, 1, 1, 1])
@@ -157,10 +157,11 @@ def make_background_plots(self):
                 ax = fig.add_subplot(grid[i])
                 ax.scatter(df['expstart_mjd'], df['median'])
 
-                # Match scaling in all plots to the first detector. Shade median+/-10% region.
+                # Match scaling in all plots to the first detector with data. Shade median+/-10% region.
                 if len(df) > 0:
-                    if i == 0:
+                    if found_limits is False:
                         first_med = np.nanmedian(df['median'])
+                        found_limits = True
                     ax.set_ylim(first_med - first_med * 0.5, first_med + first_med * 0.5)
                     med = np.nanmedian(df['median'])
                     ax.axhline(med, ls='-', color='black')
@@ -272,8 +273,8 @@ def process(self):
                 ax.set_title('N/A', fontsize=fs)
                 ax.imshow(skyflat, cmap='coolwarm', vmin=999, vmax=999, origin='lower')
             elif (len(skyflat[skyflat != 1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
-                z = ZScaleInterval()
-                vmin, vmax = z.get_limits(skyflat)
+                mean, med, stddev = sigma_clipped_stats(skyflat)
+                vmin, vmax =  med - 3 * stddev, med + 3 * stddev
                 found_scale = True
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
@@ -356,7 +357,7 @@ def run(self):
             self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
             #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment?
-            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in tt])
+            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in tt]) # todo remove
             self.detectors = np.array(tt['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))

From 744d8017b8404b8abae98e60f3290c0921d2b872 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 8 Nov 2023 12:20:35 -0500
Subject: [PATCH 18/22] addressed review comments; changed output directory for
 display

---
 .../nircam_monitors/claw_monitor.py           | 75 ++++++++++---------
 jwql/website/apps/jwql/monitor_views.py       |  6 +-
 2 files changed, 42 insertions(+), 39 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 67a3bfc49..72244aad3 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -32,7 +32,6 @@
 from astropy.time import Time
 from astroquery.mast import Mast
 import matplotlib
-matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
@@ -44,6 +43,8 @@
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
+matplotlib.use('Agg')
+
 
 class ClawMonitor():
     """Class for executing the claw monitor.
@@ -62,7 +63,7 @@ class ClawMonitor():
     outfile : str
         The name of the output plot for a given claw stack combination.
 
-    output_dir : str
+    output_dir_claws : str
         Path into which claw stack plots will be placed.
 
     output_dir_bkg : str
@@ -74,7 +75,7 @@ class ClawMonitor():
     query_end : float
         MJD end date to use for querying MAST.
 
-    wv : str
+    channel : str
         NIRCam channel for a given claw stack, either ``SW`` or ``LW``.
 
     proposal : str
@@ -101,8 +102,8 @@ def __init__(self):
         """
 
         # Define and setup the output directories for the claw and background plots.
-        self.output_dir = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
-        ensure_dir_exists(self.output_dir)
+        self.output_dir_claws = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+        ensure_dir_exists(self.output_dir_claws)
         self.output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
         ensure_dir_exists(self.output_dir_bkg)
 
@@ -185,7 +186,7 @@ def process(self):
         """
 
         # Get detector order and plot settings, depending on the wavelength channel
-        if self.wv == 'SW':
+        if self.channel == 'SW':
             detectors_to_run = ['NRCA2', 'NRCA4', 'NRCB3', 'NRCB1', 'NRCA1', 'NRCA3', 'NRCB4', 'NRCB2']  # in on-sky order, don't change order
             cols, rows = 5, 2
             grid = plt.GridSpec(rows, cols, hspace=.2, wspace=.2, width_ratios=[1, 1, 1, 1, .1])
@@ -207,20 +208,20 @@ def process(self):
             files = self.files[self.detectors == det]
             # Remove missing files; to avoid memory/speed issues, only use the first 20 files,
             # which should be plenty to see any claws.
-            files = [f for f in files if os.path.exists(f)][0:20]
+            files = [fname for fname in files if os.path.exists(fname)][0:20]
             stack = np.ma.ones((len(files), 2048, 2048))
-            for n, f in enumerate(files):
-                logging.info('Working on: {}'.format(f))
-                h = fits.open(f)
+            for n, fname in enumerate(files):
+                logging.info('Working on: {}'.format(fname))
+                hdu = fits.open(fname)
 
                 # Get plot label info from first image
                 if n == 0:
-                    obs_start = '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS'])
-                    pa_v3 = h[1].header['PA_V3']
+                    obs_start = '{}T{}'.format(hdu[0].header['DATE-OBS'], hdu[0].header['TIME-OBS'])
+                    pa_v3 = hdu[1].header['PA_V3']
 
                 # Make source segmap, add the masked data to the stack, and get background stats
-                data = h['SCI'].data
-                dq = h['DQ'].data
+                data = hdu['SCI'].data
+                dq = hdu['DQ'].data
                 threshold = detect_threshold(data, 1.0)
                 sigma = 3.0 * gaussian_fwhm_to_sigma  # FWHM = 3.
                 kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
@@ -234,18 +235,18 @@ def process(self):
 
                 # Add this file's stats to the claw database table. Can't insert values with numpy.float32
                 # datatypes into database so need to change the datatypes of these values.
-                claw_db_entry = {'filename': os.path.basename(f),
+                claw_db_entry = {'filename': os.path.basename(fname),
                                  'proposal': self.proposal,
                                  'obs': self.obs,
                                  'detector': det.upper(),
                                  'filter': self.fltr.upper(),
                                  'pupil': self.pupil.upper(),
-                                 'expstart': '{}T{}'.format(h[0].header['DATE-OBS'], h[0].header['TIME-OBS']),
-                                 'expstart_mjd': h[0].header['EXPSTART'],
-                                 'effexptm': h[0].header['EFFEXPTM'],
-                                 'ra': h[1].header['RA_V1'],
-                                 'dec': h[1].header['DEC_V1'],
-                                 'pa_v3': h[1].header['PA_V3'],
+                                 'expstart': '{}T{}'.format(hdu[0].header['DATE-OBS'], hdu[0].header['TIME-OBS']),
+                                 'expstart_mjd': hdu[0].header['EXPSTART'],
+                                 'effexptm': hdu[0].header['EFFEXPTM'],
+                                 'ra': hdu[1].header['RA_V1'],
+                                 'dec': hdu[1].header['DEC_V1'],
+                                 'pa_v3': hdu[1].header['PA_V3'],
                                  'mean': float(mean),
                                  'median': float(med),
                                  'stddev': float(stddev),
@@ -255,7 +256,7 @@ def process(self):
                                  }
                 with engine.begin() as connection:
                     connection.execute(self.stats_table.__table__.insert(), claw_db_entry)
-                h.close()
+                hdu.close()
 
             # Make the normalized skyflat for this detector
             skyflat = np.ma.median(stack, axis=0)
@@ -264,7 +265,7 @@ def process(self):
             skyflat[~np.isfinite(skyflat)] = 1  # fill missing values
 
             # Add the skyflat for this detector to the claw stack plot
-            if (self.wv == 'SW') & (i > 3):  # skip colobar axis
+            if (self.channel == 'SW') & (i > 3):  # skip colobar axis
                 idx = i + 1
             else:
                 idx = i
@@ -274,7 +275,7 @@ def process(self):
                 ax.imshow(skyflat, cmap='coolwarm', vmin=999, vmax=999, origin='lower')
             elif (len(skyflat[skyflat != 1]) > 0) & (found_scale is False):  # match scaling to first non-empty stack
                 mean, med, stddev = sigma_clipped_stats(skyflat)
-                vmin, vmax =  med - 3 * stddev, med + 3 * stddev
+                vmin, vmax = med - 3 * stddev, med + 3 * stddev
                 found_scale = True
                 ax.set_title(det, fontsize=fs)
                 im = ax.imshow(skyflat, cmap='coolwarm', vmin=vmin, vmax=vmax, origin='lower')
@@ -340,25 +341,25 @@ def run(self):
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
         self.query_start_mjd, self.query_end_mjd = 59715.28951771492, 59715.29771992559  # todo remove
-        t = self.query_mast()
-        logging.info('{} files found between {} and {}.'.format(len(t), self.query_start_mjd, self.query_end_mjd))
+        mast_table = self.query_mast()
+        logging.info('{} files found between {} and {}.'.format(len(mast_table), self.query_start_mjd, self.query_end_mjd))
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
-        combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in t])
-        t['combos'] = combos
+        combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in mast_table])
+        mast_table['combos'] = combos
         monitor_run = False
         for combo in np.unique(combos):
-            tt = t[t['combos'] == combo]
-            if 'long' in tt['filename'][0]:
-                self.wv = 'LW'
+            mast_table_combo = mast_table[mast_table['combos'] == combo]
+            if 'long' in mast_table_combo['filename'][0]:
+                self.channel = 'LW'
             else:
-                self.wv = 'SW'
+                self.channel = 'SW'
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
-            self.outfile = os.path.join(self.output_dir, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
+            self.outfile = os.path.join(self.output_dir_claws, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            #self.files = np.array([filesystem_path(row['filename']) for row in tt])  # todo uncomment?
-            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in tt]) # todo remove
-            self.detectors = np.array(tt['detector'])
+            #self.files = np.array([filesystem_path(row['filename']) for row in mast_table_combo])  # todo uncomment?
+            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in mast_table_combo]) # todo remove
+            self.detectors = np.array(mast_table_combo['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
                 self.process()
@@ -367,7 +368,7 @@ def run(self):
                 logging.info('{} already exists'.format(self.outfile))
 
         # Update the background trending plots, if any new data exists
-        if len(t) > 0:
+        if len(mast_table) > 0:
             logging.info('Making background trending plots.')
             self.make_background_plots()
 
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index a724f420c..e750d461a 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -70,8 +70,9 @@ def background_monitor(request):
     template = "background_monitor.html"
 
     # Get the background trending filters to display
+    output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
     fltrs = ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']
-    bkg_plots = ['/static/outputs/claw_monitor/backgrounds/{}_backgrounds.png'.format(fltr) for fltr in fltrs]
+    bkg_plots = [os.path.join(output_dir_bkg, '{}_backgrounds.png'.format(fltr)) for fltr in fltrs]
 
     context = {
         'inst': 'NIRCam',
@@ -155,7 +156,8 @@ def claw_monitor(request):
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
     recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 1000]))  # todo change from 1000 to 10
-    claw_stacks = ['/static/outputs/claw_monitor/claw_stacks/{}'.format(filename) for filename in recent_files]
+    output_dir_claws = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+    claw_stacks = [os.path.join(output_dir_claws, filename) for filename in recent_files]
 
     context = {
         'inst': 'NIRCam',

From 9bf381b7e26544bb95ccf1d2c9874f7086887bf3 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Wed, 8 Nov 2023 12:24:55 -0500
Subject: [PATCH 19/22] pep8

---
 jwql/database/database_interface.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/database/database_interface.py b/jwql/database/database_interface.py
index 47800f647..73b536eed 100644
--- a/jwql/database/database_interface.py
+++ b/jwql/database/database_interface.py
@@ -573,7 +573,7 @@ class : obj
             NIRSpecEDBDailyStats, NIRSpecEDBBlockStats, NIRSpecEDBTimeIntervalStats,
             NIRSpecEDBEveryChangeStats, FGSEDBDailyStats, FGSEDBBlockStats,
             FGSEDBTimeIntervalStats, FGSEDBEveryChangeStats],
-    'claw': [NIRCamClawQueryHistory, NIRCamClawStats],}
+    'claw': [NIRCamClawQueryHistory, NIRCamClawStats], }
 
 if __name__ == '__main__':
     base.metadata.create_all(engine)

From 4c57c27ca0465f25ec84da790aea20facfc508d5 Mon Sep 17 00:00:00 2001
From: Mees Fix <mfix@stsci.edu>
Date: Mon, 18 Dec 2023 11:28:06 -0500
Subject: [PATCH 20/22] updating static file path for claw monitor

---
 jwql/website/apps/jwql/monitor_views.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index e750d461a..49d1f257a 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -34,6 +34,7 @@
 from bokeh.resources import CDN, INLINE
 from django.http import HttpResponse, JsonResponse
 from django.shortcuts import render
+from django.templatetags.static import static
 import json
 import pandas as pd
 
@@ -70,7 +71,8 @@ def background_monitor(request):
     template = "background_monitor.html"
 
     # Get the background trending filters to display
-    output_dir_bkg = os.path.join(get_config()['outputs'], 'claw_monitor', 'backgrounds')
+    server_name = get_config()['outputs'].split("outputs/", 1)[1]
+    output_dir_bkg = static(os.path.join("outputs", server_name, "claw_monitor", "backgrounds"))
     fltrs = ['F070W', 'F090W', 'F115W', 'F150W', 'F200W', 'F277W', 'F356W', 'F444W']
     bkg_plots = [os.path.join(output_dir_bkg, '{}_backgrounds.png'.format(fltr)) for fltr in fltrs]
 
@@ -156,7 +158,10 @@ def claw_monitor(request):
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
     recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 1000]))  # todo change from 1000 to 10
-    output_dir_claws = os.path.join(get_config()['outputs'], 'claw_monitor', 'claw_stacks')
+    
+    server_name = get_config()['outputs'].split("outputs/", 1)[1]
+    output_dir_claws = static(os.path.join("outputs", server_name, "claw_monitor", "claw_stacks"))
+
     claw_stacks = [os.path.join(output_dir_claws, filename) for filename in recent_files]
 
     context = {

From 31a3416bfc4b8fd8ef2f01e89025bd6b0da8923e Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Mon, 18 Dec 2023 11:45:48 -0500
Subject: [PATCH 21/22] removed testing statements

---
 jwql/instrument_monitors/nircam_monitors/claw_monitor.py | 4 +---
 jwql/website/apps/jwql/monitor_views.py                  | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 72244aad3..72f45021f 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -340,7 +340,6 @@ def run(self):
         # Query MAST for new NIRCam full-frame imaging data from the last 2 days
         self.query_end_mjd = Time.now().mjd
         self.query_start_mjd = self.query_end_mjd - 2
-        self.query_start_mjd, self.query_end_mjd = 59715.28951771492, 59715.29771992559  # todo remove
         mast_table = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(mast_table), self.query_start_mjd, self.query_end_mjd))
 
@@ -357,8 +356,7 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir_claws, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            #self.files = np.array([filesystem_path(row['filename']) for row in mast_table_combo])  # todo uncomment?
-            self.files = np.array([os.path.join(get_config()['filesystem'], 'public', filesystem_path(row['filename'])) for row in mast_table_combo]) # todo remove
+            self.files = np.array([filesystem_path(row['filename']) for row in mast_table_combo])
             self.detectors = np.array(mast_table_combo['detector'])
             if not os.path.exists(self.outfile):
                 logging.info('Working on {}'.format(self.outfile))
diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py
index 49d1f257a..5b8607ac1 100644
--- a/jwql/website/apps/jwql/monitor_views.py
+++ b/jwql/website/apps/jwql/monitor_views.py
@@ -157,7 +157,7 @@ def claw_monitor(request):
     # Get all recent claw stack images from the last 10 days
     query = session.query(NIRCamClawStats.expstart_mjd, NIRCamClawStats.skyflat_filename).order_by(NIRCamClawStats.expstart_mjd.desc()).all()
     df = pd.DataFrame(query, columns=['expstart_mjd', 'skyflat_filename'])
-    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 1000]))  # todo change from 1000 to 10
+    recent_files = list(pd.unique(df['skyflat_filename'][df['expstart_mjd'] > Time.now().mjd - 10]))
     
     server_name = get_config()['outputs'].split("outputs/", 1)[1]
     output_dir_claws = static(os.path.join("outputs", server_name, "claw_monitor", "claw_stacks"))

From bc497280c460835b67c3734735fc2f38166fd834 Mon Sep 17 00:00:00 2001
From: Ben Sunnquist <bsunnquist@stsci.edu>
Date: Mon, 18 Dec 2023 15:36:30 -0500
Subject: [PATCH 22/22] skipping claw stacks that are missing cal files

---
 .../nircam_monitors/claw_monitor.py                  | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
index 72f45021f..c92dde3e2 100644
--- a/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
+++ b/jwql/instrument_monitors/nircam_monitors/claw_monitor.py
@@ -356,14 +356,20 @@ def run(self):
             self.proposal, self.obs, self.fltr, self.pupil = combo.split('_')
             self.outfile = os.path.join(self.output_dir_claws, 'prop{}_obs{}_{}_{}_cal_norm_skyflat.png'.format(str(self.proposal).zfill(5),
                                         self.obs, self.fltr, self.pupil).lower())
-            self.files = np.array([filesystem_path(row['filename']) for row in mast_table_combo])
+            existing_files = []
+            for row in mast_table_combo:
+                try:
+                    existing_files.append(filesystem_path(row['filename']))
+                except:
+                    pass
+            self.files = np.array(existing_files)
             self.detectors = np.array(mast_table_combo['detector'])
-            if not os.path.exists(self.outfile):
+            if (not os.path.exists(self.outfile)) & (len(existing_files) == len(mast_table_combo)):
                 logging.info('Working on {}'.format(self.outfile))
                 self.process()
                 monitor_run = True
             else:
-                logging.info('{} already exists'.format(self.outfile))
+                logging.info('{} already exists or is missing cal files ({}/{} files found).'.format(self.outfile, len(existing_files), len(mast_table_combo)))
 
         # Update the background trending plots, if any new data exists
         if len(mast_table) > 0: