[WIP]: Get EDB monitor working

CHANGES.rst

@@ -1,5 +1,21 @@
 ## What's Changed
 
+1.2.5 (2024-03-19)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Fix Bokeh `file_html` Call by @mfixstsci
+- Update Bad Pix Exclude Line by @mfixstsci
+- Interactive preview image - updates for Bokeh 3 by @bhilbert4
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Allow creation of pngs from 3D and 4D arrays by @bhilbert4
+- Add max length to charfield by @BradleySappington
+- Header fix by @BradleySappington
+
+
 1.2.4 (2024-03-11)
 ==================
 

jwql/edb/engineering_database.py

@@ -45,7 +45,7 @@
 """
 import calendar
 from collections import OrderedDict
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from numbers import Number
 import os
 import warnings
@@ -1265,8 +1265,13 @@ def timed_stats(self, sigma=3):
                 good = ((date_arr >= min_date) & (date_arr < max_date))
                 if self.meta['TlmMnemonics'][0]['AllPoints'] != 0:
                     avg, med, dev = sigma_clipped_stats(self.data["euvalues"][good], sigma=sigma)
-                    maxval = np.max(self.data["euvalues"][good])
-                    minval = np.min(self.data["euvalues"][good])
+                    # if self.data is empty, or good is empty, then calculating the max and
+                    # min values will not work.
+                    try:
+                        maxval = np.max(self.data["euvalues"][good])
+                        minval = np.min(self.data["euvalues"][good])
+                    except ValueError:
+                        pass
                 else:
                     avg, med, dev, maxval, minval = change_only_stats(self.data["dates"][good], self.data["euvalues"][good], sigma=sigma)
                 if np.isfinite(avg):
@@ -1412,9 +1417,15 @@ def change_only_bounding_points(date_list, value_list, starttime, endtime):
     if isinstance(starttime, Time):
         starttime = starttime.datetime
 
+    if starttime.tzinfo == None or starttime.tzinfo.utcoffset(starttime) == None:
+        starttime = starttime.replace(tzinfo=timezone.utc)
+
     if isinstance(endtime, Time):
         endtime = endtime.datetime
 
+    if endtime.tzinfo == None or endtime.tzinfo.utcoffset(endtime) == None:
+        endtime = endtime.replace(tzinfo=timezone.utc)
+
     valid_idx = np.where((date_list_arr <= endtime) & (date_list_arr >= starttime))[0]
     before_startime = np.where(date_list_arr < starttime)[0]
     before_endtime = np.where(date_list_arr < endtime)[0]
@@ -1601,7 +1612,7 @@ def get_mnemonic(mnemonic_identifier, start_time, end_time):
     data = service.get_values(mnemonic_identifier, start_time, end_time, include_obstime=True,
                               include_bracket_values=bracket)
 
-    dates = [datetime.strptime(row.obstime.iso, "%Y-%m-%d %H:%M:%S.%f") for row in data]
+    dates = [datetime.strptime(row.obstime.iso, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=timezone.utc) for row in data]
     values = [row.value for row in data]
 
     if bracket:

jwql/instrument_monitors/nircam_monitors/claw_monitor.py

@@ -192,6 +192,7 @@ def make_background_plots(self, plot_type='bkg'):
                     df = df[df['stddev'] != 0]  # older data has no accurate stddev measures
                     plot_data = df['stddev'].values
                 if plot_type == 'model':
+                    df = df[np.isfinite(df['total_bkg'])]  # the claw monitor did not track model measurements at first
                     plot_data = df['median'].values / df['total_bkg'].values
                 plot_expstarts = df['expstart_mjd'].values
 
@@ -300,7 +301,11 @@ def process(self):
 
                 # Get predicted background level using JWST background tool
                 ra, dec = hdu[1].header['RA_V1'], hdu[1].header['DEC_V1']
-                wv = self.filter_wave[self.fltr.upper()]
+                if ('N' in self.pupil.upper()) | ('M' in self.pupil.upper()):
+                    fltr_wv = self.pupil.upper()
+                else:
+                    fltr_wv = self.fltr.upper()
+                wv = self.filter_wave[fltr_wv]
                 date = hdu[0].header['DATE-BEG']
                 doy = int(Time(date).yday.split(':')[1])
                 try:
@@ -332,7 +337,7 @@ def process(self):
                                  'skyflat_filename': os.path.basename(self.outfile),
                                  'doy': float(doy),
                                  'total_bkg': float(total_bkg),
-                                 'entry_date': datetime.datetime.now()
+                                 'entry_date': datetime.datetime.now(datetime.timezone.utc)
                                  }
                 entry = self.stats_table(**claw_db_entry)
                 entry.save()
@@ -423,11 +428,13 @@ def run(self):
         mast_table = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(mast_table), self.query_start_mjd, self.query_end_mjd))
 
-        # Define pivot wavelengths
-        self.filter_wave = {'F070W': 0.704, 'F090W': 0.902, 'F115W': 1.154, 'F150W': 1.501, 'F150W2': 1.659,
-                            'F200W': 1.989, 'F212N': 2.121, 'F250M': 2.503, 'F277W': 2.762, 'F300M': 2.989,
-                            'F322W2': 3.232, 'F356W': 3.568, 'F410M': 4.082, 'F430M': 4.281, 'F444W': 4.408,
-                            'F480M': 4.874}
+        # Define pivot wavelengths - last downloaded March 8 2024 from:
+        # https://jwst-docs.stsci.edu/jwst-near-infrared-camera/nircam-instrumentation/nircam-filters
+        self.filter_wave = {'F070W': 0.704, 'F090W': 0.901, 'F115W': 1.154, 'F140M': 1.404, 'F150W': 1.501, 'F162M': 1.626, 'F164N': 1.644,
+                            'F150W2': 1.671, 'F182M': 1.845, 'F187N': 1.874, 'F200W': 1.99, 'F210M': 2.093, 'F212N': 2.12, 'F250M': 2.503,
+                            'F277W': 2.786, 'F300M': 2.996, 'F322W2': 3.247, 'F323N': 3.237, 'F335M': 3.365, 'F356W': 3.563, 'F360M': 3.621,
+                            'F405N': 4.055, 'F410M': 4.092, 'F430M': 4.28, 'F444W': 4.421, 'F460M': 4.624, 'F466N': 4.654, 'F470N': 4.707,
+                            'F480M': 4.834}
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in mast_table])
@@ -469,7 +476,7 @@ def run(self):
                      'start_time_mjd': self.query_start_mjd,
                      'end_time_mjd': self.query_end_mjd,
                      'run_monitor': monitor_run,
-                     'entry_date': datetime.datetime.now()}
+                     'entry_date': datetime.datetime.now(datetime.timezone.utc)}
         entry = self.query_table(**new_entry)
         entry.save()
 

jwql/pull_jwql_branch.sh

@@ -62,6 +62,7 @@ echo "Reset: $reset";
 echo "Notify: $notify $recipient";
 
 # 1. Pull updated code from GitHub deployment branch (keep second checkout in case its already defined for some weird reason)
+git fetch origin
 git checkout -b $branch_name --track origin/$branch_name
 git checkout $branch_name
 git fetch origin $branch_name

jwql/tests/test_data_containers.py

@@ -31,7 +31,7 @@
 import pandas as pd
 import pytest
 
-from jwql.utils.constants import ON_GITHUB_ACTIONS
+from jwql.utils.constants import ON_GITHUB_ACTIONS, DEFAULT_MODEL_CHARFIELD
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
 
@@ -45,7 +45,7 @@
     from jwql.utils.utils import get_config  # noqa: E402 (module level import not at top of file)
     from jwql.website.apps.jwql.models import RootFileInfo
 
-    
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
 def test_build_table():
     tab = data_containers.build_table('filesystem_general')
@@ -199,7 +199,6 @@ def test_get_all_proposals():
                            (['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'], {'bad'})),
                           (False, ['rate', 'uncal', 'bad', 'o006_crfints', 'o001_crf'],
                            ['uncal', 'rate', 'o001_crf', 'o006_crfints', 'bad'])])
-
 def test_get_available_suffixes(untracked, input_suffixes, expected):
     result = data_containers.get_available_suffixes(
         input_suffixes, return_untracked=untracked)
@@ -339,6 +338,7 @@ def test_get_anomaly_form_post_group(mocker):
     assert update_mock.call_count == 2
 """
 
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to django models.')
 def test_get_dashboard_components():
     request = MockPostRequest()
@@ -607,42 +607,42 @@ def test_mast_query_by_rootname():
     instrument = 'NIRCam'
     rootname1 = 'jw02767002001_02103_00005_nrcb4'
     dict_stuff = data_containers.mast_query_by_rootname(instrument, rootname1)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
     rootname2 = 'jw02084001001_04103_00001-seg003_nrca3'
     dict_stuff = data_containers.mast_query_by_rootname(instrument, rootname2)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
     instrument2 = 'FGS'
     rootname3 = 'jw01029003001_06201_00001_guider2'
     dict_stuff = data_containers.mast_query_by_rootname(instrument2, rootname3)
-    defaults = dict(filter=dict_stuff.get('filter', ''),
-                    detector=dict_stuff.get('detector', ''),
-                    exp_type=dict_stuff.get('exp_type', ''),
-                    read_pat=dict_stuff.get('readpatt', ''),
-                    grating=dict_stuff.get('grating', ''),
+    defaults = dict(filter=dict_stuff.get('filter', DEFAULT_MODEL_CHARFIELD),
+                    detector=dict_stuff.get('detector', DEFAULT_MODEL_CHARFIELD),
+                    exp_type=dict_stuff.get('exp_type', DEFAULT_MODEL_CHARFIELD),
+                    read_pat=dict_stuff.get('readpatt', DEFAULT_MODEL_CHARFIELD),
+                    grating=dict_stuff.get('grating', DEFAULT_MODEL_CHARFIELD),
                     patt_num=dict_stuff.get('patt_num', 0),
-                    aperture=dict_stuff.get('apername', ''),
-                    subarray=dict_stuff.get('subarray', ''),
-                    pupil=dict_stuff.get('pupil', ''))
+                    aperture=dict_stuff.get('apername', DEFAULT_MODEL_CHARFIELD),
+                    subarray=dict_stuff.get('subarray', DEFAULT_MODEL_CHARFIELD),
+                    pupil=dict_stuff.get('pupil', DEFAULT_MODEL_CHARFIELD))
     assert isinstance(defaults, dict)
 
 

jwql/tests/test_edb.py

@@ -17,7 +17,7 @@
 
         pytest -s test_edb.py
 """
-from datetime import datetime
+from datetime import datetime, timezone
 import os
 
 from astropy.table import Table
@@ -65,10 +65,10 @@ def test_change_only_bounding_points():
     """Make sure we correctly add starting and ending time entries to
     a set of change-only data
     """
-    dates = [datetime(2022, 3, 2, 12, i) for i in range(10)]
+    dates = [datetime(2022, 3, 2, 12, i, tzinfo=timezone.utc) for i in range(10)]
     values = np.arange(10)
-    starting_time = datetime(2022, 3, 2, 12, 3, 3)
-    ending_time = datetime(2022, 3, 2, 12, 8, 4)
+    starting_time = datetime(2022, 3, 2, 12, 3, 3, tzinfo=timezone.utc)
+    ending_time = datetime(2022, 3, 2, 12, 8, 4, tzinfo=timezone.utc)
 
     new_dates, new_values = ed.change_only_bounding_points(dates, values, starting_time, ending_time)
 

jwql/tests/test_edb_telemetry_monitor.py

@@ -43,40 +43,46 @@ def test_add_every_change_history():
     """Test that every_change data is correctly combined with an existing
     set of every_change data
     """
-    dates1 = np.array([datetime.datetime(2022, 3, 4, 1, 5, i) for i in range(10)])
-    data1 = np.array([0.1, 0.2, 0.1, 0.2, 0.1, 0.2, 0.1, 0.2, 0.1, 0.2])
-    means1 = 0.15
-    devs1 = 0.07
-    dates2 = np.array([dates1[-1] + datetime.timedelta(seconds=1 * i) for i in range(1, 11)])
-    data2 = np.array([0.3, 0.4, 0.3, 0.4, 0.3, 0.4, 0.3, 0.4, 0.3, 0.4])
-    means2 = 0.35
-    devs2 = 0.07
-    ec1 = {'0.15': (dates1, data1, means1, devs1),
-           '0.35': (dates2, data2, means2, devs2)
-           }
-    ec2 = {'0.15': (dates1, data1, means1, devs1)}
-    combine1 = etm.add_every_change_history(ec1, ec2)
-    expected1 = defaultdict(list)
-    expected1['0.15'] = (np.append(dates1, dates1), np.append(data1, data1), np.append(means1, means1), np.append(devs1, devs1))
-    expected1['0.35'] = (dates2, data2, means2, devs2)
-
-    for key in combine1:
-        print('compare ', key)
-        for i, cele in enumerate(combine1[key]):
-            assert np.all(cele == expected1[key][i])
+    dates1 = [datetime.datetime(2022, 3, 4, 1, 5, i) for i in range(10)]
+    data1 = [0.1, 0.2, 0.1, 0.2, 0.1, 0.2, 0.1, 0.2, 0.1, 0.2]
+    means1 = [0.15]
+    devs1 = [0.07]
+    dates2 = [dates1[-1] + datetime.timedelta(seconds=1 * i) for i in range(1, 11)]
+    data2 = [0.3, 0.4, 0.3, 0.4, 0.3, 0.4, 0.3, 0.4, 0.3, 0.4]
+    means2 = [0.35]
+    devs2 = [0.07]
+    #ec1 = {'0.15': (dates1, data1, means1, devs1),
+    #       '0.35': (dates2, data2, means2, devs2)
+    #       }
+    history = {'0.15': ([dates1], [data1], [means1], [devs1]),
+              '0.35': ([dates2], [data2], [means2], [devs2])
+               }
+    ec2 = {'0.15': (dates2, data2, means2, devs2)}
+    combine1 = etm.add_every_change_history(history, ec2)
 
-    dates3 = np.array([dates2[-1] + datetime.timedelta(seconds=1 * i) for i in range(1, 11)])
-    ec3 = {'0.55': (dates3, data2 + 0.2, means2 + 0.2, devs2)}
-    combine2 = etm.add_every_change_history(ec1, ec3)
+    expected1 = defaultdict(list)
+    expected_dates = [dates1]
+    expected_dates.append(dates2)
+    expected_data = [data1]
+    expected_data.append(data2)
+    expected_means = [means1]
+    expected_means.append(means2)
+    expected_devs = [devs1]
+    expected_devs.append(devs2)
+    expected1['0.15'] = (expected_dates, expected_data, expected_means, expected_devs)
+    expected1['0.35'] = ([dates2], [data2], [means2], [devs2])
+    assert combine1 == expected1
+
+    dates3 = [dates2[-1] + datetime.timedelta(seconds=1 * i) for i in range(1, 11)]
+    data3 = [e + 0.2 for e in data2]
+    means3 = [0.55]
+    ec3 = {'0.55': (dates3, data3, means3, devs2)}
+    combine2 = etm.add_every_change_history(history, ec3)
     expected2 = defaultdict(list)
-    expected2['0.15'] = (dates1, data1, means1, devs1)
-    expected2['0.35'] = (dates2, data2, means2, devs2)
-    expected2['0.55'] = (dates3, data2 + 0.2, means2 + 0.2, devs2)
-
-    for key in combine2:
-        print('compare ', key)
-        for i, cele in enumerate(combine2[key]):
-            assert np.all(cele == expected2[key][i])
+    expected2['0.15'] = ([dates1], [data1], [means1], [devs1])
+    expected2['0.35'] = ([dates2], [data2], [means2], [devs2])
+    expected2['0.55'] = ([dates3], [data3], [means3], [devs2])
+    assert combine2 == expected2
 
 
 def test_change_only_add_points():
@@ -299,11 +305,11 @@ def test_organize_every_change():
     f770mean, _, _ = sigma_clipped_stats(f770_vals, sigma=3)
     f1000mean, _, _ = sigma_clipped_stats(f1000_vals, sigma=3)
     f1500mean, _, _ = sigma_clipped_stats(f1500_vals, sigma=3)
-    expected = {'F2550W': (np.array(dates[f2550_idx]), f2550_vals, MIRI_POS_RATIO_VALUES['FW']['F2550W'][0]),
-                'F560W': (np.array(dates[f560_idx]), f560_vals, MIRI_POS_RATIO_VALUES['FW']['F560W'][0]),
-                'F770W': (np.array(dates[f770_idx]), f770_vals, MIRI_POS_RATIO_VALUES['FW']['F770W'][0]),
-                'F1000W': (np.array(dates[f1000_idx]), f1000_vals, MIRI_POS_RATIO_VALUES['FW']['F1000W'][0]),
-                'F1500W': (np.array(dates[f1500_idx]), f1500_vals, MIRI_POS_RATIO_VALUES['FW']['F1500W'][0])}
+    expected = {'F2550W': (np.array(dates[f2550_idx]), f2550_vals, [MIRI_POS_RATIO_VALUES['FW']['F2550W'][0]], [MIRI_POS_RATIO_VALUES['FW']['F2550W'][1]]),
+                'F560W': (np.array(dates[f560_idx]), f560_vals, [MIRI_POS_RATIO_VALUES['FW']['F560W'][0]], [MIRI_POS_RATIO_VALUES['FW']['F560W'][1]]),
+                'F770W': (np.array(dates[f770_idx]), f770_vals, [MIRI_POS_RATIO_VALUES['FW']['F770W'][0]], [MIRI_POS_RATIO_VALUES['FW']['F770W'][1]]),
+                'F1000W': (np.array(dates[f1000_idx]), f1000_vals, [MIRI_POS_RATIO_VALUES['FW']['F1000W'][0]], [MIRI_POS_RATIO_VALUES['FW']['F1000W'][1]]),
+                'F1500W': (np.array(dates[f1500_idx]), f1500_vals, [MIRI_POS_RATIO_VALUES['FW']['F1500W'][0]], [MIRI_POS_RATIO_VALUES['FW']['F1500W'][1]])}
 
     for key, val in expected.items():
         assert np.all(val[0] == data[key][0])

jwql/utils/constants.py

@@ -364,6 +364,9 @@
     "wfscmb",
 ]
 
+# Default Model Values
+DEFAULT_MODEL_CHARFIELD = "empty"
+
 # Filename Component Lengths
 FILE_AC_CAR_ID_LEN = 4
 FILE_AC_O_ID_LEN = 3

jwql/utils/interactive_preview_image.py

@@ -554,7 +554,7 @@ def add_interactive_controls(self, images, color_bars):
         # JS callbacks for client side controls
 
         # set alternate image visibility when scale selection changes
-        scale_group.js_on_click(CustomJS(args={'i1': images[0], 'c1': color_bars[0],
+        scale_group.js_on_change('active', CustomJS(args={'i1': images[0], 'c1': color_bars[0],
                                                'i2': images[1], 'c2': color_bars[1]},
                                          code="""
             if (i1.visible == true) {
@@ -594,10 +594,10 @@ def add_interactive_controls(self, images, color_bars):
             limit_high.js_link('value', color_bars[i].color_mapper, 'high')
 
         # reset boxes to preset range on button click
-        reset.js_on_click(limit_reset)
+        reset.js_on_event('button_click', limit_reset)
 
         # also reset when swapping limit style
-        scale_group.js_on_click(limit_reset)
+        scale_group.js_on_change('active', limit_reset)
 
         # return widgets
         spacer = Spacer(height=20)