Merge branch 'develop' into dark-monitor-integ-limits-and-dates

docs/source/database.rst

@@ -12,4 +12,4 @@ reset_database.py
 -----------------
 .. automodule:: jwql.database.reset_database
     :members:
-    :undoc-members:
+    :undoc-members:

docs/source/website.rst

@@ -44,6 +44,12 @@ monitor_views.py
     :members:
     :undoc-members:
 
+monitor_models
+--------------
+.. automodule:: jwql.website.apps.jwql.monitor_models.common
+    :members:
+    :undoc-members:
+
 settings.py
 -----------
 .. automodule:: jwql.website.jwql_proj.settings
@@ -60,4 +66,4 @@ views.py
 --------
 .. automodule:: jwql.website.apps.jwql.views
     :members:
-    :undoc-members:
+    :undoc-members:

environment_python_3.10.yml

@@ -24,7 +24,7 @@ channels:
 dependencies:
   - astropy=5.3.4
   - beautifulsoup4=4.12.2
-  - bokeh=2.4.3
+  - bokeh=3.3.0
   - celery=5.3.4
   - cryptography=41.0.4
   - django=4.2.6

environment_python_3.9.yml

@@ -24,7 +24,7 @@ channels:
 dependencies:
   - astropy=5.3.3
   - beautifulsoup4=4.12.2
-  - bokeh=2.4.3
+  - bokeh=3.3.0
   - celery=5.3.4
   - cryptography=41.0.4
   - django=4.2.5

jwql/bokeh_templating/factory.py

@@ -36,8 +36,7 @@
 
 # Figures get their own constructor so we remove references to Figures from
 # the keyword maps.
-Figure = mappings.pop("Figure")
-del sequences["figure"]
+Figure = mappings.pop("figure")
 
 
 def mapping_factory(tool, element_type):

jwql/database/database_interface.py

@@ -84,10 +84,9 @@
 from jwql.utils.constants import ANOMALIES_PER_INSTRUMENT
 from jwql.utils.constants import FILE_SUFFIX_TYPES
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES
+from jwql.utils.constants import ON_GITHUB_ACTIONS
 from jwql.utils.utils import get_config
 
-ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
-
 
 # Monkey patch Query with data_frame method
 @property

jwql/edb/engineering_database.py

@@ -58,20 +58,20 @@
 from astroquery.mast import Mast
 from bokeh.embed import components
 from bokeh.layouts import column
-from bokeh.models import BoxAnnotation, ColumnDataSource, DatetimeTickFormatter, HoverTool, Range1d
+from bokeh.models import BoxAnnotation, ColumnDataSource, DatetimeTickFormatter, HoverTool
+from bokeh.models import Range1d
 from bokeh.plotting import figure, output_file, show, save
 import numpy as np
 
 from jwst.lib.engdb_tools import ENGDB_Service
 from jwql.utils.constants import MIRI_POS_RATIO_VALUES
+from jwql.utils.constants import ON_GITHUB_ACTIONS
 from jwql.utils.credentials import get_mast_base_url, get_mast_token
 from jwql.utils.utils import get_config
 
 MAST_EDB_MNEMONIC_SERVICE = 'Mast.JwstEdb.Mnemonics'
 MAST_EDB_DICTIONARY_SERVICE = 'Mast.JwstEdb.Dictionary'
 
-# Temporary until JWST operations: switch to test string for MAST request URL
-ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
 if not ON_GITHUB_ACTIONS:
     Mast._portal_api_connection.MAST_REQUEST_URL = get_config()['mast_request_url']
 
@@ -283,7 +283,7 @@ def __mul__(self, mnem):
                 before = np.where(common_dates == self.data['dates'][block])[0]
 
                 if len(before) > 0:
-                    new_blocks.append(before[0]) # + 1)
+                    new_blocks.append(before[0])  # + 1)
             except IndexError:
                 # The final block value is usually equal to the length of the array, and will
                 # therefore cause an Index Error in the lines above. Ignore that error here.
@@ -421,7 +421,7 @@ def block_stats(self, sigma=3, ignore_vals=[], ignore_edges=False, every_change=
             # calculated, remove those every change values and block values from the EdbMnemonic
             # instance.
             if every_change:
-                if len(remove_change_indexes)  > 0:
+                if len(remove_change_indexes) > 0:
                     self.every_change_values = np.delete(self.every_change_values, remove_change_indexes)
                     self.blocks = np.delete(self.blocks, remove_change_indexes)
 
@@ -439,7 +439,7 @@ def block_stats(self, sigma=3, ignore_vals=[], ignore_edges=False, every_change=
                     stdevs.append(stdevval)
                     maxs.append(meanval)
                     mins.append(meanval)
-                    #if hasattr(self, 'every_change_values'):
+                    # if hasattr(self, 'every_change_values'):
                     #        updated_every_change_vals.append(self.every_change_values[i + 1])
         self.mean = means
         self.median = medians
@@ -510,9 +510,9 @@ def block_stats_filter_positions(self, sigma=5):
                                                                                          self.data["euvalues"].data[index:self.blocks[i + 1]],
                                                                                          sigma=sigma)
                     if np.isfinite(meanval):
-                        #this is preventing the nans above from being added. not sure what to do here.
-                        #bokeh cannot deal with nans. but we need entries in order to have the blocks indexes
-                        #remain correct. but maybe we dont care about the block indexes after averaging
+                        # this is preventing the nans above from being added. not sure what to do here.
+                        # bokeh cannot deal with nans. but we need entries in order to have the blocks indexes
+                        # remain correct. but maybe we dont care about the block indexes after averaging
                         medtimes.append(calc_median_time(self.data["dates"].data[index:self.blocks[i + 1]][good]))
                         means.append(meanval)
                         medians.append(medianval)
@@ -523,7 +523,7 @@ def block_stats_filter_positions(self, sigma=5):
             # If there were blocks composed entirely of bad data, meaning no mean values were
             # calculated, remove those every change values and block values from the EdbMnemonic
             # instance.
-            if len(remove_change_indexes)  > 0:
+            if len(remove_change_indexes) > 0:
                 self.every_change_values = np.delete(self.every_change_values, remove_change_indexes)
                 self.blocks = np.delete(self.blocks, remove_change_indexes)
 
@@ -690,37 +690,33 @@ def bokeh_plot(self, show_plot=False, savefig=False, out_dir='./', nominal_value
                 if plot_mean:
                     source_mean = ColumnDataSource(data={'mean_x': self.median_times, 'mean_y': self.mean})
                     mean_data = fig.scatter(x='mean_x', y='mean_y', line_width=1, line_color='orange', alpha=0.75, source=source_mean)
-                    mean_hover_tool = HoverTool(tooltips=[('Mean', '@mean_y'),
-                                                          ('Date', '@mean_x{%d %b %Y %H:%M:%S}')
-                                                         ], mode='mouse', renderers=[mean_data])
+                    mean_hover_tool = HoverTool(tooltips=[('Mean', '@mean_y'), ('Date', '@mean_x{%d %b %Y %H:%M:%S}')],
+                                                mode='mouse', renderers=[mean_data])
                     mean_hover_tool.formatters = {'@mean_x': 'datetime'}
                     fig.tools.append(mean_hover_tool)
 
                 if plot_median:
                     source_median = ColumnDataSource(data={'median_x': self.median_times, 'median_y': self.median})
                     median_data = fig.scatter(x='median_x', y='median_y', line_width=1, line_color='orangered', alpha=0.75, source=source_median)
-                    median_hover_tool = HoverTool(tooltips=[('Median', '@median_y'),
-                                                            ('Date', '@median_x{%d %b %Y %H:%M:%S}')
-                                                           ], mode='mouse', renderers=[median_data])
+                    median_hover_tool = HoverTool(tooltips=[('Median', '@median_y'), ('Date', '@median_x{%d %b %Y %H:%M:%S}')],
+                                                  mode='mouse', renderers=[median_data])
                     median_hover_tool.formatters = {'@median_x': 'datetime'}
                     fig.tools.append(median_hover_tool)
 
-                 # If the max and min arrays are to be plotted, create columndata sources for them as well
+                # If the max and min arrays are to be plotted, create columndata sources for them as well
                 if plot_max:
                     source_max = ColumnDataSource(data={'max_x': self.median_times, 'max_y': self.max})
                     max_data = fig.scatter(x='max_x', y='max_y', line_width=1, color='black', line_color='black', source=source_max)
-                    max_hover_tool = HoverTool(tooltips=[('Max', '@max_y'),
-                                                         ('Date', '@max_x{%d %b %Y %H:%M:%S}')
-                                                        ], mode='mouse', renderers=[max_data])
+                    max_hover_tool = HoverTool(tooltips=[('Max', '@max_y'), ('Date', '@max_x{%d %b %Y %H:%M:%S}')],
+                                               mode='mouse', renderers=[max_data])
                     max_hover_tool.formatters = {'@max_x': 'datetime'}
                     fig.tools.append(max_hover_tool)
 
                 if plot_min:
                     source_min = ColumnDataSource(data={'min_x': self.median_times, 'min_y': self.min})
                     min_data = fig.scatter(x='min_x', y='min_y', line_width=1, color='black', line_color='black', source=source_min)
-                    minn_hover_tool = HoverTool(tooltips=[('Min', '@min_y'),
-                                                          ('Date', '@min_x{%d %b %Y %H:%M:%S}')
-                                                         ], mode='mouse', renderers=[min_data])
+                    minn_hover_tool = HoverTool(tooltips=[('Min', '@min_y'), ('Date', '@min_x{%d %b %Y %H:%M:%S}')],
+                                                mode='mouse', renderers=[min_data])
                     min_hover_tool.formatters = {'@min_x': 'datetime'}
                     fig.tools.append(min_hover_tool)
 
@@ -740,12 +736,12 @@ def bokeh_plot(self, show_plot=False, savefig=False, out_dir='./', nominal_value
             fig = add_limit_boxes(fig, yellow=yellow_limits, red=red_limits)
 
         # Make the x axis tick labels look nice
-        fig.xaxis.formatter = DatetimeTickFormatter(microseconds=["%d %b %H:%M:%S.%3N"],
-                                                    seconds=["%d %b %H:%M:%S.%3N"],
-                                                    hours=["%d %b %H:%M"],
-                                                    days=["%d %b %H:%M"],
-                                                    months=["%d %b %Y %H:%M"],
-                                                    years=["%d %b %Y"]
+        fig.xaxis.formatter = DatetimeTickFormatter(microseconds="%d %b %H:%M:%S.%3N",
+                                                    seconds="%d %b %H:%M:%S.%3N",
+                                                    hours="%d %b %H:%M",
+                                                    days="%d %b %H:%M",
+                                                    months="%d %b %Y %H:%M",
+                                                    years="%d %b %Y"
                                                     )
         fig.xaxis.major_label_orientation = np.pi / 4
 
@@ -1206,12 +1202,12 @@ def plot_data_plus_devs(self, use_median=False, show_plot=False, savefig=False,
         fig_dev.line(data_dates, dev, color='red')
 
         # Make the x axis tick labels look nice
-        fig_dev.xaxis.formatter = DatetimeTickFormatter(microseconds=["%d %b %H:%M:%S.%3N"],
-                                                        seconds=["%d %b %H:%M:%S.%3N"],
-                                                        hours=["%d %b %H:%M"],
-                                                        days=["%d %b %H:%M"],
-                                                        months=["%d %b %Y %H:%M"],
-                                                        years=["%d %b %Y"]
+        fig_dev.xaxis.formatter = DatetimeTickFormatter(microseconds="%d %b %H:%M:%S.%3N",
+                                                        seconds="%d %b %H:%M:%S.%3N",
+                                                        hours="%d %b %H:%M",
+                                                        days="%d %b %H:%M",
+                                                        months="%d %b %Y %H:%M",
+                                                        years="%d %b %Y"
                                                         )
         fig.xaxis.major_label_orientation = np.pi / 4
 

jwql/example_config.json

@@ -2,21 +2,31 @@
     "admin_account" : "",
     "auth_mast" : "",
     "connection_string" : "",
-    "database" : {
+    "databases" : {
         "engine" : "",
         "name" : "",
         "user" : "",
         "password" : "",
         "host" : "",
         "port" : ""
     },
-    "django_database" : {
-        "ENGINE" : "",
-        "NAME" : "",
-        "USER" : "",
-        "PASSWORD" : "",
-        "HOST" : "",
-        "PORT" : ""
+    "django_databases" : {
+        "default": {
+            "ENGINE" : "",
+            "NAME" : "",
+            "USER" : "",
+            "PASSWORD" : "",
+            "HOST" : "",
+            "PORT" : ""
+        },
+        "monitors": {
+            "ENGINE" : "",
+            "NAME" : "",
+            "USER" : "",
+            "PASSWORD" : "",
+            "HOST" : "",
+            "PORT" : ""
+        }
     },
     "jwql_dir" : "",
     "jwql_version": "",

jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py

@@ -105,19 +105,13 @@
 from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline
 from jwql.utils import crds_tools, instrument_properties, monitor_utils
 from jwql.utils.constants import DARKS_BAD_PIXEL_TYPES, DARK_EXP_TYPES, FLATS_BAD_PIXEL_TYPES, FLAT_EXP_TYPES
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, ON_GITHUB_ACTIONS
+from jwql.utils.constants import ON_READTHEDOCS
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.mast_utils import mast_query
 from jwql.utils.permissions import set_permissions
 from jwql.utils.utils import copy_files, create_png_from_fits, ensure_dir_exists, get_config, filesystem_path
 
-# Determine if the code is being run by Github Actions
-ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
-
-# Determine if the code is being run as part of a Readthedocs build
-ON_READTHEDOCS = False
-if 'READTHEDOCS' in os.environ:  # pragma: no cover
-    ON_READTHEDOCS = os.environ['READTHEDOCS']
 
 if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
     from jwql.website.apps.jwql.monitor_pages.monitor_bad_pixel_bokeh import BadPixelPlots

jwql/instrument_monitors/common_monitors/edb_telemetry_monitor.py

@@ -381,7 +381,7 @@
 from bokeh.embed import components, json_item
 from bokeh.layouts import gridplot
 from bokeh.models import BoxAnnotation, ColumnDataSource, DatetimeTickFormatter, HoverTool, Range1d
-from bokeh.models.widgets import Tabs, Panel
+from bokeh.models.layouts import Tabs
 from bokeh.plotting import figure, output_file, save, show
 from bokeh.palettes import Turbo256
 from jwql.database import database_interface
@@ -991,14 +991,14 @@ def get_dependency_data(self, dependency, starttime, endtime):
         if dependency["name"] in self.query_results:
 
             # We need the full time to be covered
-            if ((self.query_results[dependency["name"]].requested_start_time <= starttime) and
-              (self.query_results[dependency["name"]].requested_end_time >= endtime)):
+            if ((self.query_results[dependency["name"]].requested_start_time <= starttime)
+            and (self.query_results[dependency["name"]].requested_end_time >= endtime)):
 
                 logging.info(f'Dependency {dependency["name"]} is already present in self.query_results.')
 
                 # Extract data for the requested time range
-                matching_times = np.where((self.query_results[dependency["name"]].data["dates"] >= starttime) &
-                                          (self.query_results[dependency["name"]].data["dates"] <= endtime))
+                matching_times = np.where((self.query_results[dependency["name"]].data["dates"] >= starttime)
+                                        & (self.query_results[dependency["name"]].data["dates"] <= endtime))
                 dep_mnemonic = {"dates": self.query_results[dependency["name"]].data["dates"][matching_times],
                                 "euvalues": self.query_results[dependency["name"]].data["euvalues"][matching_times]}
 
@@ -1138,16 +1138,16 @@ def get_history_every_change(self, mnemonic, start_date, end_date):
                     devs = []
 
                 # Keep only data that fall at least partially within the plot range
-                if (((np.min(row.time) > self._plot_start) & (np.min(row.time) < self._plot_end)) 
-                  | ((np.max(row.time) > self._plot_start) & (np.max(row.time) < self._plot_end))):
+                if (((np.min(row.time) > self._plot_start) & (np.min(row.time) < self._plot_end))
+                | ((np.max(row.time) > self._plot_start) & (np.max(row.time) < self._plot_end))):
                     times.extend(row.time)
                     values.extend(row.mnemonic_value)
                     medians.append(row.median)
                     devs.append(row.stdev)
                     hist[row.dependency_value] = (times, values, medians, devs)
             else:
-                if (((np.min(row.time) > self._plot_start) & (np.min(row.time) < self._plot_end)) 
-                  | ((np.max(row.time) > self._plot_start) & (np.max(row.time) < self._plot_end))):
+                if (((np.min(row.time) > self._plot_start) & (np.min(row.time) < self._plot_end))
+                | ((np.max(row.time) > self._plot_start) & (np.max(row.time) < self._plot_end))):
                     hist[row.dependency_value] = (row.time, row.mnemonic_value, row.median, row.stdev)
 
         return hist
@@ -2143,12 +2143,12 @@ def plot_every_change_data(data, mnem_name, units, show_plot=False, savefig=True
         fig = add_limit_boxes(fig, yellow=yellow_limits, red=red_limits)
 
     # Make the x axis tick labels look nice
-    fig.xaxis.formatter = DatetimeTickFormatter(microseconds=["%d %b %H:%M:%S.%3N"],
-                                                seconds=["%d %b %H:%M:%S.%3N"],
-                                                hours=["%d %b %H:%M"],
-                                                days=["%d %b %H:%M"],
-                                                months=["%d %b %Y %H:%M"],
-                                                years=["%d %b %Y"]
+    fig.xaxis.formatter = DatetimeTickFormatter(microseconds="%d %b %H:%M:%S.%3N",
+                                                seconds="%d %b %H:%M:%S.%3N",
+                                                hours="%d %b %H:%M",
+                                                days="%d %b %H:%M",
+                                                months="%d %b %Y %H:%M",
+                                                years="%d %b %Y"
                                                 )
     fig.xaxis.major_label_orientation = np.pi / 4