Datetime timezones, and modify db model name for blockmeans

jwql/edb/engineering_database.py

@@ -45,7 +45,7 @@
 """
 import calendar
 from collections import OrderedDict
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from numbers import Number
 import os
 import warnings
@@ -1417,9 +1417,15 @@ def change_only_bounding_points(date_list, value_list, starttime, endtime):
     if isinstance(starttime, Time):
         starttime = starttime.datetime
 
+    if starttime.tzinfo == None or starttime.tzinfo.utcoffset(starttime) == None:
+        starttime = starttime.replace(tzinfo=timezone.utc)
+
     if isinstance(endtime, Time):
         endtime = endtime.datetime
 
+    if endtime.tzinfo == None or endtime.tzinfo.utcoffset(endtime) == None:
+        endtime = endtime.replace(tzinfo=timezone.utc)
+
     valid_idx = np.where((date_list_arr <= endtime) & (date_list_arr >= starttime))[0]
     before_startime = np.where(date_list_arr < starttime)[0]
     before_endtime = np.where(date_list_arr < endtime)[0]
@@ -1606,7 +1612,7 @@ def get_mnemonic(mnemonic_identifier, start_time, end_time):
     data = service.get_values(mnemonic_identifier, start_time, end_time, include_obstime=True,
                               include_bracket_values=bracket)
 
-    dates = [datetime.strptime(row.obstime.iso, "%Y-%m-%d %H:%M:%S.%f") for row in data]
+    dates = [datetime.strptime(row.obstime.iso, "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=timezone.utc) for row in data]
     values = [row.value for row in data]
 
     if bracket:

jwql/instrument_monitors/common_monitors/edb_telemetry_monitor.py

@@ -385,7 +385,6 @@
 from bokeh.models.layouts import TabPanel, Tabs
 from bokeh.plotting import figure, output_file, save, show
 from bokeh.palettes import Turbo256
-from jwql.database import database_interface
 from jwql.edb import engineering_database as ed
 from jwql.instrument_monitors.common_monitors.edb_telemetry_monitor_utils import condition
 from jwql.instrument_monitors.common_monitors.edb_telemetry_monitor_utils import utils
@@ -395,7 +394,7 @@
 from jwql.utils.constants import EDB_DEFAULT_PLOT_RANGE, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, MIRI_POS_RATIO_VALUES
 from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import ensure_dir_exists, get_config
+from jwql.utils.utils import add_timezone_to_datetime, ensure_dir_exists, get_config
 
 if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
     # Need to set up django apps before we can access the models
@@ -516,10 +515,9 @@ def add_new_block_db_entry(self, mnem, query_time):
         logging.info(f"Adding new entry for {mnem.mnemonic_identifier} to history table.")
         times = mnem.data["dates"].data
         data = mnem.data["euvalues"].data
-        stdevs = mnem.stdev
         times = ensure_list(times)
         data = ensure_list(data)
-        stdevs = ensure_list(stdevs)
+        stdevs = ensure_list(mnem.stdev)
         medians = ensure_list(mnem.median)
         maxs = ensure_list(mnem.max)
         mins = ensure_list(mnem.min)
@@ -537,6 +535,15 @@ def add_new_block_db_entry(self, mnem, query_time):
             if (isinstance(medians[0], int) | isinstance(medians[0], np.integer)):
                 medians = [float(v) for v in medians]
 
+
+        print('In add_new_block_db_entry:')
+        for ll, name in zip([times, data, stdevs, medians, maxs, mins],['times', 'data', 'stdevs', 'medians', 'maxs', 'mins']):
+            if isinstance(ll, np.ndarray):
+                print(f'{name} is a numpy array.')
+                print(ll)
+
+
+
         db_entry = {'mnemonic': mnem.mnemonic_identifier,
                     'latest_query': query_time,
                     'times': times,
@@ -1090,8 +1097,13 @@ def get_history(self, mnemonic, start_date, end_date, info={}, meta={}):
         # outside of the plot range. Return only the points inside the desired
         # plot range
         for row in data:
-            good = np.where((np.array(row.times) > self._plot_start) & (np.array(row.times) < self._plot_end))[0]
-            times = list(np.array(row.times)[good])
+            # Make sure the data from the database has timezone info
+            time_vals = row.times
+            if time_vals[0].tzinfo == None or tie_vals[0].tzinfo.utcoffset(time_vals[0]) == None:
+                time_vals = [val.replace(tzinfo=datetime.timezone.utc) for val in time_vals]
+
+            good = np.where((np.array(time_vals) > self._plot_start) & (np.array(time_vals) < self._plot_end))[0]
+            times = list(np.array(time_vals)[good])
             data = list(np.array(row.data)[good])
             medians = list(np.array(row.median)[good])
             maxs = list(np.array(row.max)[good])
@@ -1273,8 +1285,8 @@ def identify_tables(self, inst, tel_type):
             Examples include "every_change", "daily", "all", etc
         """
         mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst]
-        if '_means' in tel_type:
-            tel_type = tel_type.strip('_means')
+        #if '_means' in tel_type:
+        #    tel_type = tel_type.strip('_means')
         tel_type = tel_type.title().replace('_', '')
         self.history_table_name = f'{mixed_case_name}Edb{tel_type}Stats'
         self.history_table = eval(self.history_table_name)
@@ -1299,11 +1311,13 @@ def most_recent_search(self, telem_name):
 
         if len(query) == 0:
             base_time = '2022-11-15 00:00:0.0'
-            query_result = datetime.datetime.strptime(base_time, '%Y-%m-%d %H:%M:%S.%f')
+            query_result = datetime.datetime.strptime(base_time, '%Y-%m-%d %H:%M:%S.%f').replace(tzinfo=datetime.timezone.utc)
             logging.info(f'\tNo query history for {telem_name}. Returning default "previous query" date of {base_time}.')
         else:
             # Negative indexing not allowed in QuerySet
             query_result = query[len(query) - 1].latest_query
+            if query_result.tzinfo == None or query_result.tzinfo.utcoffset(query_result) == None:
+                query_result = query_result.replace(tzinfo=datetime.timezone.utc)
             logging.info(f'For {telem_name}, the previous query time is {query_result}')
 
         return query_result
@@ -1527,7 +1541,7 @@ def run(self, instrument, mnemonic_dict, plot_start=None, plot_end=None):
 
         # SPEED UP TESTING. REMOVE BEFORE MERGING
         plot_start = self._today - datetime.timedelta(days=3.)
-        plot_end = self._today
+        plot_end = self._today #- datetime.timedelta(days=56.)
 
 
 
@@ -1609,7 +1623,7 @@ def run(self, instrument, mnemonic_dict, plot_start=None, plot_end=None):
                     # For daily_means mnemonics, we force the search to always start at noon, and
                     # have a 1 day cadence
                     if telem_type == 'daily_means':
-                        most_recent_search = datetime.datetime.combine(most_recent_search.date(), datetime.time(hour=12))
+                        most_recent_search = datetime.datetime.combine(most_recent_search.date(), datetime.time(hour=12)).replace(tzinfo=datetime.timezone.utc)
 
                     logging.info(f'Most recent search is {most_recent_search}.')
                     logging.info(f'Query cadence is {self.query_cadence}')
@@ -2116,8 +2130,13 @@ def ensure_list(var):
     var : list
         var, translated into a list if necessary
     """
-    if not isinstance(var, list) and not isinstance(var, np.ndarray):
-        return [var]
+    if not isinstance(var, list):
+        if not isinstance(var, np.ndarray):
+            # Here we assume var is a single float, int, str, etc.
+            return [var]
+        else:
+            # Here we convert a numpy array to a list
+            return var.tolist()
     else:
         return var
 

jwql/utils/utils.py

@@ -27,6 +27,7 @@
     - JWST TR JWST-STScI-004800, SM-12
  """
 
+import datetime
 import getpass
 import glob
 import itertools
@@ -153,6 +154,25 @@ def _validate_config(config_file_dict):
         )
 
 
+def add_timezone_to_datetime(dt):
+    """Check to see if timezone information is present in the given
+    datetime.datetime object. If not, set it to UTC.
+
+    Parameters
+    ----------
+    dt : datetime.datetime
+        Datetime object
+
+    Returns
+    -------
+    dt : datetime.datetime
+        Datetime object with UTC timezone info added
+    """
+    if dt.tzinfo == None or dt.tzinfo.utcoffset(dt) == None:
+        dt = dt.replace(tzinfo=datetime.timezone.utc)
+    return dt
+
+
 def create_png_from_fits(filename, outdir):
     """Create and save a png file of the provided file. The file
     will be saved with the same filename as the input file, but

jwql/website/apps/jwql/bokeh_containers.py

@@ -230,7 +230,7 @@ def edb_monitor_tabs(instrument):
     """
     html_file_list = file_list[instrument]
     print('read in html files')
-
+    #CLEARLY THIS IS NOT BEING USED
 
 def generic_telemetry_plot(times, values, name, nominal_value=None, yellow_limits=None,
                            red_limits=None, save=True):

jwql/website/apps/jwql/monitor_models/edb.py

@@ -32,7 +32,7 @@
 from jwql.utils.constants import MAX_LEN_DEPENDENCY_VALUE, MAX_LEN_MNEMONIC
 
 
-class FGSEdbBlocksStats(models.Model):
+class FGSEdbBlockMeansStats(models.Model):
     mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True)
     latest_query = models.DateTimeField(blank=True, null=True)
     times = ArrayField(models.DateTimeField())
@@ -45,7 +45,7 @@ class FGSEdbBlocksStats(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'fgs_edb_blocks_stats'
+        db_table = 'fgs_edb_block_means_stats'
         unique_together = (('id', 'entry_date'),)
 
 
@@ -114,7 +114,7 @@ class Meta:
         unique_together = (('id', 'entry_date'),)
 
 
-class MIRIEdbBlocksStats(models.Model):
+class MIRIEdbBlockMeansStats(models.Model):
     mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True)
     latest_query = models.DateTimeField(blank=True, null=True)
     times = ArrayField(models.DateTimeField())
@@ -127,7 +127,7 @@ class MIRIEdbBlocksStats(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'miri_edb_blocks_stats'
+        db_table = 'miri_edb_block_means_stats'
         unique_together = (('id', 'entry_date'),)
 
 
@@ -196,7 +196,7 @@ class Meta:
         unique_together = (('id', 'entry_date'),)
 
 
-class NIRCamEdbBlocksStats(models.Model):
+class NIRCamEdbBlockMeansStats(models.Model):
     mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True)
     latest_query = models.DateTimeField(blank=True, null=True)
     times = ArrayField(models.DateTimeField())
@@ -209,7 +209,7 @@ class NIRCamEdbBlocksStats(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'nircam_edb_blocks_stats'
+        db_table = 'nircam_edb_block_means_stats'
         unique_together = (('id', 'entry_date'),)
 
 
@@ -278,7 +278,7 @@ class Meta:
         unique_together = (('id', 'entry_date'),)
 
 
-class NIRISSEdbBlocksStats(models.Model):
+class NIRISSEdbBlockMeansStats(models.Model):
     mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True)
     latest_query = models.DateTimeField(blank=True, null=True)
     times = ArrayField(models.DateTimeField())
@@ -291,7 +291,7 @@ class NIRISSEdbBlocksStats(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'niriss_edb_blocks_stats'
+        db_table = 'niriss_edb_block_means_stats'
         unique_together = (('id', 'entry_date'),)
 
 
@@ -360,7 +360,7 @@ class Meta:
         unique_together = (('id', 'entry_date'),)
 
 
-class NIRSpecEdbBlocksStats(models.Model):
+class NIRSpecEdbBlockMeansStats(models.Model):
     mnemonic = models.CharField(max_length=MAX_LEN_MNEMONIC, blank=True, null=True)
     latest_query = models.DateTimeField(blank=True, null=True)
     times = ArrayField(models.DateTimeField())
@@ -373,7 +373,7 @@ class NIRSpecEdbBlocksStats(models.Model):
 
     class Meta:
         managed = True
-        db_table = 'nirspec_edb_blocks_stats'
+        db_table = 'nirspec_edb_block_means_stats'
         unique_together = (('id', 'entry_date'),)