Update Bad Pixel Monitor to use Django DB Models

jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py

@@ -95,25 +95,25 @@
 from jwst_reffiles.bad_pixel_mask import bad_pixel_mask
 import numpy as np
 
-from jwql.database.database_interface import engine, session
-from jwql.database.database_interface import NIRCamBadPixelQueryHistory, NIRCamBadPixelStats
-from jwql.database.database_interface import NIRISSBadPixelQueryHistory, NIRISSBadPixelStats
-from jwql.database.database_interface import MIRIBadPixelQueryHistory, MIRIBadPixelStats
-from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats
-from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats
 from jwql.instrument_monitors import pipeline_tools
 from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline
 from jwql.utils import crds_tools, instrument_properties, monitor_utils
 from jwql.utils.constants import DARKS_BAD_PIXEL_TYPES, DARK_EXP_TYPES, FLATS_BAD_PIXEL_TYPES, FLAT_EXP_TYPES
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, ON_GITHUB_ACTIONS
-from jwql.utils.constants import ON_READTHEDOCS
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.mast_utils import mast_query
 from jwql.utils.permissions import set_permissions
 from jwql.utils.utils import copy_files, create_png_from_fits, ensure_dir_exists, get_config, filesystem_path
 
 
 if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    # Need to set up django apps before we can access the models
+    import django  # noqa: E402 (module level import not at top of file)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    django.setup()
+
+    from jwql.website.apps.jwql.monitor_models.bad_pixel import *
     from jwql.website.apps.jwql.monitor_pages.monitor_bad_pixel_bokeh import BadPixelPlots
 
 THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'bad_pixel_file_thresholds.txt')
@@ -435,8 +435,8 @@ def add_bad_pix(self, coordinates, pixel_type, files, obs_start_time, obs_mid_ti
                  'obs_end_time': obs_end_time,
                  'baseline_file': baseline_file,
                  'entry_date': datetime.datetime.now()}
-        with engine.begin() as connection:
-            connection.execute(self.pixel_table.__table__.insert(), entry)
+        entry = self.pixel_table(**entry)
+        entry.save()
 
     def filter_query_results(self, results, datatype):
         """Filter MAST query results. For input flats, keep only those
@@ -583,14 +583,15 @@ def exclude_existing_badpix(self, badpix, pixel_type):
         if pixel_type not in ['hot', 'dead', 'noisy']:
             raise ValueError('Unrecognized bad pixel type: {}'.format(pixel_type))
 
-        db_entries = session.query(self.pixel_table) \
-            .filter(self.pixel_table.type == pixel_type) \
-            .filter(self.pixel_table.detector == self.detector) \
-            .all()
+        filters = {"type__iexact": pixel_type,
+                   "detector__iexact": self.detector
+                   }
 
+        records = self.pixel_table.objects.filter(**filters).all()
         already_found = []
-        if len(db_entries) != 0:
-            for _row in db_entries:
+
+        if len(records) != 0:
+            for _row in records:
                 x_coords = _row.x_coord
                 y_coords = _row.y_coord
                 for x, y in zip(x_coords, y_coords):
@@ -606,17 +607,15 @@ def exclude_existing_badpix(self, badpix, pixel_type):
                 new_pixels_x.append(x)
                 new_pixels_y.append(y)
 
-        session.close()
-
         return (new_pixels_x, new_pixels_y)
 
     def identify_tables(self):
         """Determine which database tables to use for a run of the bad pixel
         monitor
         """
         mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self.instrument]
-        self.query_table = eval('{}BadPixelQueryHistory'.format(mixed_case_name))
-        self.pixel_table = eval('{}BadPixelStats'.format(mixed_case_name))
+        self.query_table = eval(f'{mixed_case_name}BadPixelQueryHistory')
+        self.pixel_table = eval(f'{mixed_case_name}BadPixelStats')
 
     def map_uncal_and_rate_file_lists(self, uncal_files, rate_files, rate_files_to_copy, obs_type):
         """Copy uncal and rate files from the filesystem to the working
@@ -701,30 +700,29 @@ def most_recent_search(self, file_type='dark'):
             where the dark monitor was run.
         """
         if file_type.lower() == 'dark':
-            run_field = self.query_table.run_bpix_from_darks
+            run_field = "run_bpix_from_darks"
+            sort_field = "-dark_end_time_mjd"
         elif file_type.lower() == 'flat':
-            run_field = self.query_table.run_bpix_from_flats
+            run_field = "run_bpix_from_flats"
+            sort_field = "-flat_end_time_mjd"
 
-        query = session.query(self.query_table).filter(self.query_table.aperture == self.aperture). \
-            filter(run_field == True)  # noqa: E712 (comparison to true)
+        filters = {"aperture__iexact": self.aperture,
+                   run_field: True}
 
-        dates = np.zeros(0)
-        if file_type.lower() == 'dark':
-            for instance in query:
-                dates = np.append(dates, instance.dark_end_time_mjd)
-        elif file_type.lower() == 'flat':
-            for instance in query:
-                dates = np.append(dates, instance.flat_end_time_mjd)
+        record = self.query_table.objects.filter(**filters).order_by(sort_field).first()
 
-        query_count = len(dates)
-        if query_count == 0:
+        # Record is django QuerySet object, when empty QuerySet object is returned (<QuerySet []>)
+        # the result of record.first() is None
+        if record is None:
             query_result = 59607.0  # a.k.a. Jan 28, 2022 == First JWST images (MIRI)
             logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'
                          .format(self.aperture, query_result)))
         else:
-            query_result = np.max(dates)
+            if file_type.lower() == 'dark':
+                query_result = record.dark_end_time_mjd
+            elif file_type.lower() == 'flat':
+                query_result = record.flat_end_time_mjd
 
-        session.close()
         return query_result
 
     def make_crds_parameter_dict(self):
@@ -1287,8 +1285,8 @@ def run(self):
                              'run_bpix_from_flats': run_flats,
                              'run_monitor': run_flats or run_darks,
                              'entry_date': datetime.datetime.now()}
-                with engine.begin() as connection:
-                    connection.execute(self.query_table.__table__.insert(), new_entry)
+                entry = self.query_table(**new_entry)
+                entry.save()
                 logging.info('\tUpdated the query history table')
 
         # Update the figures to be shown in the web app. Only update figures

jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py

@@ -32,20 +32,22 @@
 import numpy as np
 from sqlalchemy import and_, func
 
-from jwql.database.database_interface import get_unique_values_per_column, session
-from jwql.database.database_interface import NIRCamBadPixelQueryHistory, NIRCamBadPixelStats
-from jwql.database.database_interface import NIRISSBadPixelQueryHistory, NIRISSBadPixelStats
-from jwql.database.database_interface import MIRIBadPixelQueryHistory, MIRIBadPixelStats
-from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats
-from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats
 from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES
 from jwql.utils.constants import DETECTOR_PER_INSTRUMENT, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE
+from jwql.utils.constants import ON_GITHUB_ACTIONS, ON_READTHEDOCS
 from jwql.utils.permissions import set_permissions
 from jwql.utils.utils import filesystem_path, get_config, read_png, save_png
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 OUTPUT_DIR = get_config()['outputs']
 
+if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    # Need to set up django apps before we can access the models
+    import django  # noqa: E402 (module level import not at top of file)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    django.setup()
+    from jwql.website.apps.jwql.monitor_models.bad_pixel import *
+
 
 class BadPixelPlots():
     """Class for creating the bad pixel monitor plots and figures to be displayed
@@ -263,40 +265,33 @@ def __init__(self, pixel_table, instrument, detector):
             self.get_trending_data(badtype)
 
     def get_most_recent_entry(self):
-        """Get all nedded data from the database tables.
+        """For the given detector, get the latest entry for each bad pixel type
         """
-        # For the given detector, get the latest entry for each bad pixel type
-        subq = (session
-                .query(self.pixel_table.type, func.max(self.pixel_table.entry_date).label("max_created"))
-                .filter(self.pixel_table.detector == self.detector)
-                .group_by(self.pixel_table.type)
-                .subquery()
-                )
-
-        query = (session.query(self.pixel_table)
-                 .join(subq, self.pixel_table.entry_date == subq.c.max_created)
-                 )
-
-        latest_entries_by_type = query.all()
-        session.close()
-
-        # Organize the results
-        for row in latest_entries_by_type:
-            self.new_bad_pix[row.type] = (row.x_coord, row.y_coord)
-            self.background_file[row.type] = row.source_files[0]
-            self.obs_start_time[row.type] = row.obs_start_time
-            self.obs_end_time[row.type] = row.obs_end_time
-            self.num_files[row.type] = len(row.source_files)
-            self.baseline_file[row.type] = row.baseline_file
-
-        # If no data is retrieved from the database at all, add a dummy generic entry
-        if len(self.new_bad_pix.keys()) == 0:
-            self.new_bad_pix[self.badtypes[0]] = ([], [])
-            self.background_file[self.badtypes[0]] = ''
-            self.obs_start_time[self.badtypes[0]] = datetime.datetime.today()
-            self.obs_end_time[self.badtypes[0]] = datetime.datetime.today()
-            self.num_files[self.badtypes[0]] = 0
-            self.baseline_file[self.badtypes[0]] = ''
+
+        bad_pixel_types = self.pixel_table.objects.values('type').distinct()
+
+        for bad_type in bad_pixel_types:
+            bad_filters = {'detector__iexact': self.detector, 
+                           'type': bad_type}
+
+            record = (self.pixel_table.objects
+                        .filter(**bad_filters)
+                        .order_by("-obs_end_time").first())
+
+            if record is None:
+                self.new_bad_pix[bad_type] = ([], [])
+                self.background_file[bad_type] = ''
+                self.obs_start_time[bad_type] = datetime.datetime.today()
+                self.obs_end_time[bad_type] = datetime.datetime.today()
+                self.num_files[bad_type] = 0
+                self.baseline_file[bad_type] = ''
+            else:
+                self.new_bad_pix[bad_type] = (record.x_coord, record.y)
+                self.background_file[bad_type] = record.source_file
+                self.obs_start_time[bad_type] = record.obs_start_time
+                self.obs_end_time[bad_type] = record.obs_end_time
+                self.num_files[bad_type] = len(record.source_files)
+                self.baseline_file[bad_type] = record.baseline_file
 
     def get_trending_data(self, badpix_type):
         """Retrieve and organize the data needed to produce the trending plot.
@@ -306,21 +301,20 @@ def get_trending_data(self, badpix_type):
         badpix_type : str
             The type of bad pixel to query for, e.g. 'dead'
         """
-        # Query database for all data in the table with a matching detector and bad pixel type
-        all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1),
-                                            self.pixel_table.obs_mid_time) \
-            .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \
-            .all()
+        filters = {"type": badpix_type,
+                   "detector": self.detector}
+
+        all_entries_by_type = self.pixel_table.objects.filter(**filters).all()
 
-        # Organize the results
         num_pix = []
         times = []
+
         for i, row in enumerate(all_entries_by_type):
             if i == 0:
-                badtype = row[0]
-                detector = row[1]
-            num_pix.append(row[2])
-            times.append(row[3])
+                badtype = row.type
+                detector = row.detector
+            num_pix.append(len(row.x_coord))
+            times.append(row.obs_mid_time)
 
         # If there was no data in the database, create an empty entry
         if len(num_pix) == 0:
@@ -329,9 +323,7 @@ def get_trending_data(self, badpix_type):
             num_pix = [0]
             times = [datetime.datetime.today()]
 
-        # Add results to self.trending_data
-        self.trending_data[badpix_type] = (detector, num_pix, times)
-        session.close()
+        self.trending_data[badtype] = (detector, num_pix, times)
 
 
 class NewBadPixPlot():