Merge branch 'develop' into bokeh_templating

.pep8speaks.yml

@@ -17,6 +17,7 @@ scanner:
 pycodestyle:
     max-line-length: 100  # Default is 79 in PEP8
     ignore:  # Errors and warnings to ignore
+        - E501  # Line too long
         - E505
 
 only_mention_files_with_errors: True  # If False, a separate status comment for each file is made.

.pyup.yml

@@ -18,7 +18,7 @@ branch: develop
 # update schedule
 # default: empty
 # allowed: "every day", "every week", ..
-schedule: "every week"
+schedule: "every month"
 
 # search for requirement files
 # default: True

CHANGES.rst

@@ -1,3 +1,32 @@
+0.22.0 (2019-08-26)
+===================
+
+New Features
+------------
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Added slides from July 2019 TIPS presentation to ``presentations/`` directory
+
+
+``jwql`` Repository
+~~~~~~~~~~~~~~~~~~~
+
+- Updated dark monitor to support all five JWST instruments, instead of only NIRCam
+- Changed the ``jwql-3.5`` and ``jwql-3.6`` conda environments to be more simple and to work on Linux distributions
+- Added library code for creating instrument monitoring ``bokeh`` plots with new ``bokeh`` templating software
+
+
+Bug Fixes
+---------
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Fixed various bugs that were causing the ``sphinx`` API documentation to crash on ReadTheDocs
+
+
 0.21.0 (2019-07-23)
 ===================
 

Jenkinsfile

@@ -27,7 +27,7 @@ if (utils.scm_checkout()) return
 
 // Establish OS and Python version variables for the matrix
 matrix_os = ["linux-stable"] // (Note that Jenkins can only be run with Linux, not MacOSX/Windows)
-matrix_python = ["3.5", "3.6"]
+matrix_python = ["3.6"]
 
 // Set up the matrix of builds
 matrix = []

README.md

@@ -80,7 +80,7 @@ Next, activate the `base` or `root` environment (depending on your version of `c
 source activate base/root
 ```
 
-Lastly, create the `jwql` environment with either Python 3.5 or 3.6, via the `environment_python_3_5.yml` or `environment_python_3_6.yml` file, respectively. We recommend installing with the 3.6 version:
+Lastly, create the `jwql` environment via the `environment_python_3_6.yml` file:
 
 ```
 conda env create -f environment_python_3_6.yml --name jwql-3.6
@@ -94,7 +94,7 @@ Next, you need to install the `jwql` package under development mode.  This can b
 python setup.py develop
 ```
 
-or 
+or
 
 ```
 pip install -e .

environment_python_3_6.yml

@@ -1,37 +1,38 @@
 channels:
 - defaults
-- http://ssb.stsci.edu/astroconda-dev
+- http://ssb.stsci.edu/astroconda
 dependencies:
-- astroquery=0.3.9
-- bokeh=1.3.4
-- django=2.2.1
-- flake8=3.7.7
+- astroquery=0.3.10
+- bokeh>=1.0,<1.4
+- django=2.2.5
+- flake8=3.7.9
 - inflection=0.3.1
-- ipython=7.7.0
-- jinja2=2.10
-- jsonschema=2.6.0
-- jwst=0.13.0
+- ipython=7.13.0
+- jinja2=2.10.3
+- jsonschema=3.2.0
 - matplotlib=3.1.0
+- nodejs=10.13.0
 - numpy=1.16.4
-- numpydoc=0.9.0
-- pandas=0.24.2
+- numpydoc=0.9.2
+- pandas=1.0.3
 - pip=19.1.1
 - postgresql=9.6.6
 - psycopg2=2.7.5
+- pysiaf==0.7.1
 - python=3.6.4
-- pytest=5.0.1
-- pytest-cov=2.7.1
+- pytest=5.4.2
+- pytest-cov=2.8.1
 - scipy=1.3.0
 - setuptools=41.0.1
-- sphinx=2.1.0
-- sqlalchemy=1.3.5
-- stsci_rtd_theme=0.0.2
-- twine=1.13.0
+- sphinx=3.0.3
+- sqlalchemy=1.3.17
+- twine=2.0.0
 - pip:
-  - asdf==2.3.3
-  - astropy==3.2.1
-  - authlib==0.11
-  - codecov==2.0.15
+  - asdf==2.6.0
+  - astropy==4.0.1
+  - authlib==0.14.3
+  - codecov==2.1.3
   - jwedb>=0.0.3
-  - pysiaf==0.4.0
   - pysqlite3==0.2.2
+  - stsci_rtd_theme==0.0.2
+  - git+https://github.com/spacetelescope/[email protected]

jwql/bokeh_templating/example/main.py

@@ -15,7 +15,6 @@
 """
 
 import os
-
 import numpy as np
 
 from jwql.bokeh_templating import BokehTemplate

jwql/bokeh_templating/factory.py

@@ -219,7 +219,6 @@ def figure_constructor(tool, loader, node):
         key = element.pop('kind')
         shape = {'line': ('Line', figure.line),
                  'circle': ('Circle', figure.circle),
-                 #'step': ('Step', figure.step), #not supported in 0.12.5
                  'diamond': ('Diamond', figure.diamond),
                  'triangle': ('Triangle', figure.triangle),
                  'square': ('Square', figure.square),

jwql/bokeh_templating/keyword_map.py

@@ -47,6 +47,7 @@ def _parse_module(module):
     test = lambda nm, mem: (not nm.startswith("_")) and (module.__name__ in mem.__module__)
     seqs = {nm: mem for nm, mem in getmembers(module, isfunction) if test(nm, mem)}
     maps = {nm: mem for nm, mem in getmembers(module, isclass) if test(nm, mem)}
+
     # these need to be mappings
     if 'gridplot' in seqs:
         maps['gridplot'] = seqs.pop('gridplot')

jwql/bokeh_templating/template.py

@@ -75,7 +75,6 @@ class BokehTemplate(object):
     _sequence_factory = factory.sequence_factory
     _figure_constructor = factory.figure_constructor
     _document_constructor = factory.document_constructor
-
     _embed = False
     document = None
     format_string = ""
@@ -113,7 +112,7 @@ def _include_formatting(self):
         if not self.format_string:
             return
 
-        self.formats = yaml.load(self.format_string)
+        self.formats = yaml.load(self.format_string, Loader=yaml.Loader)
 
     def _parse_interface(self):
         """Parse the YAML interface file using the registered
@@ -138,7 +137,6 @@ def _parse_interface(self):
         # entire string with yaml. We don't need to assign the result to a
         # variable, since the constructors store everything in self.refs
         # (and self.document, for the document).
-
         try:
             yaml.load_all(interface)
         except yaml.YAMLError as exc:

jwql/database/database_interface.py

@@ -79,7 +79,7 @@
 from sqlalchemy.orm.query import Query
 from sqlalchemy.types import ARRAY
 
-from jwql.utils.constants import ANOMALIES, FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES
+from jwql.utils.constants import ANOMALIES_PER_INSTRUMENT, FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES
 from jwql.utils.utils import get_config
 
 ON_JENKINS = '/home/jenkins' in os.path.expanduser('~')
@@ -218,7 +218,7 @@ class Monitor(base):
     monitor_name = Column(String(), nullable=False)
     start_time = Column(DateTime, nullable=False)
     end_time = Column(DateTime, nullable=True)
-    status = Column(Enum('SUCESS', 'FAILURE', name='monitor_status'), nullable=True)
+    status = Column(Enum('SUCCESS', 'FAILURE', name='monitor_status'), nullable=True)
     affected_tables = Column(ARRAY(String, dimensions=1), nullable=True)
     log_file = Column(String(), nullable=False)
 
@@ -241,8 +241,14 @@ class : obj
     data_dict = {}
     data_dict['__tablename__'] = class_name.lower()
 
+    instrument = data_dict['__tablename__'].split('_')[0]
+    instrument_anomalies = []
+    for anomaly in ANOMALIES_PER_INSTRUMENT:
+        if instrument in ANOMALIES_PER_INSTRUMENT[anomaly]:
+            instrument_anomalies.append(anomaly)
+
     # Define anomaly table column names
-    data_dict['columns'] = ANOMALIES
+    data_dict['columns'] = instrument_anomalies
     data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']]
 
     # Create a table with the appropriate Columns
@@ -381,7 +387,11 @@ class : obj
 
 
 # Create tables from ORM factory
-Anomaly = anomaly_orm_factory('anomaly')
+NIRCamAnomaly = anomaly_orm_factory('nircam_anomaly')
+NIRISSAnomaly = anomaly_orm_factory('niriss_anomaly')
+NIRSpecAnomaly = anomaly_orm_factory('nirspec_anomaly')
+MIRIAnomaly = anomaly_orm_factory('miri_anomaly')
+FGSAnomaly = anomaly_orm_factory('fgs_anomaly')
 NIRCamDarkQueryHistory = monitor_orm_factory('nircam_dark_query_history')
 NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats')
 NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current')
@@ -397,6 +407,8 @@ class : obj
 FGSDarkQueryHistory = monitor_orm_factory('fgs_dark_query_history')
 FGSDarkPixelStats = monitor_orm_factory('fgs_dark_pixel_stats')
 FGSDarkDarkCurrent = monitor_orm_factory('fgs_dark_dark_current')
+NIRCamBiasQueryHistory = monitor_orm_factory('nircam_bias_query_history')
+NIRCamBiasStats = monitor_orm_factory('nircam_bias_stats')
 
 
 if __name__ == '__main__':

jwql/database/monitor_table_definitions/nircam/nircam_bias_query_history.txt

@@ -0,0 +1,8 @@
+INSTRUMENT, string
+APERTURE, string
+START_TIME_MJD, float
+END_TIME_MJD, float
+ENTRIES_FOUND, integer
+FILES_FOUND, integer
+RUN_MONITOR, bool
+ENTRY_DATE, datetime

jwql/database/monitor_table_definitions/nircam/nircam_bias_stats.txt

@@ -0,0 +1,19 @@
+APERTURE, string
+UNCAL_FILENAME, string
+CAL_FILENAME, string
+CAL_IMAGE, string
+EXPSTART, string
+MEAN, float
+MEDIAN, float
+STDDEV, float
+COLLAPSED_ROWS, float_array_1d
+COLLAPSED_COLUMNS, float_array_1d
+AMP1_EVEN_MED, float
+AMP1_ODD_MED, float
+AMP2_EVEN_MED, float
+AMP2_ODD_MED, float
+AMP3_EVEN_MED, float
+AMP3_ODD_MED, float
+AMP4_EVEN_MED, float
+AMP4_ODD_MED, float
+ENTRY_DATE, datetime