Resolve conflicts

.pep8speaks.yml

@@ -17,6 +17,7 @@ scanner:
 pycodestyle:
     max-line-length: 100  # Default is 79 in PEP8
     ignore:  # Errors and warnings to ignore
+        - E501  # Line too long
         - E505
 
 only_mention_files_with_errors: True  # If False, a separate status comment for each file is made.

.pyup.yml

@@ -18,7 +18,7 @@ branch: develop
 # update schedule
 # default: empty
 # allowed: "every day", "every week", ..
-schedule: "every week"
+schedule: "every month"
 
 # search for requirement files
 # default: True

environment_python_3_6.yml

@@ -3,37 +3,38 @@ channels:
 - http://ssb.stsci.edu/astroconda
 dependencies:
 - astroquery=0.3.10
-- bokeh=1.4.0
+- bokeh>=1.0,<1.4
 - django=2.2.5
-- flake8=3.7.8
+- flake8=3.7.9
 - inflection=0.3.1
-- ipython=7.9.0
+- ipython=7.13.0
 - jinja2=2.10.3
 - jsonschema=3.2.0
 - matplotlib=3.1.0
+- nodejs=10.13.0
 - numpy=1.16.4
 - numpydoc=0.9.2
-- pandas=1.0.0
+- pandas=1.0.3
 - pip=19.1.1
 - postgresql=9.6.6
 - psycopg2=2.7.5
 - pysiaf==0.7.1
 - python=3.6.4
-- pytest=5.2.2
-- pytest-cov=2.7.1
+- pytest=5.4.2
+- pytest-cov=2.8.1
 - scipy=1.3.0
 - setuptools=41.0.1
-- sphinx=2.4.0
-- sqlalchemy=1.3.13
+- sphinx=3.0.3
+- sqlalchemy=1.3.17
 - twine=2.0.0
 - pip:
-  - asdf==2.4.2
-  - astropy==4.0
-  - authlib==0.13
-  - codecov==2.0.15
+  - asdf==2.6.0
+  - astropy==4.0.1
+  - authlib==0.14.3
+  - codecov==2.1.3
   - crds==7.5.0.0
   - jwedb>=0.0.3
   - pysqlite3==0.2.2
   - stsci_rtd_theme==0.0.2
-  - git+https://github.com/spacetelescope/jwst#0.16.2
+  - git+https://github.com/spacetelescope/jwst@0.16.2
   - git+https://github.com/spacetelescope/jwst_reffiles

jwql/bokeh_templating/example/main.py

@@ -6,7 +6,6 @@
 @author: gkanarek
 """
 import os
-
 import numpy as np
 
 from jwql.bokeh_templating import BokehTemplate

jwql/bokeh_templating/factory.py

@@ -109,7 +109,6 @@ def figure_constructor(tool, loader, node):
         key = element.pop('kind')
         shape = {'line': ('Line', figure.line),
                  'circle': ('Circle', figure.circle),
-                 #'step': ('Step', figure.step), #not supported in 0.12.5
                  'diamond': ('Diamond', figure.diamond),
                  'triangle': ('Triangle', figure.triangle),
                  'square': ('Square', figure.square),

jwql/bokeh_templating/keyword_map.py

@@ -19,6 +19,7 @@ def parse_module(module):
     test = lambda nm, mem: (not nm.startswith("_")) and (module.__name__ in mem.__module__)
     seqs = {nm: mem for nm, mem in getmembers(module, isfunction) if test(nm, mem)}
     maps = {nm: mem for nm, mem in getmembers(module, isclass) if test(nm, mem)}
+
     # these need to be mappings
     if 'gridplot' in seqs:
         maps['gridplot'] = seqs.pop('gridplot')

jwql/bokeh_templating/template.py

@@ -24,7 +24,7 @@ class BokehTemplateEmbedError(Exception):
     """
 
 
-class BokehTemplate(object):
+class BokehTemplate():
     """
     This is the base class for creating Bokeh web apps using a YAML templating
     framework.
@@ -34,13 +34,13 @@ class BokehTemplate(object):
     _sequence_factory = factory.sequence_factory
     _figure_constructor = factory.figure_constructor
     _document_constructor = factory.document_constructor
-
     _embed = False
 
     def _self_constructor(self, loader, tag_suffix, node):
         """
         A multi_constructor for `!self` tag in the interface file.
         """
+
         yield eval("self" + tag_suffix, globals(), locals())
 
     def _register_default_constructors(self):
@@ -104,7 +104,7 @@ def include_formatting(self):
         if not self.format_string:
             return
 
-        self.formats = yaml.load(self.format_string)
+        self.formats = yaml.load(self.format_string, Loader=yaml.Loader)
 
     def parse_interface(self):
         """
@@ -133,10 +133,10 @@ def parse_interface(self):
         # variable, since the constructors store everything in self.refs
         # (and self.document, for the document)
 
-        self.full_stream = list(yaml.load_all(interface))
+        self.full_stream = list(yaml.load(interface, Loader=yaml.Loader))
 
     def parse_string(self, yaml_string):
-        return list(yaml.load_all(yaml_string))
+        return list(yaml.load(yaml_string, Loader=yaml.Loader))
 
     def embed(self, ref):
         element = self.refs.get(ref, None)

jwql/database/database_interface.py

@@ -79,7 +79,7 @@
 from sqlalchemy.orm.query import Query
 from sqlalchemy.types import ARRAY
 
-from jwql.utils.constants import ANOMALIES, FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES
+from jwql.utils.constants import ANOMALIES_PER_INSTRUMENT, FILE_SUFFIX_TYPES, JWST_INSTRUMENT_NAMES
 from jwql.utils.utils import get_config
 
 ON_JENKINS = '/home/jenkins' in os.path.expanduser('~')
@@ -241,8 +241,14 @@ class : obj
     data_dict = {}
     data_dict['__tablename__'] = class_name.lower()
 
+    instrument = data_dict['__tablename__'].split('_')[0]
+    instrument_anomalies = []
+    for anomaly in ANOMALIES_PER_INSTRUMENT:
+        if instrument in ANOMALIES_PER_INSTRUMENT[anomaly]:
+            instrument_anomalies.append(anomaly)
+
     # Define anomaly table column names
-    data_dict['columns'] = ANOMALIES
+    data_dict['columns'] = instrument_anomalies
     data_dict['names'] = [name.replace('_', ' ') for name in data_dict['columns']]
 
     # Create a table with the appropriate Columns
@@ -381,7 +387,11 @@ class : obj
 
 
 # Create tables from ORM factory
-Anomaly = anomaly_orm_factory('anomaly')
+NIRCamAnomaly = anomaly_orm_factory('nircam_anomaly')
+NIRISSAnomaly = anomaly_orm_factory('niriss_anomaly')
+NIRSpecAnomaly = anomaly_orm_factory('nirspec_anomaly')
+MIRIAnomaly = anomaly_orm_factory('miri_anomaly')
+FGSAnomaly = anomaly_orm_factory('fgs_anomaly')
 NIRCamDarkQueryHistory = monitor_orm_factory('nircam_dark_query_history')
 NIRCamDarkPixelStats = monitor_orm_factory('nircam_dark_pixel_stats')
 NIRCamDarkDarkCurrent = monitor_orm_factory('nircam_dark_dark_current')

jwql/jwql_monitors/monitor_mast.py

@@ -249,6 +249,8 @@ def monitor_mast():
     """
     logging.info('Beginning database monitoring.')
 
+    outputs_dir = os.path.join(get_config()['outputs'], 'monitor_mast')
+
     # Perform inventory of the JWST service
     jwst_inventory(instruments=JWST_INSTRUMENT_NAMES,
                    dataproducts=['image', 'spectrum', 'cube'],
@@ -267,4 +269,4 @@ def monitor_mast():
     configure_logging(module)
 
     # Run the monitors
-    monitor_mast()
+    monitor_mast()

jwql/tests/test_database_interface.py

@@ -25,7 +25,7 @@
 import string
 
 from jwql.database import database_interface as di
-from jwql.utils.constants import ANOMALIES
+from jwql.utils.constants import ANOMALIES_PER_INSTRUMENT
 from jwql.utils.utils import get_config
 
 # Determine if tests are being run on jenkins
@@ -61,14 +61,14 @@ def test_anomaly_orm_factory():
     creates an ORM and contains the appropriate columns"""
 
     test_table_name = 'test_anomaly_table'
-    TestAnomalyTable = di.anomaly_orm_factory('test_anomaly_table')
+    TestAnomalyTable = di.anomaly_orm_factory(test_table_name)
     table_attributes = TestAnomalyTable.__dict__.keys()
 
     assert str(TestAnomalyTable) == "<class 'jwql.database.database_interface.{}'>"\
         .format(test_table_name)
 
-    for anomaly in ANOMALIES:
-        assert anomaly in table_attributes
+    for item in ['id', 'rootname', 'flag_date', 'user']:
+        assert item in table_attributes
 
 
 @pytest.mark.skipif(ON_JENKINS, reason='Requires access to development database server.')
@@ -79,15 +79,15 @@ def test_anomaly_records():
     random_rootname = ''.join(random.SystemRandom().choice(string.ascii_lowercase + \
                                                            string.ascii_uppercase + \
                                                            string.digits) for _ in range(10))
-    di.session.add(di.Anomaly(rootname=random_rootname,
+    di.session.add(di.FGSAnomaly(rootname=random_rootname,
                               flag_date=datetime.datetime.today(),
                               user='test', ghost=True))
     di.session.commit()
 
     # Test the ghosts column
-    ghosts = di.session.query(di.Anomaly)\
-        .filter(di.Anomaly.rootname == random_rootname)\
-        .filter(di.Anomaly.ghost == "True")
+    ghosts = di.session.query(di.FGSAnomaly)\
+        .filter(di.FGSAnomaly.rootname == random_rootname)\
+        .filter(di.FGSAnomaly.ghost == "True")
     assert ghosts.data_frame.iloc[0]['ghost'] == True
 
 

jwql/utils/constants.py

@@ -47,13 +47,31 @@
                                  '3': [(2, 1032, 4), (0, 1024, 1)],
                                  '4': [(3, 1032, 4), (0, 1024, 1)]}}
 
-
-# Defines the possible anomalies to flag through the web app
-ANOMALIES = ['snowball', 'cosmic_ray_shower', 'crosstalk', 'data_transfer_error', 'diffraction_spike',
-             'excessive_saturation', 'ghost', 'guidestar_failure', 'persistence', 'satellite_trail', 'other']
+ANOMALIES_PER_INSTRUMENT = {
+    # anomalies affecting all instruments:
+    'cosmic_ray_shower': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
+    'diffraction_spike': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
+    'excessive_saturation': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
+    'guidestar_failure': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
+    'persistence': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec'],
+    #anomalies affecting multiple instruments:
+    'crosstalk': ['fgs', 'nircam', 'niriss', 'nirspec'],
+    'data_transfer_error': ['fgs', 'nircam', 'niriss', 'nirspec'],
+    'ghost': ['fgs', 'nircam', 'niriss', 'nirspec'],
+    'snowball': ['fgs', 'nircam', 'niriss', 'nirspec'],
+    # instrument-specific anomalies:
+    'column_pull_up': ['miri'],
+    'dominant_msa_leakage': ['nirspec'],
+    'dragons_breath': ['nircam'],
+    'glow': ['miri'],
+    'internal_reflection': ['miri'],
+    'optical_short': ['nirspec'],  # Only for MOS observations
+    'row_pull_down': ['miri'],
+    # additional anomalies:
+    'other': ['fgs', 'miri', 'nircam', 'niriss', 'nirspec']}
 
 # Defines the possible anomalies (with rendered name) to flag through the web app
-ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES]
+ANOMALY_CHOICES = [(anomaly, inflection.titleize(anomaly)) for anomaly in ANOMALIES_PER_INSTRUMENT]
 
 # Possible exposure types for dark current data
 DARK_EXP_TYPES = {'nircam': ['NRC_DARK'],
@@ -76,6 +94,15 @@
 
 FOUR_AMP_SUBARRAYS = ['WFSS128R', 'WFSS64R', 'WFSS128C', 'WFSS64C']
 
+# Names of full-frame apertures for all instruments
+FULL_FRAME_APERTURES = {'NIRCAM': ['NRCA1_FULL', 'NRCA2_FULL', 'NRCA3_FULL', 'NRCA4_FULL',
+                                   'NRCA5_FULL', 'NRCB1_FULL', 'NRCB2_FULL', 'NRCB3_FULL',
+                                   'NRCB4_FULL', 'NRCB5_FULL'],
+                        'NIRISS': ['NIS_CEN'],
+                        'NIRSPEC': ['NRS1_FULL', 'NRS2_FULL'],
+                        'MIRI': ['MIRIM_FULL']
+                        }
+
 # Possible suffix types for nominal files
 GENERIC_SUFFIX_TYPES = ['uncal', 'cal', 'rateints', 'rate', 'trapsfilled', 'i2d',
                         'x1dints', 'x1d', 's2d', 's3d', 'dark', 'crfints',
@@ -132,7 +159,7 @@
     'nircam': [('Bias Monitor', '#'),
                ('Readnoise Monitor', '#'),
                ('Gain Level Monitor', '#'),
-               ('Mean Dark Current Rate Monitor', '#'),
+               ('Mean Dark Current Rate Monitor', '/nircam/dark_monitor'),
                ('Photometric Stability Monitor', '#')],
     'niriss': [('Bad Pixel Monitor', '#'),
                ('Readnoise Monitor', '#'),

jwql/utils/logging_functions.py

@@ -10,8 +10,9 @@
 -------
 
     - Catherine Martlin
-    - Alex Viana (WFC3 QL Version)
+    - Alex Viana (wfc3ql Version)
     - Matthew Bourque
+    - Jason Neal
 
 Use
 ---
@@ -59,6 +60,7 @@ def my_main_function():
 import os
 import pwd
 import socket
+import subprocess
 import sys
 import time
 import traceback
@@ -236,7 +238,10 @@ def wrapped(*args, **kwargs):
             except (ImportError, AttributeError) as err:
                 logging.warning(err)
 
-        logging.info('')
+        environment = subprocess.check_output(['conda', 'env', 'export'], universal_newlines=True)
+        logging.info('Environment:')
+        for line in environment.split('\n'):
+            logging.info(line)
 
         # Call the function and time it
         t1_cpu = time.clock()