Merge branch 'develop' into ta-monitor-initial

README.md

@@ -83,16 +83,16 @@ source activate base/root
 
 **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step.
 
-Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
+Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
 
 ```
-conda env create -f environment_python_3_9.yml
+conda env create -f environment_python_3.9.yml
 ```
 
 or
 
 ```
-conda env create -f environment_python_3_10.yml
+conda env create -f environment_python_3.10.yml
 ```
 
 ### Configuration File
@@ -154,10 +154,11 @@ Any questions about the `jwql` project or its software can be directed to `jwql@
 - Mees Fix (Technical Lead, INS) [@mfixstsci](https://github.com/mfixstsci)
 - Misty Cracraft (INS) [@cracraft](https://github.com/cracraft)
 - Mike Engesser (INS) [@mengesser](https://github.com/mengesser)
-- Shannon Osborne (INS) [@shanosborne](https://github.com/shanosborne)
 - Maria Pena-Guerrero [@penaguerrero](https://github.com/penaguerrero)
 - Ben Sunnquist (INS) [@bsunnquist](https://github.com/bsunnquist)
 - Brian York (INS) [@york-stsci](https://github.com/york-stsci)
+- Bradley Sappington (INS) [@bradleysappington](https://github.com/bradleysappington)
+- Melanie Clarke (INS) [@melanieclarke](https://github.com/melanieclarke)
 
 ## Past Development Team Members
 - Matthew Bourque (INS) [@bourque](https://github.com/bourque)
@@ -168,7 +169,7 @@ Any questions about the `jwql` project or its software can be directed to `jwql@
 - Sara Ogaz (DMD) [@SaOgaz](https://github.com/SaOgaz)
 - Catherine Martlin (INS) [@catherine-martlin](https://github.com/catherine-martlin)
 - Johannes Sahlmann (INS) [@Johannes-Sahlmann](https://github.com/johannes-sahlmann)
-
+- Shannon Osborne (INS) [@shanosborne](https://github.com/shanosborne)
 
 ## Acknowledgments:
 - Faith Abney (DMD)

environment_python_3.10.yml

@@ -1,10 +1,10 @@
 # This file describes a conda environment that can be to install jwql
 #
 # Run the following command to set up this environment:
-# $ conda env create -f environment_python_3_10.yml
+# $ conda env create -f environment_python_3.10.yml
 #
 # The environment name can be overridden with the following command:
-# $ conda env create -n <custom name> -f environment_python_3_10.yml
+# $ conda env create -n <custom name> -f environment_python_3.10.yml
 #
 # Run the following command to activate the environment:
 # $ source activate jwql-3.10
@@ -22,49 +22,49 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.2.2
-  - bokeh=2.4.3
+  - astropy=5.3.2
   - beautifulsoup4=4.12.2
-  - celery=5.2.7
-  - cryptography=40.0.2
-  - django=4.2.1
+  - bokeh=2.4.3
+  - celery=5.3.1
+  - cryptography=41.0.3
+  - django=4.2.3
   - inflection=0.5.1
-  - ipython=8.13.2
+  - ipython=8.14.0
   - jinja2=3.1.2
-  - jsonschema=4.17.3
-  - matplotlib=3.7.0
-  - nodejs=18.15.0
-  - numpy=1.24.3
+  - jsonschema=4.19.0
+  - matplotlib=3.7.2
+  - nodejs=20.5.1
+  - numpy=1.25.2
   - numpydoc=1.5.0
-  - pandas=2.0.1
-  - pip=23.1.2
-  - postgresql=15.2
-  - psycopg2=2.9.3
-  - pytest=7.3.1
-  - pytest-cov=4.0.0
-  - pytest-mock=3.10.0
-  - python=3.10.9
+  - pandas=2.0.3
+  - pip=23.2.1
+  - postgresql=15.4
+  - psycopg2=2.9.6
+  - pytest=7.4.0
+  - pytest-cov=4.1.0
+  - pytest-mock=3.11.1
+  - python=3.10.12
   - pyyaml=6.0
   - redis
-  - ruff=0.0.269
+  - ruff=0.0.285
   - scipy=1.9.3
-  - setuptools=67.7.2
+  - setuptools=68.0.0
   - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.15
+  - sphinx_rtd_theme=1.2.2
+  - sqlalchemy=2.0.20
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
     - astroquery==0.4.6
     - bandit==1.7.5
-    - jwst==1.10.2
-    - pysiaf==0.19.1
-    - pysqlite3==0.5.0
-    - pyvo==1.4.1
-    - redis==4.5.5
-    - selenium==4.9.1
-    - stdatamodels==1.3.1
+    - jwst==1.11.4
+    - pysiaf==0.20.0
+    - pysqlite3==0.5.1
+    - pyvo==1.4.2
+    - redis==5.0.0
+    - selenium==4.11.2
+    - stdatamodels==1.7.2
     - stsci_rtd_theme==1.0.0
     - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles

environment_python_3.9.yml

@@ -1,10 +1,10 @@
 # This file describes a conda environment that can be to install jwql
 #
 # Run the following command to set up this environment:
-# $ conda env create -f environment_python_3_9.yml
+# $ conda env create -f environment_python_3.9.yml
 #
 # The environment name can be overridden with the following command:
-# $ conda env create -n <custom name> -f environment_python_3_9.yml
+# $ conda env create -n <custom name> -f environment_python_3.9.yml
 #
 # Run the following command to activate the environment:
 # $ source activate jwql-3.9
@@ -22,49 +22,49 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.2.2
-  - bokeh=2.4.3
+  - astropy=5.3.2
   - beautifulsoup4=4.12.2
-  - celery=5.2.7
-  - cryptography=40.0.2
-  - django=4.2.1
+  - bokeh=2.4.3
+  - celery=5.3.1
+  - cryptography=41.0.3
+  - django=4.2.3
   - inflection=0.5.1
-  - ipython=8.13.2
+  - ipython=8.14.0
   - jinja2=3.1.2
-  - jsonschema=4.17.3
-  - matplotlib=3.7.0
-  - nodejs=18.15.0
-  - numpy=1.24.3
+  - jsonschema=4.19.0
+  - matplotlib=3.7.2
+  - nodejs=20.5.1
+  - numpy=1.25.2
   - numpydoc=1.5.0
-  - pandas=2.0.1
-  - pip=23.1.2
-  - postgresql=15.2
-  - psycopg2=2.9.3
-  - pytest=7.3.1
-  - pytest-cov=4.0.0
-  - pytest-mock=3.10.0
-  - python=3.9.16
+  - pandas=2.0.3
+  - pip=23.2.1
+  - postgresql=15.4
+  - psycopg2=2.9.6
+  - pytest=7.4.0
+  - pytest-cov=4.1.0
+  - pytest-mock=3.11.1
+  - python=3.9.17
   - pyyaml=6.0
   - redis
-  - ruff=0.0.269
+  - ruff=0.0.285
   - scipy=1.9.3
-  - setuptools=67.7.2
+  - setuptools=68.0.0
   - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.0
-  - sqlalchemy=2.0.15
+  - sphinx_rtd_theme=1.2.2
+  - sqlalchemy=2.0.20
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
     - astroquery==0.4.6
     - bandit==1.7.5
-    - jwst==1.10.2
-    - pysiaf==0.19.1
-    - pysqlite3==0.5.0
-    - pyvo==1.4.1
-    - redis==4.5.5
-    - selenium==4.9.1
-    - stdatamodels==1.3.1
+    - jwst==1.11.4
+    - pysiaf==0.20.0
+    - pysqlite3==0.5.1
+    - pyvo==1.4.2
+    - redis==5.0.0
+    - selenium==4.11.2
+    - stdatamodels==1.7.2
     - stsci_rtd_theme==1.0.0
     - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles

jwql/example_config.json

@@ -10,6 +10,14 @@
         "host" : "",
         "port" : ""
     },
+    "django_database" : {
+        "ENGINE" : "",
+        "NAME" : "",
+        "USER" : "",
+        "PASSWORD" : "",
+        "HOST" : "",
+        "PORT" : ""
+    },
     "jwql_dir" : "",
     "jwql_version": "",
     "server_type": "",

jwql/instrument_monitors/common_monitors/bias_monitor.py

@@ -432,7 +432,13 @@ def process(self, file_list):
             # Add this new entry to the bias database table
             with engine.begin() as connection:
                 connection.execute(self.stats_table.__table__.insert(), bias_db_entry)
-            logging.info('\tNew entry added to bias database table: {}'.format(bias_db_entry))
+
+            # Don't print long arrays of numbers to the log file
+            log_dict = {}
+            for key in bias_db_entry:
+                if key not in ['collapsed_rows', 'collapsed_columns', 'counts', 'bin_centers']:
+                    log_dict[key] = bias_db_entry[key]
+            logging.info('\tNew entry added to bias database table: {}'.format(log_dict))
 
             # Remove the raw and calibrated files to save memory space
             os.remove(filename)

jwql/instrument_monitors/common_monitors/dark_monitor.py

@@ -649,23 +649,14 @@ def overplot_bad_pix(self, pix_type, coords, values):
             values = []
 
         sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0],
-                                                       pixels_y=coords[1],
-                                                       values=values
+                                                       pixels_y=coords[1]
                                                       )
                                              )
 
         # Overplot the bad pixel locations
         badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y',
                                                  source=sources[pix_type], color=colors[pix_type])
 
-        # Create hover tools for the bad pixel types
-        #hover_tools[pix_type] = HoverTool(tooltips=[(f'{pix_type} (x, y):', '(@pixels_x, @pixels_y)'),
-        #                                            ('value:', f'@values'),
-        #                                            ],
-        #                                    renderers=[badpixplots[pix_type]])
-        # Add tool to plot
-        #self.plot.tools.append(hover_tools[pix_type])
-
         # Add to the legend
         if  numpix > 0:
             if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT:
@@ -711,7 +702,11 @@ def process(self, file_list):
                 logging.info("\t\tAdding {} to calibration set".format(filename))
                 pipeline_files.append(filename)
 
-        outputs = run_parallel_pipeline(pipeline_files, "dark", "rate", self.instrument)
+        # Specify that we want to skip the dark current correction step
+        step_args = {'dark_current': {'skip': True}}
+
+        # Call the pipeline
+        outputs = run_parallel_pipeline(pipeline_files, "dark", ["rate"], self.instrument, step_args=step_args)
         for filename in file_list:
             processed_file = filename.replace("_dark", "_rate")
             if processed_file not in slope_files and os.path.isfile(processed_file):
@@ -810,7 +805,7 @@ def process(self, file_list):
                 logging.info('\tFound {} new noisy pixels'.format(len(new_noisy_pixels[0])))
                 self.add_bad_pix(new_noisy_pixels, 'noisy', file_list, mean_slope_file, baseline_file, min_time, mid_time, max_time)
 
-            logging.info("Creating Mean Slope Image {}".format(slope_image))
+            logging.info("Creating Mean Slope Image")
             # Create png file of mean slope image. Add bad pixels only for full frame apertures
             self.create_mean_slope_figure(slope_image, len(slope_files), hotxy=new_hot_pix, deadxy=new_dead_pix,
                                           noisyxy=new_noisy_pixels, baseline_file=baseline_file)
@@ -1027,6 +1022,19 @@ def run(self):
                             # Copy files from filesystem
                             dark_files, not_copied = copy_files(new_filenames, self.data_dir)
 
+                            # Check that there were no problems with the file copying. If any of the copied
+                            # files have different sizes between the MAST filesystem and the JWQL filesystem,
+                            # then throw them out.
+                            for dark_file in dark_files:
+                                copied_size = os.stat(dark_file).st_size
+                                orig_size = os.stat(filesystem_path(os.path.basename(dark_file))).st_size
+                                if orig_size != copied_size:
+                                    logging.info(f"\tProblem copying {os.path.basename(dark_file)} from the filesystem.")
+                                    logging.info(f"Size in filesystem: {orig_size}, size of copy: {copied_size}. Skipping file.")
+                                    not_copied.append(dark_file)
+                                    dark_files.remove(dark_file)
+                                    os.remove(dark_file)
+
                             logging.info('\tNew_filenames: {}'.format(new_filenames))
                             logging.info('\tData dir: {}'.format(self.data_dir))
                             logging.info('\tCopied to working dir: {}'.format(dark_files))

jwql/instrument_monitors/pipeline_tools.py

@@ -25,6 +25,7 @@
 from jwst.dq_init import DQInitStep
 from jwst.dark_current import DarkCurrentStep
 from jwst.firstframe import FirstFrameStep
+from jwst.gain_scale import GainScaleStep
 from jwst.group_scale import GroupScaleStep
 from jwst.ipc import IPCStep
 from jwst.jump import JumpStep
@@ -34,6 +35,7 @@
 from jwst.pipeline.calwebb_detector1 import Detector1Pipeline
 from jwst.ramp_fitting import RampFitStep
 from jwst.refpix import RefPixStep
+from jwst.reset import ResetStep
 from jwst.rscd import RscdStep
 from jwst.saturation import SaturationStep
 from jwst.superbias import SuperBiasStep
@@ -43,16 +45,17 @@
 
 # Define the fits header keyword that accompanies each step
 PIPE_KEYWORDS = {'S_GRPSCL': 'group_scale', 'S_DQINIT': 'dq_init', 'S_SATURA': 'saturation',
-                 'S_REFPIX': 'refpix', 'S_SUPERB': 'superbias',
+                 'S_REFPIX': 'refpix', 'S_SUPERB': 'superbias', 'S_RESET': 'reset',
                  'S_PERSIS': 'persistence', 'S_DARK': 'dark_current', 'S_LINEAR': 'linearity',
                  'S_FRSTFR': 'firstframe', 'S_LASTFR': 'lastframe', 'S_RSCD': 'rscd',
-                 'S_JUMP': 'jump', 'S_RAMP': 'rate'}
+                 'S_JUMP': 'jump', 'S_RAMP': 'rate', 'S_GANSCL': 'gain_scale', 'S_IPC': 'ipc'}
 
 PIPELINE_STEP_MAPPING = {'dq_init': DQInitStep, 'dark_current': DarkCurrentStep,
-                         'firstframe': FirstFrameStep, 'group_scale': GroupScaleStep,
-                         'ipc': IPCStep, 'jump': JumpStep, 'lastframe': LastFrameStep,
-                         'linearity': LinearityStep, 'persistence': PersistenceStep,
-                         'rate': RampFitStep, 'refpix': RefPixStep, 'rscd': RscdStep,
+                         'firstframe': FirstFrameStep, 'gain_scale': GainScaleStep,
+                         'group_scale': GroupScaleStep, 'ipc': IPCStep, 'jump': JumpStep,
+                         'lastframe': LastFrameStep, 'linearity': LinearityStep,
+                         'persistence': PersistenceStep, 'rate': RampFitStep,
+                         'refpix': RefPixStep, 'reset': ResetStep, 'rscd': RscdStep,
                          'saturation': SaturationStep, 'superbias': SuperBiasStep}
 
 # Readout patterns that have nframes != a power of 2. These readout patterns
@@ -165,12 +168,8 @@ def get_pipeline_steps(instrument):
 
     # Order is important in 'steps' lists below!!
     if instrument == 'MIRI':
-        steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', 'lastframe',
-                 'linearity', 'rscd', 'dark_current', 'refpix', 'persistence', 'jump', 'rate']
-        # No persistence correction for MIRI
-        steps.remove('persistence')
-        # MIRI is limited to one frame per group
-        steps.remove('group_scale')
+        steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'firstframe', 'lastframe', 'reset',
+                 'linearity', 'rscd', 'dark_current', 'refpix', 'jump', 'rate', 'gain_scale']
     else:
         steps = ['group_scale', 'dq_init', 'saturation', 'ipc', 'superbias', 'refpix', 'linearity',
                  'persistence', 'dark_current', 'jump', 'rate']
@@ -186,13 +185,10 @@ def get_pipeline_steps(instrument):
     # IPC correction currently not done for any instrument
     steps.remove('ipc')
 
-    # Initialize using PIPE_KEYWORDS so the steps will be in the right order
+    # Initialize using OrderedDict so the steps will be in the right order
     required_steps = OrderedDict({})
     for key in steps:
         required_steps[key] = True
-    for key in PIPE_KEYWORDS.values():
-        if key not in required_steps.keys():
-            required_steps[key] = False
 
     return required_steps