Merge branch 'develop' into issue-1354

README.md

@@ -83,16 +83,16 @@ source activate base/root
 
 **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step.
 
-Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
+Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
 
 ```
-conda env create -f environment_python_3_9.yml
+conda env create -f environment_python_3.9.yml
 ```
 
 or
 
 ```
-conda env create -f environment_python_3_10.yml
+conda env create -f environment_python_3.10.yml
 ```
 
 ### Configuration File

environment_python_3.10.yml

@@ -22,49 +22,49 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.2
+  - astropy=5.3.4
   - beautifulsoup4=4.12.2
   - bokeh=2.4.3
-  - celery=5.3.1
-  - cryptography=41.0.3
-  - django=4.2.3
+  - celery=5.3.4
+  - cryptography=41.0.4
+  - django=4.2.6
   - inflection=0.5.1
-  - ipython=8.14.0
+  - ipython=8.16.1
   - jinja2=3.1.2
-  - jsonschema=4.19.0
-  - matplotlib=3.7.2
-  - nodejs=20.5.1
+  - jsonschema=4.19.1
+  - matplotlib=3.8.0
+  - nodejs=20.8.0
   - numpy=1.25.2
   - numpydoc=1.5.0
-  - pandas=2.0.3
+  - pandas=2.1.1
   - pip=23.2.1
   - postgresql=15.4
-  - psycopg2=2.9.6
-  - pytest=7.4.0
+  - psycopg2=2.9.7
+  - pytest=7.4.2
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.10.12
-  - pyyaml=6.0
-  - redis
-  - ruff=0.0.285
+  - pyyaml=6.0.1
+  - redis=5.0.0
+  - ruff=0.0.292
   - scipy=1.9.3
-  - setuptools=68.0.0
-  - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.2
-  - sqlalchemy=2.0.20
+  - setuptools=68.2.2
+  - sphinx=7.2.6
+  - sphinx_rtd_theme=1.3.0
+  - sqlalchemy=2.0.21
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
     - astroquery==0.4.6
     - bandit==1.7.5
-    - jwst==1.11.4
+    - jwst==1.12.3
     - pysiaf==0.20.0
-    - pysqlite3==0.5.1
+    - pysqlite3==0.5.2
     - pyvo==1.4.2
     - redis==5.0.0
-    - selenium==4.11.2
-    - stdatamodels==1.7.2
+    - selenium==4.13.0
+    - stdatamodels==1.8.3
     - stsci_rtd_theme==1.0.0
     - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles

environment_python_3.9.yml

@@ -22,49 +22,49 @@ channels:
   - defaults
 
 dependencies:
-  - astropy=5.3.2
+  - astropy=5.3.3
   - beautifulsoup4=4.12.2
   - bokeh=2.4.3
-  - celery=5.3.1
-  - cryptography=41.0.3
-  - django=4.2.3
+  - celery=5.3.4
+  - cryptography=41.0.4
+  - django=4.2.5
   - inflection=0.5.1
-  - ipython=8.14.0
+  - ipython=8.16.1
   - jinja2=3.1.2
-  - jsonschema=4.19.0
-  - matplotlib=3.7.2
-  - nodejs=20.5.1
+  - jsonschema=4.19.1
+  - matplotlib=3.8.0
+  - nodejs=20.8.0
   - numpy=1.25.2
   - numpydoc=1.5.0
-  - pandas=2.0.3
+  - pandas=2.1.1
   - pip=23.2.1
   - postgresql=15.4
-  - psycopg2=2.9.6
-  - pytest=7.4.0
+  - psycopg2=2.9.7
+  - pytest=7.4.2
   - pytest-cov=4.1.0
   - pytest-mock=3.11.1
   - python=3.9.17
-  - pyyaml=6.0
-  - redis
-  - ruff=0.0.285
+  - pyyaml=6.0.1
+  - redis=5.0.0
+  - ruff=0.0.292
   - scipy=1.9.3
-  - setuptools=68.0.0
-  - sphinx=6.2.1
-  - sphinx_rtd_theme=1.2.2
-  - sqlalchemy=2.0.20
+  - setuptools=68.2.2
+  - sphinx=7.2.6
+  - sphinx_rtd_theme=1.3.0
+  - sqlalchemy=2.0.21
   - twine=4.0.2
   - wtforms=3.0.1
 
   - pip:
     - astroquery==0.4.6
     - bandit==1.7.5
-    - jwst==1.11.4
+    - jwst==1.12.3
     - pysiaf==0.20.0
-    - pysqlite3==0.5.1
+    - pysqlite3==0.5.2
     - pyvo==1.4.2
     - redis==5.0.0
-    - selenium==4.11.2
-    - stdatamodels==1.7.2
+    - selenium==4.13.0
+    - stdatamodels==1.8.3
     - stsci_rtd_theme==1.0.0
     - vine==5.0.0
     - git+https://github.com/spacetelescope/jwst_reffiles

jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py

@@ -984,9 +984,10 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun
                 else:
                     index += 1
 
-            min_dark_time = min(dark_obstimes)
-            max_dark_time = max(dark_obstimes)
-            mid_dark_time = instrument_properties.mean_time(dark_obstimes)
+            if len(dark_slope_files) > 0:
+                min_dark_time = min(dark_obstimes)
+                max_dark_time = max(dark_obstimes)
+                mid_dark_time = instrument_properties.mean_time(dark_obstimes)
 
         # Check whether there are still enough files left to meet the threshold
         if illuminated_slope_files is None:

jwql/instrument_monitors/common_monitors/dark_monitor.py

@@ -84,7 +84,6 @@
 from astropy.modeling import models
 from astropy.stats import sigma_clipped_stats
 from astropy.time import Time
-from bokeh.io import export_png
 from bokeh.models import ColorBar, ColumnDataSource, HoverTool, Legend
 from bokeh.models import LinearColorMapper
 from bokeh.plotting import figure
@@ -106,7 +105,7 @@
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
+from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png
 
 THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt')
 
@@ -276,14 +275,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
 
             # Create figure
             start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y")
-            end_time  = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y")
+            end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y")
 
             self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='')
             #                   tools='pan,box_zoom,reset,wheel_zoom,save')
             self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0
 
             # Create the color mapper that will be used to scale the image
-            mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
+            mapper = LinearColorMapper(palette='Viridis256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev)))
 
             # Plot image and add color bar
             imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny,
@@ -292,13 +291,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
             color_bar = ColorBar(color_mapper=mapper, width=8, title='DN/sec')
             self.plot.add_layout(color_bar, 'right')
 
-            # Add hover tool for all pixel values
-            #hover_tool = HoverTool(tooltips=[('(x, y):', '($x{int}, $y{int})'),
-            #                                 ('value:', '@image')
-            #                                 ],
-            #                       renderers=[imgplot])
-            #self.plot.tools.append(hover_tool)
-
             if (('FULL' in self.aperture) or ('_CEN' in self.aperture)):
 
                 if hotxy is not None:
@@ -338,22 +330,21 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
                     base_start = Time(float(base_parts[3]), format='mjd').tt.datetime
                     base_end = Time(float(base_parts[5]), format='mjd').tt.datetime
                     base_start_time = base_start.strftime("%m/%d/%Y")
-                    base_end_time  = base_end.strftime("%m/%d/%Y")
+                    base_end_time = base_end.strftime("%m/%d/%Y")
                     legend_title = f'Compared to dark from {base_start_time} to {base_end_time}'
                 else:
                     legend_title = 'Compared to previous mean dark'
                 legend = Legend(items=[hot_legend, dead_legend, noisy_legend],
                                 location="center",
                                 orientation='vertical',
-                                title = legend_title)
+                                title=legend_title)
 
                 self.plot.add_layout(legend, 'below')
 
             # Save the plot in a png
-            export_png(self.plot, filename=output_filename)
+            save_png(self.plot, filename=output_filename)
             set_permissions(output_filename)
 
-
     def get_metadata(self, filename):
         """Collect basic metadata from a fits file
 
@@ -650,15 +641,15 @@ def overplot_bad_pix(self, pix_type, coords, values):
 
         sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0],
                                                        pixels_y=coords[1]
-                                                      )
+                                                       )
                                              )
 
         # Overplot the bad pixel locations
         badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y',
                                                  source=sources[pix_type], color=colors[pix_type])
 
         # Add to the legend
-        if  numpix > 0:
+        if numpix > 0:
             if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT:
                 text = f"{numpix} pix {adjective[pix_type]} than baseline"
             else:

jwql/shared_tasks/run_pipeline.py

@@ -10,6 +10,7 @@
 import shutil
 import sys
 import time
+import traceback
 
 from jwst import datamodels
 from jwst.dq_init import DQInitStep
@@ -173,7 +174,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co
         with open(status_file, "a+") as status_f:
             status_f.write("EXCEPTION\n")
             status_f.write("{}\n".format(e))
-            status_f.write("FAILED")
+            status_f.write("FAILED\n")
+            status_f.write(traceback.format_exc())
         sys.exit(1)
 
     with open(status_file, "a+") as status_f:
@@ -229,6 +231,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
             params['refpix'] = dict(odd_even_rows=False)
 
         # Default CR rejection threshold is too low
+        params['jump'] = {}
         params['jump']['rejection_threshold'] = 15
 
         # Set up to save jump step output
@@ -296,7 +299,8 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
         with open(status_file, "a+") as status_f:
             status_f.write("EXCEPTION\n")
             status_f.write("{}\n".format(e))
-            status_f.write("FAILED")
+            status_f.write("FAILED\n")
+            status_f.write(traceback.format_exc())
         sys.exit(1)
 
     with open(status_file, "a+") as status_f:
@@ -356,7 +360,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T
     outputs = args.outputs
     step_args = args.step_args
 
-    status_file = os.path.join(working_path, short_name+"_status.txt")
+    status_file = os.path.join(working_path, short_name + "_status.txt")
     with open(status_file, 'w') as out_file:
         out_file.write("Starting Process\n")
         out_file.write("\tpipeline is {} ({})\n".format(pipe_type, type(pipe_type)))