Merge branch 'develop' into remove-readnoise-bokeh

README.md

@@ -83,16 +83,16 @@ source activate base/root
 
 **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step.
 
-Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
+Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
 
 ```
-conda env create -f environment_python_3_9.yml
+conda env create -f environment_python_3.9.yml
 ```
 
 or
 
 ```
-conda env create -f environment_python_3_10.yml
+conda env create -f environment_python_3.10.yml
 ```
 
 ### Configuration File

jwql/instrument_monitors/common_monitors/dark_monitor.py

@@ -84,7 +84,6 @@
 from astropy.modeling import models
 from astropy.stats import sigma_clipped_stats
 from astropy.time import Time
-from bokeh.io import export_png
 from bokeh.models import ColorBar, ColumnDataSource, HoverTool, Legend
 from bokeh.models import LinearColorMapper
 from bokeh.plotting import figure
@@ -106,7 +105,7 @@
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
+from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png
 
 THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt')
 
@@ -276,14 +275,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
 
             # Create figure
             start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y")
-            end_time  = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y")
+            end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y")
 
             self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='')
             #                   tools='pan,box_zoom,reset,wheel_zoom,save')
             self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0
 
             # Create the color mapper that will be used to scale the image
-            mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
+            mapper = LinearColorMapper(palette='Viridis256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev)))
 
             # Plot image and add color bar
             imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny,
@@ -292,13 +291,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
             color_bar = ColorBar(color_mapper=mapper, width=8, title='DN/sec')
             self.plot.add_layout(color_bar, 'right')
 
-            # Add hover tool for all pixel values
-            #hover_tool = HoverTool(tooltips=[('(x, y):', '($x{int}, $y{int})'),
-            #                                 ('value:', '@image')
-            #                                 ],
-            #                       renderers=[imgplot])
-            #self.plot.tools.append(hover_tool)
-
             if (('FULL' in self.aperture) or ('_CEN' in self.aperture)):
 
                 if hotxy is not None:
@@ -338,22 +330,21 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no
                     base_start = Time(float(base_parts[3]), format='mjd').tt.datetime
                     base_end = Time(float(base_parts[5]), format='mjd').tt.datetime
                     base_start_time = base_start.strftime("%m/%d/%Y")
-                    base_end_time  = base_end.strftime("%m/%d/%Y")
+                    base_end_time = base_end.strftime("%m/%d/%Y")
                     legend_title = f'Compared to dark from {base_start_time} to {base_end_time}'
                 else:
                     legend_title = 'Compared to previous mean dark'
                 legend = Legend(items=[hot_legend, dead_legend, noisy_legend],
                                 location="center",
                                 orientation='vertical',
-                                title = legend_title)
+                                title=legend_title)
 
                 self.plot.add_layout(legend, 'below')
 
             # Save the plot in a png
-            export_png(self.plot, filename=output_filename)
+            save_png(self.plot, filename=output_filename)
             set_permissions(output_filename)
 
-
     def get_metadata(self, filename):
         """Collect basic metadata from a fits file
 
@@ -650,15 +641,15 @@ def overplot_bad_pix(self, pix_type, coords, values):
 
         sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0],
                                                        pixels_y=coords[1]
-                                                      )
+                                                       )
                                              )
 
         # Overplot the bad pixel locations
         badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y',
                                                  source=sources[pix_type], color=colors[pix_type])
 
         # Add to the legend
-        if  numpix > 0:
+        if numpix > 0:
             if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT:
                 text = f"{numpix} pix {adjective[pix_type]} than baseline"
             else:

jwql/tests/test_utils.py

@@ -22,7 +22,11 @@
 from pathlib import Path
 import pytest
 
-from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, _validate_config
+from bokeh.models import LinearColorMapper
+from bokeh.plotting import figure
+import numpy as np
+
+from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, save_png, _validate_config
 
 
 # Determine if tests are being run on Github Actions
@@ -479,6 +483,17 @@ def test_filesystem_path():
     assert check == location
 
 
+def test_save_png():
+    """Test that we can create a png file"""
+    plot = figure(title='test', tools='')
+    image = np.zeros((200, 200))
+    image[100:105, 100:105] = 1
+    ny, nx = image.shape
+    mapper = LinearColorMapper(palette='Viridis256', low=0, high=1.1)
+    imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image")
+    save_png(plot, filename='test.png')
+
+
 @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.')
 def test_validate_config():
     """Test that the config validator works."""

jwql/utils/utils.py

@@ -45,14 +45,15 @@
 from bokeh.plotting import figure
 import numpy as np
 from PIL import Image
+from selenium import webdriver
 
 from jwql.utils import permissions
 from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \
-                                 FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \
-                                 FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \
-                                 FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \
-                                 FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \
-                                 JWST_INSTRUMENT_NAMES_SHORTHAND
+    FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \
+    FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \
+    FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \
+    FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \
+    JWST_INSTRUMENT_NAMES_SHORTHAND
 
 __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 
@@ -157,8 +158,7 @@ def create_png_from_fits(filename, outdir):
         plot.ygrid.visible = False
 
         # Create the color mapper that will be used to scale the image
-        #mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
-        mapper = LogColorMapper(palette='Greys256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev))
+        mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev)))
 
         # Plot image
         imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny,
@@ -169,8 +169,8 @@ def create_png_from_fits(filename, outdir):
         plot.yaxis.visible = False
 
         # Save the plot in a png
-        output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png'))
-        export_png(plot, filename=output_filename)
+        output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png'))
+        save_png(plot, filename=output_filename)
         permissions.set_permissions(output_filename)
         return output_filename
     else:
@@ -749,13 +749,34 @@ def read_png(filename):
 
         # Copy the RGBA image into view, flipping it so it comes right-side up
         # with a lower-left origin
-        view[:,:,:] = np.flipud(np.asarray(rgba_img))
+        view[:, :, :] = np.flipud(np.asarray(rgba_img))
     else:
         view = None
     # Return the 2D version
     return img
 
 
+def save_png(fig, filename=''):
+    """Starting with selenium version 4.10.0, our testing has shown that on the JWQL
+    servers, we need to specify an instance of a web driver when exporting a Bokeh
+    figure as a png. This is a wrapper function that creates the web driver instance
+    and calls Bokeh's export_png function.
+
+    Parameters
+    ----------
+    fig : bokeh.plotting.figure
+        Bokeh figure to be saved as a png
+
+    filename : str
+        Filename to use for the png file
+    """
+    options = webdriver.FirefoxOptions()
+    options.add_argument('-headless')
+    driver = webdriver.Firefox(options=options)
+    export_png(fig, filename=filename, webdriver=driver)
+    driver.quit()
+
+
 def grouper(iterable, chunksize):
     """
     Take a list of items (iterable), and group it into chunks of chunksize, with the

jwql/website/apps/jwql/migrations/0001_initial.py

@@ -0,0 +1,45 @@
+# Generated by Django 3.1.7 on 2022-09-09 17:47
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ImageData',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('inst', models.CharField(choices=[('FGS', 'FGS'), ('MIRI', 'MIRI'), ('NIRCam', 'NIRCam'), ('NIRISS', 'NIRISS'), ('NIRSpec', 'NIRSpec')], default=None, max_length=7, verbose_name='instrument')),
+                ('pub_date', models.DateTimeField(verbose_name='date published')),
+                ('filepath', models.FilePathField(path='/user/lchambers/jwql/')),
+            ],
+            options={
+                'verbose_name_plural': 'image data',
+                'db_table': 'imagedata',
+            },
+        ),
+        migrations.CreateModel(
+            name='InstrumentFilterHandler',
+            fields=[
+                ('instrument', models.CharField(max_length=10, primary_key=True, serialize=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ThumbnailFilterInfo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('observation', models.PositiveIntegerField()),
+                ('proposal', models.PositiveIntegerField()),
+                ('root_name', models.CharField(max_length=300)),
+                ('marked_viewed', models.BooleanField(default=False)),
+                ('inst_handler', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.instrumentfilterhandler')),
+            ],
+        ),
+    ]

jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.1.7 on 2022-09-13 20:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='thumbnailfilterinfo',
+            name='id',
+        ),
+        migrations.AlterField(
+            model_name='instrumentfilterhandler',
+            name='instrument',
+            field=models.TextField(max_length=10, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='thumbnailfilterinfo',
+            name='root_name',
+            field=models.TextField(max_length=300, primary_key=True, serialize=False),
+        ),
+    ]

jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py

@@ -0,0 +1,58 @@
+# Generated by Django 3.1.7 on 2022-09-21 14:55
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0002_auto_20220913_1525'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Archive',
+            fields=[
+                ('instrument', models.CharField(help_text='Instrument name', max_length=7, primary_key=True, serialize=False)),
+            ],
+            options={
+                'ordering': ['instrument'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Observation',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('obsnum', models.CharField(help_text='Observation number, as a 3 digit string', max_length=3)),
+                ('number_of_files', models.IntegerField(default=0, help_text='Number of files in the proposal')),
+                ('obsstart', models.FloatField(default=0.0, help_text='Time of the beginning of the observation in MJD')),
+                ('obsend', models.FloatField(default=0.0, help_text='Time of the end of the observation in MJD')),
+                ('exptypes', models.CharField(default='', help_text='Comma-separated list of exposure types', max_length=100)),
+            ],
+            options={
+                'ordering': ['-obsnum'],
+            },
+        ),
+        migrations.CreateModel(
+            name='Proposal',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('prop_id', models.CharField(help_text='5-digit proposal ID string', max_length=5)),
+                ('thumbnail_path', models.CharField(default='', help_text='Path to the proposal thumbnail', max_length=100)),
+                ('archive', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.archive')),
+            ],
+            options={
+                'ordering': ['-prop_id'],
+                'unique_together': {('prop_id', 'archive')},
+            },
+        ),
+        migrations.DeleteModel(
+            name='ImageData',
+        ),
+        migrations.AddField(
+            model_name='observation',
+            name='proposal',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.proposal'),
+        ),
+    ]

jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py

@@ -0,0 +1,33 @@
+# Generated by Django 3.1.7 on 2022-09-22 14:11
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jwql', '0003_auto_20220921_0955'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='thumbnailfilterinfo',
+            name='obsnum',
+            field=models.CharField(default=11, help_text='Observation number, as a 3 digit string', max_length=3),
+            preserve_default=False,
+        ),
+        migrations.CreateModel(
+            name='RootFileInfo',
+            fields=[
+                ('instrument', models.CharField(help_text='Instrument name', max_length=7)),
+                ('proposal', models.CharField(help_text='5-digit proposal ID string', max_length=5)),
+                ('root_name', models.TextField(max_length=300, primary_key=True, serialize=False)),
+                ('viewed', models.BooleanField(default=False)),
+                ('obsnum', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.observation')),
+            ],
+            options={
+                'ordering': ['-root_name'],
+            },
+        ),
+    ]