From d2e4aae701bdfad403e2320f5ace0fe66524eab3 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Fri, 13 Apr 2018 16:11:21 -0400
Subject: [PATCH 01/23] Initial commit of module to generate preview images

---
 jwql/preview_image/generate_preview_images.py | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 jwql/preview_image/generate_preview_images.py

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
new file mode 100644
index 000000000..e49a0c5d9
--- /dev/null
+++ b/jwql/preview_image/generate_preview_images.py
@@ -0,0 +1,66 @@
+#! /usr/bin/env python
+
+"""Generate preview images for all files in the jwql filesystem.
+
+Execution of this script will generate preview images for each file in
+the jwql filesystem, if it does not already exist.
+
+Authors
+-------
+
+    Matthew Bourque
+
+
+Use
+---
+
+    This script is intended to be executed as such:
+
+    >>> python generate_preview_images.py
+
+Dependencies
+------------
+"""
+
+import glob
+import os
+
+from jwql.permissions import permissions
+from jwql.preview_image.preview_image import PreviewImage
+from jwql.utils.utils import get_config
+
+
+def generate_preview_images():
+    """The main function of the generate_preview_image module."""
+
+    filesystem = get_config()['filesystem']
+    preview_image_filesystem = get_config()['preview_image_filesystem']
+    filenames = glob.glob(os.path.join(filesystem, '*/*.fits'))
+
+    for filename in filenames:
+
+        # Determine the save location
+        rootname_elements = os.path.basename(filename).split('.fits')[0].split('_')
+        preview_image_dir = '{}_{}'.format(
+            rootname_elements[0],  # Program ID + observation + visit
+            rootname_elements[1])  # Visit group + parallel sequence ID + activity
+        output_directory = os.path.join(preview_image_filesystem, preview_image_dir)
+
+        # Create the output directory if necessary
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+            permissions.set_permissions(output_directory, verbose=False)
+
+        # Create and save the preview image
+        im = PreviewImage(filename, "SCI")
+        im.clip_percent = 0.01
+        im.scaling = 'log'
+        im.output_format = 'jpg'
+        im.output_directory = output_directory
+        im.make_image()
+        print('Created preview image for {}'.format(filename))
+
+
+if __name__ == '__main__':
+
+    generate_preview_images()

From 2b61034e50c9943a2bbef1c4af207250bc88a6f8 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Tue, 17 Apr 2018 11:26:22 -0400
Subject: [PATCH 02/23] Fixed bug in utils.py where the config file couldnt be
 found unless the script was in the utils subpackage.

---
 jwql/utils/utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
index a0f122afc..ead466e52 100644
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -15,6 +15,9 @@
 """
 
 import json
+import os
+
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
 
 
 def get_config():
@@ -27,7 +30,7 @@ def get_config():
         A dictionary that holds the contents of the config file.
     """
 
-    with open('config.json', 'r') as config_file:
+    with open(os.path.join(__location__, 'config.json'), 'r') as config_file:
         settings = json.load(config_file)
 
     return settings

From 7704b14aea88a0e21d26276af38cfd97496c81b8 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 19 Apr 2018 12:05:37 -0400
Subject: [PATCH 03/23] Allowing for user to define the colormap used, default
 switched from "gray" to "viridis".  Also changed variable name "fname" to
 "outfile" to be more consistent within the module.

---
 jwql/preview_image/preview_image.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py
index 34f6521c7..e3f682a5c 100755
--- a/jwql/preview_image/preview_image.py
+++ b/jwql/preview_image/preview_image.py
@@ -36,11 +36,12 @@
 import sys
 
 from astropy.io import fits
+from jwst.datamodels import dqflags
 import matplotlib.pyplot as plt
 import matplotlib.colors as colors
 import numpy as np
 
-from jwst.datamodels import dqflags
+from jwql.permissions import permissions
 
 class PreviewImage():
 
@@ -57,6 +58,7 @@ def __init__(self, filename, extension):
         self.file = filename
         self.clip_percent = 0.01
         self.scaling = 'log'
+        self.cmap = 'viridis'
         self.output_format = 'jpg'
         self.output_directory = None
 
@@ -205,7 +207,7 @@ def make_figure(self, image, integration_number, min_value, max_value,
             # Log normalize the colormap
             cax = ax.imshow(shiftdata,
                             norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
-                            cmap='gray')
+                            cmap=self.cmap)
 
             # Add colorbar, with original data values
             tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
@@ -279,15 +281,18 @@ def make_image(self):
             plt.close()
 
     @staticmethod
-    def save_image(image, fname):
+    def save_image(image, outfile):
         """
-        Save an image in the requested output format
+        Save an image in the requested output format and sets the
+        appropriate permissions
 
         Parameters:
         ----------
         image : obj
             A matplotlib figure object
-        fname : str
+        outfile : str
             Output filename
         """
-        image.savefig(fname, bbox_inches='tight')
+        image.savefig(outfile, bbox_inches='tight')
+        permissions.set_permissions(outfile, verbose=False)
+        print('Saved image to {}'.format(outfile))

From fcf0fa00b4e361ce65d3529718dfc14cec9eddde Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 19 Apr 2018 12:06:36 -0400
Subject: [PATCH 04/23] Adding try/except around preview image generation so
 that script doesnt outright crash.

---
 jwql/preview_image/generate_preview_images.py | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index e49a0c5d9..20b947200 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -1,9 +1,9 @@
 #! /usr/bin/env python
 
-"""Generate preview images for all files in the jwql filesystem.
+"""Generate preview images for all files in the ``jwql`` filesystem.
 
 Execution of this script will generate preview images for each file in
-the jwql filesystem, if it does not already exist.
+the ``jwql`` filesystem, if it does not already exist.
 
 Authors
 -------
@@ -16,10 +16,15 @@
 
     This script is intended to be executed as such:
 
-    >>> python generate_preview_images.py
+    ::
 
-Dependencies
-------------
+        python generate_preview_images.py
+
+Notes
+-----
+
+    Some of this code could be simplified by using a filename parser
+    utility function, which is in the works for ``jwql``.
 """
 
 import glob
@@ -52,13 +57,16 @@ def generate_preview_images():
             permissions.set_permissions(output_directory, verbose=False)
 
         # Create and save the preview image
-        im = PreviewImage(filename, "SCI")
-        im.clip_percent = 0.01
-        im.scaling = 'log'
-        im.output_format = 'jpg'
-        im.output_directory = output_directory
-        im.make_image()
-        print('Created preview image for {}'.format(filename))
+        try:
+            im = PreviewImage(filename, "SCI")
+            im.clip_percent = 0.01
+            im.scaling = 'log'
+            im.cmap = 'viridis'
+            im.output_format = 'jpg'
+            im.output_directory = output_directory
+            im.make_image()
+        except ValueError as error:
+            print(error)
 
 
 if __name__ == '__main__':

From bd5338c777d84821eb46b739a8d3b91177802e36 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Fri, 13 Apr 2018 16:11:21 -0400
Subject: [PATCH 05/23] Initial commit of module to generate preview images

---
 jwql/preview_image/generate_preview_images.py | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 jwql/preview_image/generate_preview_images.py

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
new file mode 100644
index 000000000..e49a0c5d9
--- /dev/null
+++ b/jwql/preview_image/generate_preview_images.py
@@ -0,0 +1,66 @@
+#! /usr/bin/env python
+
+"""Generate preview images for all files in the jwql filesystem.
+
+Execution of this script will generate preview images for each file in
+the jwql filesystem, if it does not already exist.
+
+Authors
+-------
+
+    Matthew Bourque
+
+
+Use
+---
+
+    This script is intended to be executed as such:
+
+    >>> python generate_preview_images.py
+
+Dependencies
+------------
+"""
+
+import glob
+import os
+
+from jwql.permissions import permissions
+from jwql.preview_image.preview_image import PreviewImage
+from jwql.utils.utils import get_config
+
+
+def generate_preview_images():
+    """The main function of the generate_preview_image module."""
+
+    filesystem = get_config()['filesystem']
+    preview_image_filesystem = get_config()['preview_image_filesystem']
+    filenames = glob.glob(os.path.join(filesystem, '*/*.fits'))
+
+    for filename in filenames:
+
+        # Determine the save location
+        rootname_elements = os.path.basename(filename).split('.fits')[0].split('_')
+        preview_image_dir = '{}_{}'.format(
+            rootname_elements[0],  # Program ID + observation + visit
+            rootname_elements[1])  # Visit group + parallel sequence ID + activity
+        output_directory = os.path.join(preview_image_filesystem, preview_image_dir)
+
+        # Create the output directory if necessary
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+            permissions.set_permissions(output_directory, verbose=False)
+
+        # Create and save the preview image
+        im = PreviewImage(filename, "SCI")
+        im.clip_percent = 0.01
+        im.scaling = 'log'
+        im.output_format = 'jpg'
+        im.output_directory = output_directory
+        im.make_image()
+        print('Created preview image for {}'.format(filename))
+
+
+if __name__ == '__main__':
+
+    generate_preview_images()

From 8930c9f5b6a9130516f3199be994c395d2ad8757 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 19 Apr 2018 12:05:37 -0400
Subject: [PATCH 06/23] Allowing for user to define the colormap used, default
 switched from "gray" to "viridis".  Also changed variable name "fname" to
 "outfile" to be more consistent within the module.

---
 jwql/preview_image/preview_image.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py
index 34f6521c7..e3f682a5c 100755
--- a/jwql/preview_image/preview_image.py
+++ b/jwql/preview_image/preview_image.py
@@ -36,11 +36,12 @@
 import sys
 
 from astropy.io import fits
+from jwst.datamodels import dqflags
 import matplotlib.pyplot as plt
 import matplotlib.colors as colors
 import numpy as np
 
-from jwst.datamodels import dqflags
+from jwql.permissions import permissions
 
 class PreviewImage():
 
@@ -57,6 +58,7 @@ def __init__(self, filename, extension):
         self.file = filename
         self.clip_percent = 0.01
         self.scaling = 'log'
+        self.cmap = 'viridis'
         self.output_format = 'jpg'
         self.output_directory = None
 
@@ -205,7 +207,7 @@ def make_figure(self, image, integration_number, min_value, max_value,
             # Log normalize the colormap
             cax = ax.imshow(shiftdata,
                             norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
-                            cmap='gray')
+                            cmap=self.cmap)
 
             # Add colorbar, with original data values
             tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
@@ -279,15 +281,18 @@ def make_image(self):
             plt.close()
 
     @staticmethod
-    def save_image(image, fname):
+    def save_image(image, outfile):
         """
-        Save an image in the requested output format
+        Save an image in the requested output format and sets the
+        appropriate permissions
 
         Parameters:
         ----------
         image : obj
             A matplotlib figure object
-        fname : str
+        outfile : str
             Output filename
         """
-        image.savefig(fname, bbox_inches='tight')
+        image.savefig(outfile, bbox_inches='tight')
+        permissions.set_permissions(outfile, verbose=False)
+        print('Saved image to {}'.format(outfile))

From 377e2a4d75a771603de7770498fb3fe7e23ba591 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 19 Apr 2018 12:06:36 -0400
Subject: [PATCH 07/23] Adding try/except around preview image generation so
 that script doesnt outright crash.

---
 jwql/preview_image/generate_preview_images.py | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index e49a0c5d9..20b947200 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -1,9 +1,9 @@
 #! /usr/bin/env python
 
-"""Generate preview images for all files in the jwql filesystem.
+"""Generate preview images for all files in the ``jwql`` filesystem.
 
 Execution of this script will generate preview images for each file in
-the jwql filesystem, if it does not already exist.
+the ``jwql`` filesystem, if it does not already exist.
 
 Authors
 -------
@@ -16,10 +16,15 @@
 
     This script is intended to be executed as such:
 
-    >>> python generate_preview_images.py
+    ::
 
-Dependencies
-------------
+        python generate_preview_images.py
+
+Notes
+-----
+
+    Some of this code could be simplified by using a filename parser
+    utility function, which is in the works for ``jwql``.
 """
 
 import glob
@@ -52,13 +57,16 @@ def generate_preview_images():
             permissions.set_permissions(output_directory, verbose=False)
 
         # Create and save the preview image
-        im = PreviewImage(filename, "SCI")
-        im.clip_percent = 0.01
-        im.scaling = 'log'
-        im.output_format = 'jpg'
-        im.output_directory = output_directory
-        im.make_image()
-        print('Created preview image for {}'.format(filename))
+        try:
+            im = PreviewImage(filename, "SCI")
+            im.clip_percent = 0.01
+            im.scaling = 'log'
+            im.cmap = 'viridis'
+            im.output_format = 'jpg'
+            im.output_directory = output_directory
+            im.make_image()
+        except ValueError as error:
+            print(error)
 
 
 if __name__ == '__main__':

From 551f3a4639e44ae12a4c8be15aefce757ed6aded Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Fri, 27 Apr 2018 11:35:55 -0400
Subject: [PATCH 08/23] Permissions should allow group execution in order to
 actually open the files.

---
 jwql/permissions/permissions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/permissions/permissions.py b/jwql/permissions/permissions.py
index 5c8b1aa45..b9889d026 100644
--- a/jwql/permissions/permissions.py
+++ b/jwql/permissions/permissions.py
@@ -77,7 +77,7 @@
 DEFAULT_GROUP = 'jwql_dev'
 
 # set the default mode for DEFAULT_OWNER
-DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP  # equivalent to '?rwxr-----'
+DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP # equivalent to '?rwxr-x---'
 
 def get_group_string(pathname):
     """Return the group of pathname in string representation.

From 235161275a4cbe9c63c6c865b6aba50b7555fd1a Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Fri, 27 Apr 2018 11:37:12 -0400
Subject: [PATCH 09/23] Changes to enable generation of thumbnail images that
 have to axis labels, title, or colorbar.

---
 jwql/preview_image/generate_preview_images.py | 16 ++++
 jwql/preview_image/preview_image.py           | 81 +++++++++++--------
 2 files changed, 64 insertions(+), 33 deletions(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index 20b947200..ad0475d09 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -40,6 +40,7 @@ def generate_preview_images():
 
     filesystem = get_config()['filesystem']
     preview_image_filesystem = get_config()['preview_image_filesystem']
+    thumbnail_filesystem = get_config()['thumbnail_filesystem']
     filenames = glob.glob(os.path.join(filesystem, '*/*.fits'))
 
     for filename in filenames:
@@ -50,6 +51,7 @@ def generate_preview_images():
             rootname_elements[0],  # Program ID + observation + visit
             rootname_elements[1])  # Visit group + parallel sequence ID + activity
         output_directory = os.path.join(preview_image_filesystem, preview_image_dir)
+        thumbnail_output_directory = os.path.join(thumbnail_filesystem, preview_image_dir)
 
         # Create the output directory if necessary
         if not os.path.exists(output_directory):
@@ -68,6 +70,20 @@ def generate_preview_images():
         except ValueError as error:
             print(error)
 
+        # Create and save a thumbnail of the preview image
+        try:
+            im = PreviewImage(filename, "SCI")
+            im.clip_percent = 0.01
+            im.scaling = 'log'
+            im.cmap = 'viridis'
+            im.output_format = 'thumb'
+            im.output_directory = thumbnail_output_directory
+            im.thumbnail = True
+            print(im)
+            im.make_image()
+        except ValueError as error:
+            print(error)
+
 
 if __name__ == '__main__':
 
diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py
index e3f682a5c..a58960fa9 100755
--- a/jwql/preview_image/preview_image.py
+++ b/jwql/preview_image/preview_image.py
@@ -37,12 +37,17 @@
 
 from astropy.io import fits
 from jwst.datamodels import dqflags
-import matplotlib.pyplot as plt
-import matplotlib.colors as colors
 import numpy as np
 
 from jwql.permissions import permissions
 
+# Use the 'Agg' backend to avoid invoking $DISPLAY
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import matplotlib.colors as colors
+
+
 class PreviewImage():
 
     def __init__(self, filename, extension):
@@ -55,12 +60,13 @@ def __init__(self, filename, extension):
         extension : str
             Extension name to be read in
         """
-        self.file = filename
         self.clip_percent = 0.01
-        self.scaling = 'log'
         self.cmap = 'viridis'
+        self.file = filename
         self.output_format = 'jpg'
         self.output_directory = None
+        self.scaling = 'log'
+        self.thumbnail = False
 
         # Read in file
         self.data, self.dq = self.get_data(self.file, extension)
@@ -159,8 +165,7 @@ def get_data(self, filename, ext):
                                      .format(filename)))
         return data, dq
 
-    def make_figure(self, image, integration_number, min_value, max_value,
-                    scale, maxsize = 8):
+    def make_figure(self, image, integration_number, min_value, max_value, scale):
         """
         Create the matplotlib figure of the image
 
@@ -182,12 +187,17 @@ def make_figure(self, image, integration_number, min_value, max_value,
         result : obj
             Matplotlib Figure object
         """
+
         # Check the input scaling
         if scale not in ['linear','log']:
             raise ValueError(("WARNING: scaling option {} not supported."
                               .format(scale)))
 
         # Set the figure size
+        if self.thumbnail:
+            maxsize = 2
+        else:
+            maxsize = 8
         yd, xd = image.shape
         ratio = yd / xd
         if xd >= yd:
@@ -210,36 +220,41 @@ def make_figure(self, image, integration_number, min_value, max_value,
                             cmap=self.cmap)
 
             # Add colorbar, with original data values
-            tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
-            tlabelflt = tickvals + min_value - 1
-
-            # Adjust the number of digits after the decimal point
-            # in the colorbar labels based on the signal range
-            delta = tlabelflt[-1] - tlabelflt[0]
-            if delta >= 100:
-                dig = 0
-            elif ((delta < 100) & (delta >= 10)):
-                dig = 1
-            elif ((delta < 10) & (delta >= 1)):
-                dig = 2
-            elif delta < 1:
-                dig = 3
-            format_string = "%.{}f".format(dig)
-            tlabelstr = [format_string % number for number in tlabelflt]
-            cbar = fig.colorbar(cax, ticks=tickvals)
-            cbar.ax.set_yticklabels(tlabelstr)
-            ax.set_xlabel('Pixels')
-            ax.set_ylabel('Pixels')
-            plt.rcParams.update({'axes.titlesize': 'small'})
+            if not self.thumbnail:
+                tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
+                tlabelflt = tickvals + min_value - 1
+
+                # Adjust the number of digits after the decimal point
+                # in the colorbar labels based on the signal range
+                delta = tlabelflt[-1] - tlabelflt[0]
+                if delta >= 100:
+                    dig = 0
+                elif ((delta < 100) & (delta >= 10)):
+                    dig = 1
+                elif ((delta < 10) & (delta >= 1)):
+                    dig = 2
+                elif delta < 1:
+                    dig = 3
+                format_string = "%.{}f".format(dig)
+                tlabelstr = [format_string % number for number in tlabelflt]
+                cbar = fig.colorbar(cax, ticks=tickvals)
+                cbar.ax.set_yticklabels(tlabelstr)
+                ax.set_xlabel('Pixels')
+                ax.set_ylabel('Pixels')
+                plt.rcParams.update({'axes.titlesize': 'small'})
 
         elif scale == 'linear':
-            cax = ax.imshow(image, clim=(min_value, max_value), cmap='gray')
-            cbar = fig.colorbar(cax)
-            ax.set_xlabel('Pixels')
-            ax.set_ylabel('Pixels')
+            cax = ax.imshow(image, clim=(min_value, max_value), cmap=self.cmap)
+
+            if not self.thumbnail:
+                cbar = fig.colorbar(cax)
+                ax.set_xlabel('Pixels')
+                ax.set_ylabel('Pixels')
+
+        if not self.thumbnail:
+            filename = os.path.split(self.file)[-1]
+            ax.set_title(filename + ' Int: {}'.format(np.int(integration_number)))
 
-        filename = os.path.split(self.file)[-1]
-        ax.set_title(filename + ' Int: {}'.format(np.int(integration_number)))
         return fig
 
     def make_image(self):

From 0f4d761dabaa306f3024b7eb580a11ef9df14405 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Fri, 13 Apr 2018 16:11:21 -0400
Subject: [PATCH 10/23] Initial commit of module to generate preview images

---
 jwql/preview_image/generate_preview_images.py | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 jwql/preview_image/generate_preview_images.py

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
new file mode 100644
index 000000000..e49a0c5d9
--- /dev/null
+++ b/jwql/preview_image/generate_preview_images.py
@@ -0,0 +1,66 @@
+#! /usr/bin/env python
+
+"""Generate preview images for all files in the jwql filesystem.
+
+Execution of this script will generate preview images for each file in
+the jwql filesystem, if it does not already exist.
+
+Authors
+-------
+
+    Matthew Bourque
+
+
+Use
+---
+
+    This script is intended to be executed as such:
+
+    >>> python generate_preview_images.py
+
+Dependencies
+------------
+"""
+
+import glob
+import os
+
+from jwql.permissions import permissions
+from jwql.preview_image.preview_image import PreviewImage
+from jwql.utils.utils import get_config
+
+
+def generate_preview_images():
+    """The main function of the generate_preview_image module."""
+
+    filesystem = get_config()['filesystem']
+    preview_image_filesystem = get_config()['preview_image_filesystem']
+    filenames = glob.glob(os.path.join(filesystem, '*/*.fits'))
+
+    for filename in filenames:
+
+        # Determine the save location
+        rootname_elements = os.path.basename(filename).split('.fits')[0].split('_')
+        preview_image_dir = '{}_{}'.format(
+            rootname_elements[0],  # Program ID + observation + visit
+            rootname_elements[1])  # Visit group + parallel sequence ID + activity
+        output_directory = os.path.join(preview_image_filesystem, preview_image_dir)
+
+        # Create the output directory if necessary
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+            permissions.set_permissions(output_directory, verbose=False)
+
+        # Create and save the preview image
+        im = PreviewImage(filename, "SCI")
+        im.clip_percent = 0.01
+        im.scaling = 'log'
+        im.output_format = 'jpg'
+        im.output_directory = output_directory
+        im.make_image()
+        print('Created preview image for {}'.format(filename))
+
+
+if __name__ == '__main__':
+
+    generate_preview_images()

From 4489e02289bab0d150d0bf8a6db6b66bc7baac47 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 19 Apr 2018 12:05:37 -0400
Subject: [PATCH 11/23] Allowing for user to define the colormap used, default
 switched from "gray" to "viridis".  Also changed variable name "fname" to
 "outfile" to be more consistent within the module.

---
 jwql/preview_image/preview_image.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py
index 34f6521c7..e3f682a5c 100755
--- a/jwql/preview_image/preview_image.py
+++ b/jwql/preview_image/preview_image.py
@@ -36,11 +36,12 @@
 import sys
 
 from astropy.io import fits
+from jwst.datamodels import dqflags
 import matplotlib.pyplot as plt
 import matplotlib.colors as colors
 import numpy as np
 
-from jwst.datamodels import dqflags
+from jwql.permissions import permissions
 
 class PreviewImage():
 
@@ -57,6 +58,7 @@ def __init__(self, filename, extension):
         self.file = filename
         self.clip_percent = 0.01
         self.scaling = 'log'
+        self.cmap = 'viridis'
         self.output_format = 'jpg'
         self.output_directory = None
 
@@ -205,7 +207,7 @@ def make_figure(self, image, integration_number, min_value, max_value,
             # Log normalize the colormap
             cax = ax.imshow(shiftdata,
                             norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
-                            cmap='gray')
+                            cmap=self.cmap)
 
             # Add colorbar, with original data values
             tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
@@ -279,15 +281,18 @@ def make_image(self):
             plt.close()
 
     @staticmethod
-    def save_image(image, fname):
+    def save_image(image, outfile):
         """
-        Save an image in the requested output format
+        Save an image in the requested output format and sets the
+        appropriate permissions
 
         Parameters:
         ----------
         image : obj
             A matplotlib figure object
-        fname : str
+        outfile : str
             Output filename
         """
-        image.savefig(fname, bbox_inches='tight')
+        image.savefig(outfile, bbox_inches='tight')
+        permissions.set_permissions(outfile, verbose=False)
+        print('Saved image to {}'.format(outfile))

From dbd2b02cf12e05d7f10ddc739118483965b747ac Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 19 Apr 2018 12:06:36 -0400
Subject: [PATCH 12/23] Adding try/except around preview image generation so
 that script doesnt outright crash.

---
 jwql/preview_image/generate_preview_images.py | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index e49a0c5d9..20b947200 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -1,9 +1,9 @@
 #! /usr/bin/env python
 
-"""Generate preview images for all files in the jwql filesystem.
+"""Generate preview images for all files in the ``jwql`` filesystem.
 
 Execution of this script will generate preview images for each file in
-the jwql filesystem, if it does not already exist.
+the ``jwql`` filesystem, if it does not already exist.
 
 Authors
 -------
@@ -16,10 +16,15 @@
 
     This script is intended to be executed as such:
 
-    >>> python generate_preview_images.py
+    ::
 
-Dependencies
-------------
+        python generate_preview_images.py
+
+Notes
+-----
+
+    Some of this code could be simplified by using a filename parser
+    utility function, which is in the works for ``jwql``.
 """
 
 import glob
@@ -52,13 +57,16 @@ def generate_preview_images():
             permissions.set_permissions(output_directory, verbose=False)
 
         # Create and save the preview image
-        im = PreviewImage(filename, "SCI")
-        im.clip_percent = 0.01
-        im.scaling = 'log'
-        im.output_format = 'jpg'
-        im.output_directory = output_directory
-        im.make_image()
-        print('Created preview image for {}'.format(filename))
+        try:
+            im = PreviewImage(filename, "SCI")
+            im.clip_percent = 0.01
+            im.scaling = 'log'
+            im.cmap = 'viridis'
+            im.output_format = 'jpg'
+            im.output_directory = output_directory
+            im.make_image()
+        except ValueError as error:
+            print(error)
 
 
 if __name__ == '__main__':

From 93b6ee465c410d6ea383013152d8b98326524369 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Fri, 27 Apr 2018 11:35:55 -0400
Subject: [PATCH 13/23] Permissions should allow group execution in order to
 actually open the files.

---
 jwql/permissions/permissions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/permissions/permissions.py b/jwql/permissions/permissions.py
index 5c8b1aa45..b9889d026 100644
--- a/jwql/permissions/permissions.py
+++ b/jwql/permissions/permissions.py
@@ -77,7 +77,7 @@
 DEFAULT_GROUP = 'jwql_dev'
 
 # set the default mode for DEFAULT_OWNER
-DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP  # equivalent to '?rwxr-----'
+DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP # equivalent to '?rwxr-x---'
 
 def get_group_string(pathname):
     """Return the group of pathname in string representation.

From f5ba9d7d5230a03b86fec94b8a611740af114179 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Fri, 27 Apr 2018 11:37:12 -0400
Subject: [PATCH 14/23] Changes to enable generation of thumbnail images that
 have to axis labels, title, or colorbar.

---
 jwql/preview_image/generate_preview_images.py | 16 ++++
 jwql/preview_image/preview_image.py           | 81 +++++++++++--------
 2 files changed, 64 insertions(+), 33 deletions(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index 20b947200..ad0475d09 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -40,6 +40,7 @@ def generate_preview_images():
 
     filesystem = get_config()['filesystem']
     preview_image_filesystem = get_config()['preview_image_filesystem']
+    thumbnail_filesystem = get_config()['thumbnail_filesystem']
     filenames = glob.glob(os.path.join(filesystem, '*/*.fits'))
 
     for filename in filenames:
@@ -50,6 +51,7 @@ def generate_preview_images():
             rootname_elements[0],  # Program ID + observation + visit
             rootname_elements[1])  # Visit group + parallel sequence ID + activity
         output_directory = os.path.join(preview_image_filesystem, preview_image_dir)
+        thumbnail_output_directory = os.path.join(thumbnail_filesystem, preview_image_dir)
 
         # Create the output directory if necessary
         if not os.path.exists(output_directory):
@@ -68,6 +70,20 @@ def generate_preview_images():
         except ValueError as error:
             print(error)
 
+        # Create and save a thumbnail of the preview image
+        try:
+            im = PreviewImage(filename, "SCI")
+            im.clip_percent = 0.01
+            im.scaling = 'log'
+            im.cmap = 'viridis'
+            im.output_format = 'thumb'
+            im.output_directory = thumbnail_output_directory
+            im.thumbnail = True
+            print(im)
+            im.make_image()
+        except ValueError as error:
+            print(error)
+
 
 if __name__ == '__main__':
 
diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py
index e3f682a5c..a58960fa9 100755
--- a/jwql/preview_image/preview_image.py
+++ b/jwql/preview_image/preview_image.py
@@ -37,12 +37,17 @@
 
 from astropy.io import fits
 from jwst.datamodels import dqflags
-import matplotlib.pyplot as plt
-import matplotlib.colors as colors
 import numpy as np
 
 from jwql.permissions import permissions
 
+# Use the 'Agg' backend to avoid invoking $DISPLAY
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import matplotlib.colors as colors
+
+
 class PreviewImage():
 
     def __init__(self, filename, extension):
@@ -55,12 +60,13 @@ def __init__(self, filename, extension):
         extension : str
             Extension name to be read in
         """
-        self.file = filename
         self.clip_percent = 0.01
-        self.scaling = 'log'
         self.cmap = 'viridis'
+        self.file = filename
         self.output_format = 'jpg'
         self.output_directory = None
+        self.scaling = 'log'
+        self.thumbnail = False
 
         # Read in file
         self.data, self.dq = self.get_data(self.file, extension)
@@ -159,8 +165,7 @@ def get_data(self, filename, ext):
                                      .format(filename)))
         return data, dq
 
-    def make_figure(self, image, integration_number, min_value, max_value,
-                    scale, maxsize = 8):
+    def make_figure(self, image, integration_number, min_value, max_value, scale):
         """
         Create the matplotlib figure of the image
 
@@ -182,12 +187,17 @@ def make_figure(self, image, integration_number, min_value, max_value,
         result : obj
             Matplotlib Figure object
         """
+
         # Check the input scaling
         if scale not in ['linear','log']:
             raise ValueError(("WARNING: scaling option {} not supported."
                               .format(scale)))
 
         # Set the figure size
+        if self.thumbnail:
+            maxsize = 2
+        else:
+            maxsize = 8
         yd, xd = image.shape
         ratio = yd / xd
         if xd >= yd:
@@ -210,36 +220,41 @@ def make_figure(self, image, integration_number, min_value, max_value,
                             cmap=self.cmap)
 
             # Add colorbar, with original data values
-            tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
-            tlabelflt = tickvals + min_value - 1
-
-            # Adjust the number of digits after the decimal point
-            # in the colorbar labels based on the signal range
-            delta = tlabelflt[-1] - tlabelflt[0]
-            if delta >= 100:
-                dig = 0
-            elif ((delta < 100) & (delta >= 10)):
-                dig = 1
-            elif ((delta < 10) & (delta >= 1)):
-                dig = 2
-            elif delta < 1:
-                dig = 3
-            format_string = "%.{}f".format(dig)
-            tlabelstr = [format_string % number for number in tlabelflt]
-            cbar = fig.colorbar(cax, ticks=tickvals)
-            cbar.ax.set_yticklabels(tlabelstr)
-            ax.set_xlabel('Pixels')
-            ax.set_ylabel('Pixels')
-            plt.rcParams.update({'axes.titlesize': 'small'})
+            if not self.thumbnail:
+                tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
+                tlabelflt = tickvals + min_value - 1
+
+                # Adjust the number of digits after the decimal point
+                # in the colorbar labels based on the signal range
+                delta = tlabelflt[-1] - tlabelflt[0]
+                if delta >= 100:
+                    dig = 0
+                elif ((delta < 100) & (delta >= 10)):
+                    dig = 1
+                elif ((delta < 10) & (delta >= 1)):
+                    dig = 2
+                elif delta < 1:
+                    dig = 3
+                format_string = "%.{}f".format(dig)
+                tlabelstr = [format_string % number for number in tlabelflt]
+                cbar = fig.colorbar(cax, ticks=tickvals)
+                cbar.ax.set_yticklabels(tlabelstr)
+                ax.set_xlabel('Pixels')
+                ax.set_ylabel('Pixels')
+                plt.rcParams.update({'axes.titlesize': 'small'})
 
         elif scale == 'linear':
-            cax = ax.imshow(image, clim=(min_value, max_value), cmap='gray')
-            cbar = fig.colorbar(cax)
-            ax.set_xlabel('Pixels')
-            ax.set_ylabel('Pixels')
+            cax = ax.imshow(image, clim=(min_value, max_value), cmap=self.cmap)
+
+            if not self.thumbnail:
+                cbar = fig.colorbar(cax)
+                ax.set_xlabel('Pixels')
+                ax.set_ylabel('Pixels')
+
+        if not self.thumbnail:
+            filename = os.path.split(self.file)[-1]
+            ax.set_title(filename + ' Int: {}'.format(np.int(integration_number)))
 
-        filename = os.path.split(self.file)[-1]
-        ax.set_title(filename + ' Int: {}'.format(np.int(integration_number)))
         return fig
 
     def make_image(self):

From 9ff43e28db6247be36e480356e00e1553b8d1f59 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Tue, 1 May 2018 16:13:10 -0400
Subject: [PATCH 15/23] Fixed bug wherein I forgot to create the thumbnail
 directory when necessary.  Also now using utils.filename_parser.

---
 jwql/preview_image/generate_preview_images.py | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index ad0475d09..747cdee89 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -33,6 +33,7 @@
 from jwql.permissions import permissions
 from jwql.preview_image.preview_image import PreviewImage
 from jwql.utils.utils import get_config
+from jwql.utils.utils import filename_parser
 
 
 def generate_preview_images():
@@ -46,17 +47,17 @@ def generate_preview_images():
     for filename in filenames:
 
         # Determine the save location
-        rootname_elements = os.path.basename(filename).split('.fits')[0].split('_')
-        preview_image_dir = '{}_{}'.format(
-            rootname_elements[0],  # Program ID + observation + visit
-            rootname_elements[1])  # Visit group + parallel sequence ID + activity
-        output_directory = os.path.join(preview_image_filesystem, preview_image_dir)
-        thumbnail_output_directory = os.path.join(thumbnail_filesystem, preview_image_dir)
-
-        # Create the output directory if necessary
+        identifier = 'jw{}'.format(filename_parser(filename)['program_id'])
+        output_directory = os.path.join(preview_image_filesystem, identifier)
+        thumbnail_output_directory = os.path.join(thumbnail_filesystem, identifier)
+
+        # Create the output directories if necessary
         if not os.path.exists(output_directory):
             os.makedirs(output_directory)
             permissions.set_permissions(output_directory, verbose=False)
+        if not os.path.exists(thumbnail_output_directory):
+            os.makedirs(thumbnail_output_directory)
+            permissions.set_permissions(thumbnail_output_directory, verbose=False)
 
         # Create and save the preview image
         try:
@@ -76,10 +77,8 @@ def generate_preview_images():
             im.clip_percent = 0.01
             im.scaling = 'log'
             im.cmap = 'viridis'
-            im.output_format = 'thumb'
             im.output_directory = thumbnail_output_directory
             im.thumbnail = True
-            print(im)
             im.make_image()
         except ValueError as error:
             print(error)

From d6d0bffab479e95eca73fa982dc2608a8823b01e Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Tue, 1 May 2018 16:13:42 -0400
Subject: [PATCH 16/23] Turned off axes for thumbnails.

---
 jwql/preview_image/preview_image.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py
index a58960fa9..8e7d677be 100755
--- a/jwql/preview_image/preview_image.py
+++ b/jwql/preview_image/preview_image.py
@@ -251,10 +251,15 @@ def make_figure(self, image, integration_number, min_value, max_value, scale):
                 ax.set_xlabel('Pixels')
                 ax.set_ylabel('Pixels')
 
+        # If preview image, set a title
         if not self.thumbnail:
             filename = os.path.split(self.file)[-1]
             ax.set_title(filename + ' Int: {}'.format(np.int(integration_number)))
 
+        # If thumbnail, turn off the axes
+        if self.thumbnail:
+            plt.axis('off')
+
         return fig
 
     def make_image(self):
@@ -295,8 +300,7 @@ def make_image(self):
             self.save_image(fig, outfile)
             plt.close()
 
-    @staticmethod
-    def save_image(image, outfile):
+    def save_image(self, image, outfile):
         """
         Save an image in the requested output format and sets the
         appropriate permissions
@@ -308,6 +312,14 @@ def save_image(image, outfile):
         outfile : str
             Output filename
         """
+
         image.savefig(outfile, bbox_inches='tight')
         permissions.set_permissions(outfile, verbose=False)
-        print('Saved image to {}'.format(outfile))
+
+        # If the image is a thumbnail, rename to '.thumb'
+        if self.thumbnail:
+            new_outfile = outfile.replace('.jpg', '.thumb')
+            os.rename(outfile, new_outfile)
+            print('Saved image to {}'.format(new_outfile))
+        else:
+            print('Saved image to {}'.format(outfile))

From 48a00348daf4b2491ca356f5e30beb8212949db1 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 3 May 2018 11:55:32 -0400
Subject: [PATCH 17/23] Added code to ensure there is no whitespace around
 thumbnail images.  Also removed image parameter from save_image as it is not
 needed.

---
 jwql/preview_image/preview_image.py | 46 ++++++++++++++---------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py
index 8e7d677be..34fd6bd0d 100755
--- a/jwql/preview_image/preview_image.py
+++ b/jwql/preview_image/preview_image.py
@@ -194,10 +194,6 @@ def make_figure(self, image, integration_number, min_value, max_value, scale):
                               .format(scale)))
 
         # Set the figure size
-        if self.thumbnail:
-            maxsize = 2
-        else:
-            maxsize = 8
         yd, xd = image.shape
         ratio = yd / xd
         if xd >= yd:
@@ -206,21 +202,31 @@ def make_figure(self, image, integration_number, min_value, max_value, scale):
         else:
             ysize = maxsize
             xsize = maxsize / ratio
-        fig, ax = plt.subplots(figsize=(xsize, ysize))
 
         if scale == 'log':
+
             # Shift data so everything is positive
             shiftdata = image - min_value + 1
             shiftmin = 1
             shiftmax = max_value - min_value + 1
 
-            # Log normalize the colormap
-            cax = ax.imshow(shiftdata,
-                            norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
-                            cmap=self.cmap)
+            # If making a thumbnail, make a figure with no axes
+            if self.thumbnail:
+                fig = plt.imshow(shiftdata,
+                           norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
+                           cmap=self.cmap)
+                plt.axis('off')
+                fig.axes.get_xaxis().set_visible(False)
+                fig.axes.get_yaxis().set_visible(False)
 
-            # Add colorbar, with original data values
-            if not self.thumbnail:
+            # If preview image, add axes and colorbars
+            else:
+                fig, ax = plt.subplots(figsize=(xsize, ysize))
+                cax = ax.imshow(shiftdata,
+                                norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
+                                cmap=self.cmap)
+
+                # Add colorbar, with original data values
                 tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
                 tlabelflt = tickvals + min_value - 1
 
@@ -244,6 +250,7 @@ def make_figure(self, image, integration_number, min_value, max_value, scale):
                 plt.rcParams.update({'axes.titlesize': 'small'})
 
         elif scale == 'linear':
+            fig, ax = plt.subplots(figsize=(xsize, ysize))
             cax = ax.imshow(image, clim=(min_value, max_value), cmap=self.cmap)
 
             if not self.thumbnail:
@@ -256,12 +263,6 @@ def make_figure(self, image, integration_number, min_value, max_value, scale):
             filename = os.path.split(self.file)[-1]
             ax.set_title(filename + ' Int: {}'.format(np.int(integration_number)))
 
-        # If thumbnail, turn off the axes
-        if self.thumbnail:
-            plt.axis('off')
-
-        return fig
-
     def make_image(self):
         """
         MAIN FUNCTION
@@ -296,24 +297,23 @@ def make_image(self):
             else:
                 outdir = self.output_directory
             outfile = os.path.join(outdir, infile.split('.')[0] + suffix)
-            fig = self.make_figure(frame, i, minval, maxval, self.scaling.lower())
-            self.save_image(fig, outfile)
+            self.make_figure(frame, i, minval, maxval, self.scaling.lower())
+            self.save_image(outfile)
             plt.close()
 
-    def save_image(self, image, outfile):
+
+    def save_image(self, outfile):
         """
         Save an image in the requested output format and sets the
         appropriate permissions
 
         Parameters:
         ----------
-        image : obj
-            A matplotlib figure object
         outfile : str
             Output filename
         """
 
-        image.savefig(outfile, bbox_inches='tight')
+        plt.savefig(outfile, bbox_inches='tight', pad_inches=0)
         permissions.set_permissions(outfile, verbose=False)
 
         # If the image is a thumbnail, rename to '.thumb'

From a5647b11de9b9cf02d14b5c846dc07038e5137b0 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 3 May 2018 11:56:13 -0400
Subject: [PATCH 18/23] Changed activity to allow characters instead of just
 digits, as some activities appear to be named with letters.

---
 jwql/utils/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py
index 024b2f7e1..4c918a914 100644
--- a/jwql/utils/utils.py
+++ b/jwql/utils/utils.py
@@ -72,7 +72,7 @@ def filename_parser(filename):
                    "(?P<visit>\d{3})"
                    "_(?P<visit_group>\d{2})"
                    "(?P<parallel_seq_id>\d{1})"
-                   "(?P<activity>\d{2})"
+                   "(?P<activity>\w{2})"
                    "_(?P<exposure_id>\d+)"
                    "_(?P<detector>\w+)"
                    "_(?P<suffix>\w+).*")

From 014af6fbee6caac0195f58febababafad1ce22f1 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Thu, 3 May 2018 11:58:56 -0400
Subject: [PATCH 19/23] Adding back maxsize parameter. Also updated docstring.

---
 jwql/preview_image/preview_image.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/jwql/preview_image/preview_image.py b/jwql/preview_image/preview_image.py
index 34fd6bd0d..b42847516 100755
--- a/jwql/preview_image/preview_image.py
+++ b/jwql/preview_image/preview_image.py
@@ -165,10 +165,10 @@ def get_data(self, filename, ext):
                                      .format(filename)))
         return data, dq
 
-    def make_figure(self, image, integration_number, min_value, max_value, scale):
+    def make_figure(self, image, integration_number, min_value, max_value,
+                    scale, maxsize=8):
         """
         Create the matplotlib figure of the image
-
         Parameters:
         ----------
         image : obj
@@ -181,11 +181,6 @@ def make_figure(self, image, integration_number, min_value, max_value, scale):
             Maximum value for display
         scale : str
             Image scaling ('log', 'linear')
-
-        Returns:
-        --------
-        result : obj
-            Matplotlib Figure object
         """
 
         # Check the input scaling

From 37eeb6515e9d66160216f106d5714b1d712633c8 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Fri, 4 May 2018 10:19:45 -0400
Subject: [PATCH 20/23] Using the basename of the filename in cases where the
 JWST filename does not follow convention.  Also added more documentation.

---
 jwql/preview_image/generate_preview_images.py | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index 747cdee89..0f2129e46 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -2,8 +2,12 @@
 
 """Generate preview images for all files in the ``jwql`` filesystem.
 
-Execution of this script will generate preview images for each file in
-the ``jwql`` filesystem, if it does not already exist.
+Execution of this script will generate preview images and thumbnail
+images for each file in the ``jwql`` filesystem.  Preview images have
+axes labels, titles, and colorbars, wheras thumbnail images are
+smaller and contain no labels.  Images are saved into the
+``preview_image_filesystem`` and ``thumbnail_filesystem``, organized
+by subdirectories pertaining to the ``program_id`` in the filenames.
 
 Authors
 -------
@@ -19,12 +23,6 @@
     ::
 
         python generate_preview_images.py
-
-Notes
------
-
-    Some of this code could be simplified by using a filename parser
-    utility function, which is in the works for ``jwql``.
 """
 
 import glob
@@ -47,7 +45,11 @@ def generate_preview_images():
     for filename in filenames:
 
         # Determine the save location
-        identifier = 'jw{}'.format(filename_parser(filename)['program_id'])
+        try:
+            identifier = 'jw{}'.format(filename_parser(filename)['program_id'])
+        except ValueError as error:
+            identifier = os.path.basename(filename).split('.fits')[0]
+
         output_directory = os.path.join(preview_image_filesystem, identifier)
         thumbnail_output_directory = os.path.join(thumbnail_filesystem, identifier)
 

From d1cd01b4e5a7e8bd916d7ef6640ff529fb7ea914 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Tue, 15 May 2018 10:15:31 -0400
Subject: [PATCH 21/23] Reducing repeated code by looping over argument list.

---
 jwql/preview_image/generate_preview_images.py | 38 +++++++------------
 1 file changed, 14 insertions(+), 24 deletions(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index 0f2129e46..08cb7f9ce 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -61,30 +61,20 @@ def generate_preview_images():
             os.makedirs(thumbnail_output_directory)
             permissions.set_permissions(thumbnail_output_directory, verbose=False)
 
-        # Create and save the preview image
-        try:
-            im = PreviewImage(filename, "SCI")
-            im.clip_percent = 0.01
-            im.scaling = 'log'
-            im.cmap = 'viridis'
-            im.output_format = 'jpg'
-            im.output_directory = output_directory
-            im.make_image()
-        except ValueError as error:
-            print(error)
-
-        # Create and save a thumbnail of the preview image
-        try:
-            im = PreviewImage(filename, "SCI")
-            im.clip_percent = 0.01
-            im.scaling = 'log'
-            im.cmap = 'viridis'
-            im.output_directory = thumbnail_output_directory
-            im.thumbnail = True
-            im.make_image()
-        except ValueError as error:
-            print(error)
-
+        # Create and save the preview image and thumbnail
+        args = zip((False, output_directory), (True, thumbnail_output_directory))
+        for thumbnail_bool, directory in args:
+            try:
+                im = PreviewImage(filename, "SCI")
+                im.clip_percent = 0.01
+                im.scaling = 'log'
+                im.cmap = 'viridis'
+                im.output_format = 'jpg'
+                im.thumbnail = thumbnail_bool
+                im.output_directory = directory
+                im.make_image()
+            except ValueError as error:
+                print(error)
 
 if __name__ == '__main__':
 

From 4ba5652a0f4d67fe54118c77f5cb886d72d37e76 Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Tue, 15 May 2018 10:21:26 -0400
Subject: [PATCH 22/23] Added API docs for generate_preview_images

---
 docs/source/preview_image.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/source/preview_image.rst b/docs/source/preview_image.rst
index 8dae4ccb3..c0d3f9119 100644
--- a/docs/source/preview_image.rst
+++ b/docs/source/preview_image.rst
@@ -5,3 +5,11 @@ preview_image
 .. automodule:: jwql.preview_image.preview_image
     :members:
     :undoc-members:
+
+***********************
+generate_preview_images
+***********************
+
+.. automodule:: jwql.preview_image.generate_preview_images
+    :members:
+    :undoc-members:

From 64d9bd75040bf98877504e348a39047f9bb427ad Mon Sep 17 00:00:00 2001
From: Matthew Bourque <bourque@stsci.edu>
Date: Tue, 15 May 2018 10:30:09 -0400
Subject: [PATCH 23/23] Fixed bug where the order of the zipped lists were
 wrong.

---
 jwql/preview_image/generate_preview_images.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jwql/preview_image/generate_preview_images.py b/jwql/preview_image/generate_preview_images.py
index 08cb7f9ce..43cc60053 100644
--- a/jwql/preview_image/generate_preview_images.py
+++ b/jwql/preview_image/generate_preview_images.py
@@ -62,7 +62,7 @@ def generate_preview_images():
             permissions.set_permissions(thumbnail_output_directory, verbose=False)
 
         # Create and save the preview image and thumbnail
-        args = zip((False, output_directory), (True, thumbnail_output_directory))
+        args = zip((False, True), (output_directory, thumbnail_output_directory))
         for thumbnail_bool, directory in args:
             try:
                 im = PreviewImage(filename, "SCI")