Merge pull request #65 from bourque/generate-preview-images

Generate preview images

docs/source/preview_image.rst

@@ -5,3 +5,11 @@ preview_image
 .. automodule:: jwql.preview_image.preview_image
     :members:
     :undoc-members:
+
+***********************
+generate_preview_images
+***********************
+
+.. automodule:: jwql.preview_image.generate_preview_images
+    :members:
+    :undoc-members:

jwql/permissions/permissions.py

@@ -82,7 +82,7 @@
 DEFAULT_GROUP = 'jwql_dev'
 
 # set the default mode for DEFAULT_OWNER
-DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP  # equivalent to '?rwxr-----'
+DEFAULT_MODE = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP # equivalent to '?rwxr-x---'
 
 def get_group_string(pathname):
     """Return the group of ``pathname`` in string representation.

jwql/preview_image/generate_preview_images.py

@@ -0,0 +1,81 @@
+#! /usr/bin/env python
+
+"""Generate preview images for all files in the ``jwql`` filesystem.
+
+Execution of this script will generate preview images and thumbnail
+images for each file in the ``jwql`` filesystem.  Preview images have
+axes labels, titles, and colorbars, wheras thumbnail images are
+smaller and contain no labels.  Images are saved into the
+``preview_image_filesystem`` and ``thumbnail_filesystem``, organized
+by subdirectories pertaining to the ``program_id`` in the filenames.
+
+Authors
+-------
+
+    Matthew Bourque
+
+
+Use
+---
+
+    This script is intended to be executed as such:
+
+    ::
+
+        python generate_preview_images.py
+"""
+
+import glob
+import os
+
+from jwql.permissions import permissions
+from jwql.preview_image.preview_image import PreviewImage
+from jwql.utils.utils import get_config
+from jwql.utils.utils import filename_parser
+
+
+def generate_preview_images():
+    """The main function of the generate_preview_image module."""
+
+    filesystem = get_config()['filesystem']
+    preview_image_filesystem = get_config()['preview_image_filesystem']
+    thumbnail_filesystem = get_config()['thumbnail_filesystem']
+    filenames = glob.glob(os.path.join(filesystem, '*/*.fits'))
+
+    for filename in filenames:
+
+        # Determine the save location
+        try:
+            identifier = 'jw{}'.format(filename_parser(filename)['program_id'])
+        except ValueError as error:
+            identifier = os.path.basename(filename).split('.fits')[0]
+
+        output_directory = os.path.join(preview_image_filesystem, identifier)
+        thumbnail_output_directory = os.path.join(thumbnail_filesystem, identifier)
+
+        # Create the output directories if necessary
+        if not os.path.exists(output_directory):
+            os.makedirs(output_directory)
+            permissions.set_permissions(output_directory, verbose=False)
+        if not os.path.exists(thumbnail_output_directory):
+            os.makedirs(thumbnail_output_directory)
+            permissions.set_permissions(thumbnail_output_directory, verbose=False)
+
+        # Create and save the preview image and thumbnail
+        args = zip((False, True), (output_directory, thumbnail_output_directory))
+        for thumbnail_bool, directory in args:
+            try:
+                im = PreviewImage(filename, "SCI")
+                im.clip_percent = 0.01
+                im.scaling = 'log'
+                im.cmap = 'viridis'
+                im.output_format = 'jpg'
+                im.thumbnail = thumbnail_bool
+                im.output_directory = directory
+                im.make_image()
+            except ValueError as error:
+                print(error)
+
+if __name__ == '__main__':
+
+    generate_preview_images()

jwql/preview_image/preview_image.py

@@ -38,11 +38,17 @@
 import sys
 
 from astropy.io import fits
+from jwst.datamodels import dqflags
+import numpy as np
+
+from jwql.permissions import permissions
+
+# Use the 'Agg' backend to avoid invoking $DISPLAY
+import matplotlib
+matplotlib.use('Agg')
 import matplotlib.pyplot as plt
 import matplotlib.colors as colors
-import numpy as np
 
-from jwst.datamodels import dqflags
 
 
 class PreviewImage():
@@ -57,11 +63,13 @@ def __init__(self, filename, extension):
         extension : str
             Extension name to be read in
         """
-        self.file = filename
         self.clip_percent = 0.01
-        self.scaling = 'log'
+        self.cmap = 'viridis'
+        self.file = filename
         self.output_format = 'jpg'
         self.output_directory = None
+        self.scaling = 'log'
+        self.thumbnail = False
 
         # Read in file
         self.data, self.dq = self.get_data(self.file, extension)
@@ -160,7 +168,7 @@ def get_data(self, filename, ext):
         return data, dq
 
     def make_figure(self, image, integration_number, min_value, max_value,
-                    scale, maxsize = 8):
+                    scale, maxsize=8):
         """
         Create the matplotlib figure of the image
 
@@ -182,6 +190,7 @@ def make_figure(self, image, integration_number, min_value, max_value,
         result : obj
             Matplotlib Figure object
         """
+
         # Check the input scaling
         if scale not in ['linear','log']:
             raise ValueError(("WARNING: scaling option {} not supported."
@@ -196,51 +205,66 @@ def make_figure(self, image, integration_number, min_value, max_value,
         else:
             ysize = maxsize
             xsize = maxsize / ratio
-        fig, ax = plt.subplots(figsize=(xsize, ysize))
 
         if scale == 'log':
+
             # Shift data so everything is positive
             shiftdata = image - min_value + 1
             shiftmin = 1
             shiftmax = max_value - min_value + 1
 
-            # Log normalize the colormap
-            cax = ax.imshow(shiftdata,
-                            norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
-                            cmap='gray')
-
-            # Add colorbar, with original data values
-            tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
-            tlabelflt = tickvals + min_value - 1
-
-            # Adjust the number of digits after the decimal point
-            # in the colorbar labels based on the signal range
-            delta = tlabelflt[-1] - tlabelflt[0]
-            if delta >= 100:
-                dig = 0
-            elif ((delta < 100) & (delta >= 10)):
-                dig = 1
-            elif ((delta < 10) & (delta >= 1)):
-                dig = 2
-            elif delta < 1:
-                dig = 3
-            format_string = "%.{}f".format(dig)
-            tlabelstr = [format_string % number for number in tlabelflt]
-            cbar = fig.colorbar(cax, ticks=tickvals)
-            cbar.ax.set_yticklabels(tlabelstr)
-            ax.set_xlabel('Pixels')
-            ax.set_ylabel('Pixels')
-            plt.rcParams.update({'axes.titlesize': 'small'})
+            # If making a thumbnail, make a figure with no axes
+            if self.thumbnail:
+                fig = plt.imshow(shiftdata,
+                           norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
+                           cmap=self.cmap)
+                plt.axis('off')
+                fig.axes.get_xaxis().set_visible(False)
+                fig.axes.get_yaxis().set_visible(False)
+
+            # If preview image, add axes and colorbars
+            else:
+                fig, ax = plt.subplots(figsize=(xsize, ysize))
+                cax = ax.imshow(shiftdata,
+                                norm=colors.LogNorm(vmin=shiftmin, vmax=shiftmax),
+                                cmap=self.cmap)
+
+                # Add colorbar, with original data values
+                tickvals = np.logspace(np.log10(shiftmin), np.log10(shiftmax), 5)
+                tlabelflt = tickvals + min_value - 1
+
+                # Adjust the number of digits after the decimal point
+                # in the colorbar labels based on the signal range
+                delta = tlabelflt[-1] - tlabelflt[0]
+                if delta >= 100:
+                    dig = 0
+                elif ((delta < 100) & (delta >= 10)):
+                    dig = 1
+                elif ((delta < 10) & (delta >= 1)):
+                    dig = 2
+                elif delta < 1:
+                    dig = 3
+                format_string = "%.{}f".format(dig)
+                tlabelstr = [format_string % number for number in tlabelflt]
+                cbar = fig.colorbar(cax, ticks=tickvals)
+                cbar.ax.set_yticklabels(tlabelstr)
+                ax.set_xlabel('Pixels')
+                ax.set_ylabel('Pixels')
+                plt.rcParams.update({'axes.titlesize': 'small'})
 
         elif scale == 'linear':
-            cax = ax.imshow(image, clim=(min_value, max_value), cmap='gray')
-            cbar = fig.colorbar(cax)
-            ax.set_xlabel('Pixels')
-            ax.set_ylabel('Pixels')
+            fig, ax = plt.subplots(figsize=(xsize, ysize))
+            cax = ax.imshow(image, clim=(min_value, max_value), cmap=self.cmap)
+
+            if not self.thumbnail:
+                cbar = fig.colorbar(cax)
+                ax.set_xlabel('Pixels')
+                ax.set_ylabel('Pixels')
 
-        filename = os.path.split(self.file)[-1]
-        ax.set_title(filename + ' Int: {}'.format(np.int(integration_number)))
-        return fig
+        # If preview image, set a title
+        if not self.thumbnail:
+            filename = os.path.split(self.file)[-1]
+            ax.set_title(filename + ' Int: {}'.format(np.int(integration_number)))
 
     def make_image(self):
         """
@@ -276,14 +300,15 @@ def make_image(self):
             else:
                 outdir = self.output_directory
             outfile = os.path.join(outdir, infile.split('.')[0] + suffix)
-            fig = self.make_figure(frame, i, minval, maxval, self.scaling.lower())
-            self.save_image(fig, outfile)
+            self.make_figure(frame, i, minval, maxval, self.scaling.lower())
+            self.save_image(outfile)
             plt.close()
 
-    @staticmethod
-    def save_image(image, fname):
+
+    def save_image(self, outfile):
         """
-        Save an image in the requested output format
+        Save an image in the requested output format and sets the
+        appropriate permissions
 
         Parameters
         ----------
@@ -292,4 +317,14 @@ def save_image(image, fname):
         fname : str
             Output filename
         """
-        image.savefig(fname, bbox_inches='tight')
+
+        plt.savefig(outfile, bbox_inches='tight', pad_inches=0)
+        permissions.set_permissions(outfile, verbose=False)
+
+        # If the image is a thumbnail, rename to '.thumb'
+        if self.thumbnail:
+            new_outfile = outfile.replace('.jpg', '.thumb')
+            os.rename(outfile, new_outfile)
+            print('Saved image to {}'.format(new_outfile))
+        else:
+            print('Saved image to {}'.format(outfile))

jwql/utils/utils.py

@@ -72,7 +72,7 @@ def filename_parser(filename):
                    "(?P<visit>\d{3})"
                    "_(?P<visit_group>\d{2})"
                    "(?P<parallel_seq_id>\d{1})"
-                   "(?P<activity>\d{2})"
+                   "(?P<activity>\w{2})"
                    "_(?P<exposure_id>\d+)"
                    "_(?P<detector>\w+)"
                    "_(?P<suffix>\w+).*")