Merge branch 'develop' into widen-margins-in-exp-pages

.github/workflows/build.yml

@@ -14,9 +14,11 @@ jobs:
   build:
     uses: OpenAstronomy/github-actions-workflows/.github/workflows/publish_pure_python.yml@v1
     with:
-      upload_to_pypi: false
+      upload_to_pypi: ${{ (github.event_name == 'release') && (github.event.action == 'released') }}
+    secrets:
+      pypi_token: ${{ secrets.PYPI_PASSWORD_STSCI_MAINTAINER }}
   freeze:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: 
@@ -45,7 +47,7 @@ jobs:
         with:
           args: --strict --pattern "default-unprefixed" --style semver
       - id: filename
-        run: echo "filename=jwql_conda_py${{ matrix.python-version }}_${{ steps.version.outputs.version }}_${{ matrix.os }}.yml" >> $GITHUB_OUTPUT
+        run: echo "filename=jwql_${{ steps.version.outputs.version }}_conda_${{ runner.os }}_${{ runner.arch }}_py${{ matrix.python-version }}.yml" >> $GITHUB_OUTPUT
       - run: conda env export --no-build | grep -v "name:" | grep -v "prefix:" > ${{ steps.filename.outputs.filename }}
       - run: cat ${{ steps.filename.outputs.filename }}
       - uses: actions/upload-artifact@v4

.gitignore

@@ -19,3 +19,4 @@ jwql/website/apps/jwql/static/filesystem
 jwql/website/apps/jwql/static/outputs
 jwql/website/apps/jwql/static/preview_images
 jwql/website/apps/jwql/static/thumbnails
+jwql-current.yml

CHANGES.rst

@@ -1,5 +1,46 @@
 ## What's Changed
 
+1.2.10 (2024-07-10)
+===================
+
+Duplicate of 1.2.9, caused by changes to versioning conflicts with pypi. 
+
+
+1.2.9 (2024-07-10)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Add Download CSV button to query page by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1561
+- show file anomalies on exposure group page by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1564
+- create generic error page to handle exceptions in views. by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1549
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- final model define for faking by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1544
+- Update Redis Package Names in Environment Files by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1546
+- [SCSB-145] require Python 3.10 by @zacharyburnett in https://github.com/spacetelescope/jwql/pull/1515
+- debug false by default by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1550
+- Update NIRSpec TA Monitors to use Django DB Models by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1499
+- Update NIRSpec TA Models by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1565
+- Remove codecov.yml by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1588
+- Remove filename parser test over filesystem by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1586
+- Update remote to upstream in pull_jwql_branch.sh by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1591
+- Add Dependencies for Servers in `pyproject.toml` by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1568
+- fix release upload step condition to match workflow trigger by @zacharyburnett in https://github.com/spacetelescope/jwql/pull/1593
+- fix environment freeze workflow not picking up tag by @zacharyburnett in https://github.com/spacetelescope/jwql/pull/1594
+- fix version matching pattern by @zacharyburnett in https://github.com/spacetelescope/jwql/pull/1595
+- updating freeze matrix to include linux, mac and python 3.12 by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1596
+- Remove P750L from list of NIRSpec filters by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1598
+- [build] fix `runs-on:` and update build filename for easier parsing by @zacharyburnett in https://github.com/spacetelescope/jwql/pull/1599
+- upload to PyPI on release by @zacharyburnett in https://github.com/spacetelescope/jwql/pull/1601
+- Updating jwst_reffiles version number by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1606
+- Remove old presentations from repo by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1607
+- Num results fix by @BradleySappington in https://github.com/spacetelescope/jwql/pull/1608
+- Add Environment Update Script by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1609
+- Add new NIRISS AMI-related suffixes by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1613
+
+
 1.2.8 (2024-04-18)
 ==================
 

jwql/install-env.sh

@@ -0,0 +1,70 @@
+branchname=$1
+python_version=$2
+
+printf "UPDATING JWQL ENVIRONMENT\n\n"
+
+# Check operating system to obtain proper substring for environment
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    os_str="macOS_ARM64"
+    printf "INFORMATION: \n \t MAC OS DETECTED, USING MAC ENVIRONMENT FILE\n"
+elif [[ "$OSTYPE" == "linux-gnu"* ]]; then
+    os_str="Linux_X64"
+    printf "INFORMATION: \n \t LINUX OS DETECTED, USING LINUX ENVIRONMENT FILE\n"
+else
+    printf "EXCEPTION: \n \t $OSTYPE NOT SUPPORTED, EXITING"
+    return
+fi
+
+# Check if branch name starts with "v" for our major releases
+# Our branch names contain v prior to version number, but version names on git
+# do not contain v prior to the number.
+if [[ $branchname == v* ]]; then
+    jwql_version=${branchname:1:${#branchname}}
+    environment_url=https://github.com/spacetelescope/jwql/releases/download/$jwql_version/
+    environment_name=jwql_${jwql_version}_conda_${os_str}_py${python_version}
+    environment_filename="${environment_name}.yml"
+else
+    printf "EXCEPTION: \n \t RELEASE DOESNT FOLLOW RELEASE VERSIONING NAMING CONVENTION, EXITING"
+    return
+fi
+
+# Download asset from release and install it.
+if curl --head --silent --fail "${environment_url}${environment_filename}" 2> /dev/null;
+    then
+        # Reset back to base first before generating environment (incase one is currently activated)
+        eval "$(conda shell.bash deactivate)"
+        eval "$(conda shell.bash activate base)"
+        printf "\n SUCESSFULLY LOCATED ENVIRONMENT FILE ${environment_url}${environment_filename} \n"
+        curl -L "${environment_url}/${environment_filename}" > jwql-current.yml
+        $CONDA_EXE env create --name $environment_name --file jwql-current.yml
+    else
+        printf "EXCEPTION:\n"
+        printf "\t ${environment_url}${environment_filename} DOES NOT EXIST, EXITING\n"
+        printf "\t \nENSURE THAT: \n"
+        printf "\t https://github.com/spacetelescope/jwql/releases/tag/$branchname \n"
+        printf "EXISTS AND VERIFY ASSET FOR ${jwql_version}, ${python_version} FOR OS ${os_str}"
+        return
+fi
+
+# Update symlink
+cd ${CONDA_PREFIX}/envs/
+
+env_symlink="jwql-current"
+
+if [[ -L $env_symlink || -e $env_symlink ]]; then
+   printf "INFORMATION:\n"
+   printf "\tjwql-current SYMLINK EXISTS, UNLINKING\n"
+   unlink jwql-current
+fi
+
+printf "INFORMATION:\n\tLINKING NEW ENVIRONMENT\n"
+ln -s $environment_name jwql-current
+
+printf "\tjwql-current HAS BEEN SET TO: ${environment_name}\n"
+printf "\tTO SEE CHANGES, EXIT/RESTART SHELL\n"
+
+# return to original directory
+cd -
+
+# Conda commands change shell prompt, this just returns it to the default
+export PS1="\n(base)\h:\W \u\$ "

jwql/jwql_monitors/generate_preview_images.py

@@ -438,6 +438,7 @@ def define_options(parser=None, usage=None, conflict_handler='resolve'):
         parser = argparse.ArgumentParser(usage=usage, conflict_handler=conflict_handler)
 
     parser.add_argument('--overwrite', action='store_true', default=None, help='If set, existing preview images will be re-created and overwritten.')
+    parser.add_argument('-p', '--programs', nargs='+', type=int, help='List of program IDs to generate preview images for. If omitted, all programs will be done.')
     return parser
 
 
@@ -540,20 +541,46 @@ def get_base_output_name(filename_dict):
 
 @log_fail
 @log_info
-def generate_preview_images(overwrite):
+def generate_preview_images(overwrite, programs=None):
     """The main function of the ``generate_preview_image`` module.
     See module docstring for further details.
 
     Parameters
     ----------
     overwrite : bool
         If True, any existing preview images and thumbnails are overwritten
+
+    programs : list
+        List of program ID numbers (e.g. 1068, 01220) for which to generate preview
+        images. If None (the default), preview images are generated for all programs.
     """
+    # Get a list of programs to create preview images for. First, generate a list of all
+    # possible programs. We can compare any user inputs to this list, and if there are no
+    # user inputs, then use this entire list.
+    all_programs = [os.path.basename(item) for item in glob.glob(os.path.join(SETTINGS['filesystem'], 'public', 'jw*'))]
+    all_programs.extend([os.path.basename(item) for item in glob.glob(os.path.join(SETTINGS['filesystem'], 'proprietary', 'jw*'))])
+
+    if programs is None:
+        program_list = all_programs
+    else:
+        if not isinstance(programs, list):
+            raise ValueError(f'program_list must be a list. In this call, it is {type(program_list)}')
+        program_list = []
+        for prog in programs:
+            jwprog = f'jw{str(prog).zfill(5)}'
+            if jwprog in all_programs:
+                program_list.append(jwprog)
+            else:
+                logging.info(f'Program {prog} not present in filesystem. Excluding.')
+
+    if len(program_list) > 0:
+        program_list = sorted(program_list, reverse=True)
+    else:
+        no_prog_message = f'Empty list of programs. No preview images to be made.'
+        logging.info(no_prog_message)
+        raise ValueError(no_prog_message)
 
     # Process programs in parallel
-    program_list = [os.path.basename(item) for item in glob.glob(os.path.join(SETTINGS['filesystem'], 'public', 'jw*'))]
-    program_list.extend([os.path.basename(item) for item in glob.glob(os.path.join(SETTINGS['filesystem'], 'proprietary', 'jw*'))])
-    program_list = list(set(program_list))
     pool = multiprocessing.Pool(processes=int(SETTINGS['cores']))
     program_list = [(element, overwrite) for element in program_list]
     results = pool.starmap(process_program, program_list)
@@ -819,7 +846,7 @@ def update_listfile(filename, file_list, filetype):
 
 
 @lock_module
-def protected_code(overwrite):
+def protected_code(overwrite, programs):
     """Protected code ensures only 1 instance of module will run at any given time
 
     Parameters
@@ -830,11 +857,11 @@ def protected_code(overwrite):
     module = os.path.basename(__file__).strip('.py')
     start_time, log_file = initialize_instrument_monitor(module)
 
-    generate_preview_images(overwrite)
+    generate_preview_images(overwrite, programs=programs)
     update_monitor_table(module, start_time, log_file)
 
 
 if __name__ == '__main__':
     parser = define_options()
     args = parser.parse_args()
-    protected_code(args.overwrite)
+    protected_code(args.overwrite, args.programs)

jwql/utils/constants.py

@@ -785,7 +785,7 @@
 ]
 
 # Possible suffix types for AMI files
-NIRISS_AMI_SUFFIX_TYPES = ["amiavg", "aminorm", "ami", "psf-amiavg"]
+NIRISS_AMI_SUFFIX_TYPES = ["amiavg", "aminorm", "ami", "psf-amiavg", "psf-ami-oi", "ami-oi", "aminorm-oi", "amimulti-oi", "amilg"]
 
 # Determine if the code is being run as part of CI checking on github
 ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')

jwql/utils/preview_image.py

@@ -490,16 +490,54 @@ def make_figure(self, image, integration_number, min_value, max_value,
                     dig = 2
                 format_string = "%.{}f".format(dig)
                 tlabelstr = [format_string % number for number in tlabelflt]
-                cbar = self.fig.colorbar(cax, ticks=tickvals)
 
                 # This seems to correctly remove the ticks and labels we want to remove. It gives a warning that
                 # it doesn't work on log scales, which we don't care about. So let's ignore that warning.
                 warnings.filterwarnings("ignore", message="AutoMinorLocator does not work with logarithmic scale")
-                cbar.ax.yaxis.set_minor_locator(AutoMinorLocator(n=0))
 
-                cbar.ax.set_yticklabels(tlabelstr)
-                cbar.ax.tick_params(labelsize=maxsize * 5. / 4)
-                cbar.ax.set_ylabel(self.units, labelpad=10, rotation=270)
+                xyratio = xsize / ysize
+                if xyratio < 1.6:
+                    # For apertures that are taller than they are wide, square, or that are wider than
+                    # they are tall but still reasonably close to square, put the colorbar on the right
+                    # side of the image.
+
+                    # Some magic numbers arrived at through testing aspect ratios for all apertures
+                    if xyratio > 0.4:
+                        cb_width = 0.05
+                    else:
+                        cb_width = 0.05 * 0.4 / xyratio
+
+                    upper_x_anchor = 0.02
+                    if xyratio < 0.1:
+                        upper_x_anchor = 0.12
+
+                    cbax = self.fig.add_axes([ax.get_position().x1 + upper_x_anchor,
+                                              ax.get_position().y0,
+                                              cb_width,
+                                              ax.get_position().height
+                                              ])
+                    cbar = self.fig.colorbar(cax, cax=cbax, ticks=tickvals, orientation='vertical')
+                    cbar.ax.yaxis.set_minor_locator(AutoMinorLocator(n=0))
+                    cbar.ax.set_yticklabels(tlabelstr)
+                    cbar.ax.set_ylabel(self.units, labelpad=7, rotation=270)
+                else:
+                    # For apertures that are significantly wider than they are tall, put the colorbar
+                    # under the image.
+
+                    # Again, some magic numbers controlling the positioning and height of the
+                    # colorbar, based on testing.
+                    lower_y_anchor = 0. - (xyratio / 14.5)
+                    cb_height = 0.07 * (np.log2(xyratio) - 1)
+
+                    cbax = self.fig.add_axes([ax.get_position().x0,
+                                              ax.get_position().y0 + lower_y_anchor,
+                                              ax.get_position().width,
+                                              cb_height])
+                    cbar = self.fig.colorbar(cax, cax=cbax, ticks=tickvals, orientation='horizontal')
+                    cbar.ax.xaxis.set_minor_locator(AutoMinorLocator(n=0))
+                    cbar.ax.set_xticklabels(tlabelstr)
+                    cbar.ax.set_xlabel(self.units, labelpad=7, rotation=0)
+
                 ax.set_xlabel('Pixels', fontsize=maxsize * 5. / 4)
                 ax.set_ylabel('Pixels', fontsize=maxsize * 5. / 4)
                 ax.tick_params(labelsize=maxsize)

jwql/utils/utils.py

@@ -530,7 +530,8 @@ def filename_parser(filename):
         time_series,
         time_series_2c,
         guider,
-        guider_segment]
+        guider_segment
+    ]
 
     filename_type_names = [
         'stage_1_and_2',

jwql/website/apps/jwql/forms.py

@@ -160,7 +160,7 @@ class JwqlQueryForm(BaseForm):
     num_choices = [(50, 50), (100, 100), (200, 200), (500, 500)]
     num_per_page = forms.ChoiceField(
         required=True,
-        choices=num_choices, initial=num_choices[1],
+        choices=num_choices, initial=num_choices[3],
         widget=forms.RadioSelect)
 
     # instrument specific parameters

jwql/website/apps/jwql/views.py

@@ -131,6 +131,7 @@ def jwql_query(request):
             parameters[QueryConfigKeys.DATE_RANGE] = form.cleaned_data['date_range']
             parameters[QueryConfigKeys.PROPOSAL_CATEGORY] = form.cleaned_data['proposal_category']
             parameters[QueryConfigKeys.SORT_TYPE] = form.cleaned_data['sort_type']
+            parameters[QueryConfigKeys.NUM_PER_PAGE] = form.cleaned_data['num_per_page']
             parameters[QueryConfigKeys.ANOMALIES] = all_anomalies
             parameters[QueryConfigKeys.APERTURES] = all_apers
             parameters[QueryConfigKeys.FILTERS] = all_filters

pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "jsonschema>=4.21.1,<5",
     "jwst>=1.13.4,<2",
     "jwst_backgrounds>=1.2.0,<1.3.0",
-    "jwst_reffiles@git+https://github.com/spacetelescope/jwst_reffiles",
+    "jwst_reffiles>=1.0.1",
     "matplotlib>=3.8.3,<4",
     "numpy>=1.26.4,<2",
     "pandas>=2.2.1,<3",