From 1834232cfa409f4339b333b368fd9ea2ddb308ce Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 13 Sep 2023 15:12:15 -0400 Subject: [PATCH 01/26] Add step_args to calwebb_detector1_save_jump --- jwql/shared_tasks/shared_tasks.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index be232935d..1e8e95549 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -408,7 +408,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, @celery_app.task(name='jwql.shared_tasks.shared_tasks.calwebb_detector1_save_jump') -def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True): +def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save_fitopt=True, step_args={}): """Call ``calwebb_detector1`` on the provided file, running all steps up to the ``ramp_fit`` step, and save the result. Optionally run the ``ramp_fit`` step and save the resulting slope file as well. @@ -430,6 +430,13 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save If ``True``, the file of optional outputs from the ramp fitting step of the pipeline is saved. + step_args : dict + A dictionary containing custom arguments to supply to individual pipeline steps. + When a step is run, the dictionary will be checked for a key matching the step + name (as defined in jwql.utils.utils.get_pipeline_steps() for the provided + instrument). The value matching the step key should, itself, be a dictionary that + can be spliced in to step.call() via dereferencing (**dict) + Returns ------- jump_output : str @@ -469,7 +476,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save cores = 'all' status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, - short_name, result_file, cores) + short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") @@ -484,7 +491,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save if core_fail: cores = "half" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, - input_file, short_name, result_file, cores) + input_file, short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") managed = True @@ -498,7 +505,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save if core_fail: cores = "none" status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, - input_file, short_name, result_file, cores) + input_file, short_name, result_file, cores, step_args) if status[-1].strip() == "SUCCEEDED": logging.info("Subprocess reports successful finish.") managed = True From 477072f37d93b36ed907d77d43204d74f1f600f8 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:22:52 -0400 Subject: [PATCH 02/26] create wrapper func for saving png files --- .../common_monitors/dark_monitor.py | 5 ++-- jwql/utils/utils.py | 23 ++++++++++++++++++- .../monitor_pages/monitor_bad_pixel_bokeh.py | 6 ++--- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index c1c2ecc4c..467a8bd1c 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -84,7 +84,6 @@ from astropy.modeling import models from astropy.stats import sigma_clipped_stats from astropy.time import Time -from bokeh.io import export_png from bokeh.models import ColorBar, ColumnDataSource, HoverTool, Legend from bokeh.models import LinearColorMapper from bokeh.plotting import figure @@ -106,7 +105,7 @@ from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS from jwql.utils.logging_functions import log_info, log_fail from jwql.utils.permissions import set_permissions -from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path +from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path, save_png THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt') @@ -350,7 +349,7 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no self.plot.add_layout(legend, 'below') # Save the plot in a png - export_png(self.plot, filename=output_filename) + save_png(self.plot, filename=output_filename) set_permissions(output_filename) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 33d0c272c..75cf49ca6 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -45,6 +45,7 @@ from bokeh.plotting import figure import numpy as np from PIL import Image +from selenium import webdriver from jwql.utils import permissions from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ @@ -170,7 +171,7 @@ def create_png_from_fits(filename, outdir): # Save the plot in a png output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) - export_png(plot, filename=output_filename) + save_png(plot, filename=output_filename) permissions.set_permissions(output_filename) return output_filename else: @@ -756,6 +757,26 @@ def read_png(filename): return img +def save_png(fig, filename=''): + """Starting with selenium version 4.10.0, our testing has shown that on the JWQL + servers, we need to specify an instance of a web driver when exporting a Bokeh + figure as a png. This is a wrapper function that creates the web driver instance + and calls Bokeh's export_png function. + + Parameters + ---------- + fig : bokeh.plotting.figure + Bokeh figure to be saved as a png + + filename : str + Filename to use for the png file + """ + options = webdriver.FirefoxOptions() + options.add_argument('-headless') + driver = webdriver.Firefox(options=options) + export_png(fig, filename=filename, webdriver=driver) + + def grouper(iterable, chunksize): """ Take a list of items (iterable), and group it into chunks of chunksize, with the diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 48c47e216..b172df2d4 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -22,7 +22,7 @@ from astropy.stats import sigma_clipped_stats from astropy.time import Time from bokeh.embed import components, file_html -from bokeh.io import export_png, show +from bokeh.io import show from bokeh.layouts import layout from bokeh.models import ColumnDataSource, DatetimeTickFormatter, HoverTool, Legend, LinearColorMapper, Panel, Tabs, Text, Title from bokeh.plotting import figure @@ -40,7 +40,7 @@ from jwql.utils.constants import BAD_PIXEL_MONITOR_MAX_POINTS_TO_PLOT, BAD_PIXEL_TYPES, DARKS_BAD_PIXEL_TYPES from jwql.utils.constants import DETECTOR_PER_INSTRUMENT, FLATS_BAD_PIXEL_TYPES, JWST_INSTRUMENT_NAMES_MIXEDCASE from jwql.utils.permissions import set_permissions -from jwql.utils.utils import filesystem_path, get_config, read_png +from jwql.utils.utils import filesystem_path, get_config, read_png, save_png SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) OUTPUT_DIR = get_config()['outputs'] @@ -593,7 +593,7 @@ def switch_to_png(self, filename, title): Title to add to the Figure """ # Save the figure as a png - export_png(self.plot, filename=filename) + save_png(self.plot, filename=filename) set_permissions(filename) # Read in the png and insert into a replacement figure From 03da9a8916d3dc438a7caaae96c0710a2708092b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:38:38 -0400 Subject: [PATCH 03/26] Quit driver when done. Add test --- jwql/tests/test_utils.py | 19 ++++++++++++++++++- jwql/utils/utils.py | 1 + 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index c2f6795c1..27bee1b5e 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -22,7 +22,11 @@ from pathlib import Path import pytest -from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, _validate_config +from bokeh.models import LinearColorMapper +from bokeh.plotting import figure +import numpy as np + +from jwql.utils.utils import copy_files, get_config, filename_parser, filesystem_path, save_png, _validate_config # Determine if tests are being run on Github Actions @@ -479,6 +483,19 @@ def test_filesystem_path(): assert check == location +def test_save_png(): + """Test that we can create a png file""" + plot = figure(title='test',tools='') + image = np.zeros((200,200)) + image[100:105, 100:105] = 1 + ny, nx = image.shape + mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) + imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") + #save_png(plot, filename='test.png') + from bokeh.io import export_png + export_png(plot, filename='test.png') + + @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') def test_validate_config(): """Test that the config validator works.""" diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 75cf49ca6..182c18191 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -775,6 +775,7 @@ def save_png(fig, filename=''): options.add_argument('-headless') driver = webdriver.Firefox(options=options) export_png(fig, filename=filename, webdriver=driver) + driver.quit() def grouper(iterable, chunksize): From f239ef2ccc290c3f05c657611e0d7903044fcf7a Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 15:54:07 -0400 Subject: [PATCH 04/26] RTD needs selenium now --- rtd_requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index aa1047b5e..d4b88e1fd 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -8,6 +8,7 @@ jwst==1.11.4 pygments==2.16.1 pytest==7.4.0 redis==5.0.0 +selenium==4.11.3 sphinx==6.2.1 sphinx_rtd_theme==1.2.2 stsci_rtd_theme==1.0.0 From 23d02e79a7a3dd9752c4505fdccc2a36f791059c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 21 Sep 2023 16:05:08 -0400 Subject: [PATCH 05/26] Fix test. Adjust rtd version --- jwql/tests/test_utils.py | 4 +--- rtd_requirements.txt | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 27bee1b5e..813bb0432 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -491,9 +491,7 @@ def test_save_png(): ny, nx = image.shape mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") - #save_png(plot, filename='test.png') - from bokeh.io import export_png - export_png(plot, filename='test.png') + save_png(plot, filename='test.png') @pytest.mark.skipif(ON_GITHUB_ACTIONS, reason='Requires access to central storage.') diff --git a/rtd_requirements.txt b/rtd_requirements.txt index d4b88e1fd..39500c2b6 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -8,7 +8,7 @@ jwst==1.11.4 pygments==2.16.1 pytest==7.4.0 redis==5.0.0 -selenium==4.11.3 +selenium==4.11.2 sphinx==6.2.1 sphinx_rtd_theme==1.2.2 stsci_rtd_theme==1.0.0 From b85a1b0f7080afc1a58a1e8070b22377cf878b58 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 21:21:42 +0000 Subject: [PATCH 06/26] Bump cryptography from 41.0.3 to 41.0.4 Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.3 to 41.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.3...41.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4391be8b3..d4f3a5616 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ bandit==1.7.5 beautifulsoup4==4.12.2 bokeh==2.4.3 celery==5.3.1 -cryptography==41.0.3 +cryptography==41.0.4 django==4.2.3 inflection==0.5.1 ipython==8.14.0 From 2f0e8258c8edb0952c1b95c6a80bcf5470b4d3b6 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 22 Sep 2023 12:37:11 -0400 Subject: [PATCH 07/26] Better logging. Fix param def. Allow empty dark file list --- .../common_monitors/bad_pixel_monitor.py | 7 ++++--- jwql/shared_tasks/run_pipeline.py | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py index 546fc6d80..cc4b7fb68 100755 --- a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py +++ b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py @@ -984,9 +984,10 @@ def process(self, illuminated_raw_files, illuminated_slope_files, flat_file_coun else: index += 1 - min_dark_time = min(dark_obstimes) - max_dark_time = max(dark_obstimes) - mid_dark_time = instrument_properties.mean_time(dark_obstimes) + if len(dark_slope_files) > 0: + min_dark_time = min(dark_obstimes) + max_dark_time = max(dark_obstimes) + mid_dark_time = instrument_properties.mean_time(dark_obstimes) # Check whether there are still enough files left to meet the threshold if illuminated_slope_files is None: diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 0531f0337..82ea0689e 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -10,6 +10,7 @@ import shutil import sys import time +import traceback from jwst import datamodels from jwst.dq_init import DQInitStep @@ -161,7 +162,8 @@ def run_pipe(input_file, short_name, work_directory, instrument, outputs, max_co with open(status_file, "a+") as status_f: status_f.write("EXCEPTION\n") status_f.write("{}\n".format(e)) - status_f.write("FAILED") + status_f.write("FAILED\n") + status_f.write(traceback.format_exc()) sys.exit(1) with open(status_file, "a+") as status_f: @@ -217,6 +219,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T params['refpix'] = dict(odd_even_rows=False) # Default CR rejection threshold is too low + params['jump'] = {} params['jump']['rejection_threshold'] = 15 # Set up to save jump step output @@ -284,7 +287,8 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T with open(status_file, "a+") as status_f: status_f.write("EXCEPTION\n") status_f.write("{}\n".format(e)) - status_f.write("FAILED") + status_f.write("FAILED\n") + status_f.write(traceback.format_exc()) sys.exit(1) with open(status_file, "a+") as status_f: From 7ce0d9c89e9ace520c98f2487762e5cb208156e4 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 25 Sep 2023 13:19:07 -0400 Subject: [PATCH 08/26] Fix typo in env file names in readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 70b4e173b..377005a11 100644 --- a/README.md +++ b/README.md @@ -83,16 +83,16 @@ source activate base/root **Note:** If you have added a step activating conda to your default terminal/shell (e.g. the `.bashrc`, `.zshrc`, or `.profile` file) then you don't need to do the above step. -Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3_9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): +Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`): ``` -conda env create -f environment_python_3_9.yml +conda env create -f environment_python_3.9.yml ``` or ``` -conda env create -f environment_python_3_10.yml +conda env create -f environment_python_3.10.yml ``` ### Configuration File From 55b12f399cc7504e05879157e2c3bfae7b85866c Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 25 Sep 2023 13:49:40 -0400 Subject: [PATCH 09/26] pep8 fixes --- jwql/shared_tasks/run_pipeline.py | 2 +- jwql/shared_tasks/shared_tasks.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/jwql/shared_tasks/run_pipeline.py b/jwql/shared_tasks/run_pipeline.py index 82ea0689e..991e08d47 100755 --- a/jwql/shared_tasks/run_pipeline.py +++ b/jwql/shared_tasks/run_pipeline.py @@ -348,7 +348,7 @@ def run_save_jump(input_file, short_name, work_directory, instrument, ramp_fit=T outputs = args.outputs step_args = args.step_args - status_file = os.path.join(working_path, short_name+"_status.txt") + status_file = os.path.join(working_path, short_name + "_status.txt") with open(status_file, 'w') as out_file: out_file.write("Starting Process\n") out_file.write("\tpipeline is {} ({})\n".format(pipe_type, type(pipe_type))) diff --git a/jwql/shared_tasks/shared_tasks.py b/jwql/shared_tasks/shared_tasks.py index 1e8e95549..f076e2035 100644 --- a/jwql/shared_tasks/shared_tasks.py +++ b/jwql/shared_tasks/shared_tasks.py @@ -1,4 +1,4 @@ - #! /usr/bin/env python +#! /usr/bin/env python """This module contains code for the celery application, which is used for any demanding work which should be restricted in terms of how many iterations are run simultaneously, or @@ -204,7 +204,7 @@ def log_subprocess_output(pipe): If a subprocess STDOUT has been set to subprocess.PIPE, this function will log each line to the logging output. """ - for line in iter(pipe.readline, b''): # b'\n'-separated lines + for line in iter(pipe.readline, b''): # b'\n'-separated lines logging.info("\t{}".format(line.decode('UTF-8').strip())) @@ -224,6 +224,7 @@ def after_setup_celery_logger(logger, **kwargs): def collect_after_task(**kwargs): gc.collect() + def convert_step_args_to_string(args_dict): """Convert the nested dictionary containing pipeline step parameter keyword/value pairs to a string so that it can be passed via command line @@ -239,17 +240,17 @@ def convert_step_args_to_string(args_dict): args_str : str String representation of ``args_dict`` """ - args_str="'{" + args_str = "'{" for i, step in enumerate(args_dict): args_str += f'"{step}":' args_str += '{' for j, (param, val) in enumerate(args_dict[step].items()): args_str += f'"{param}":"{val}"' - if j < len(args_dict[step])-1: + if j < len(args_dict[step]) - 1: args_str += ', ' args_str += "}" - if i < len(args_dict)-1: + if i < len(args_dict) - 1: args_str += ',' args_str += "}'" return args_str @@ -331,7 +332,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, current_dir = os.path.dirname(__file__) cmd_name = os.path.join(current_dir, "run_pipeline.py") outputs = ",".join(ext_or_exts) - result_file = os.path.join(cal_dir, short_name+"_status.txt") + result_file = os.path.join(cal_dir, short_name + "_status.txt") if "all" in ext_or_exts: logging.info("All outputs requested") if instrument.lower() != 'miri': @@ -399,7 +400,7 @@ def run_calwebb_detector1(input_file_name, short_name, ext_or_exts, instrument, set_permissions(os.path.join(output_dir, file)) logging.info("Removing local files.") - files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) + files_to_remove = glob(os.path.join(cal_dir, short_name + "*")) for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) @@ -472,7 +473,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save output_dir = os.path.join(config["transfer_dir"], "outgoing") cmd_name = os.path.join(os.path.dirname(__file__), "run_pipeline.py") - result_file = os.path.join(cal_dir, short_name+"_status.txt") + result_file = os.path.join(cal_dir, short_name + "_status.txt") cores = 'all' status = run_subprocess(cmd_name, "jump", "all", cal_dir, instrument, input_file, @@ -531,7 +532,7 @@ def calwebb_detector1_save_jump(input_file_name, instrument, ramp_fit=True, save files["fitopt_output"] = os.path.join(output_dir, file) logging.info("Removing local files.") - files_to_remove = glob(os.path.join(cal_dir, short_name+"*")) + files_to_remove = glob(os.path.join(cal_dir, short_name + "*")) for file_name in files_to_remove: logging.info("\tRemoving {}".format(file_name)) os.remove(file_name) From ebf05a9e22ca555ee3f7d9b48fcd48dd0068de5c Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Thu, 28 Sep 2023 10:59:37 -0400 Subject: [PATCH 10/26] Include migrations in source control --- .../apps/jwql/migrations/0001_initial.py | 45 +++++++++ .../migrations/0002_auto_20220913_1525.py | 27 ++++++ .../migrations/0003_auto_20220921_0955.py | 58 +++++++++++ .../migrations/0004_auto_20220922_0911.py | 33 +++++++ .../migrations/0005_auto_20220922_1422.py | 23 +++++ .../migrations/0006_auto_20230214_1624.py | 95 +++++++++++++++++++ .../migrations/0007_auto_20230222_1157.py | 22 +++++ .../migrations/0008_rootfileinfo_exp_type.py | 18 ++++ .../migrations/0009_auto_20230303_0930.py | 63 ++++++++++++ .../migrations/0010_auto_20230313_1053.py | 23 +++++ jwql/website/apps/jwql/migrations/__init__.py | 0 11 files changed, 407 insertions(+) create mode 100644 jwql/website/apps/jwql/migrations/0001_initial.py create mode 100644 jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py create mode 100644 jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py create mode 100644 jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py create mode 100644 jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py create mode 100644 jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py create mode 100644 jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py create mode 100644 jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py create mode 100644 jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py create mode 100644 jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py create mode 100644 jwql/website/apps/jwql/migrations/__init__.py diff --git a/jwql/website/apps/jwql/migrations/0001_initial.py b/jwql/website/apps/jwql/migrations/0001_initial.py new file mode 100644 index 000000000..9de5ca6a9 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0001_initial.py @@ -0,0 +1,45 @@ +# Generated by Django 3.1.7 on 2022-09-09 17:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ImageData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('inst', models.CharField(choices=[('FGS', 'FGS'), ('MIRI', 'MIRI'), ('NIRCam', 'NIRCam'), ('NIRISS', 'NIRISS'), ('NIRSpec', 'NIRSpec')], default=None, max_length=7, verbose_name='instrument')), + ('pub_date', models.DateTimeField(verbose_name='date published')), + ('filepath', models.FilePathField(path='/user/lchambers/jwql/')), + ], + options={ + 'verbose_name_plural': 'image data', + 'db_table': 'imagedata', + }, + ), + migrations.CreateModel( + name='InstrumentFilterHandler', + fields=[ + ('instrument', models.CharField(max_length=10, primary_key=True, serialize=False)), + ], + ), + migrations.CreateModel( + name='ThumbnailFilterInfo', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('observation', models.PositiveIntegerField()), + ('proposal', models.PositiveIntegerField()), + ('root_name', models.CharField(max_length=300)), + ('marked_viewed', models.BooleanField(default=False)), + ('inst_handler', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.instrumentfilterhandler')), + ], + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py new file mode 100644 index 000000000..049901811 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0002_auto_20220913_1525.py @@ -0,0 +1,27 @@ +# Generated by Django 3.1.7 on 2022-09-13 20:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='thumbnailfilterinfo', + name='id', + ), + migrations.AlterField( + model_name='instrumentfilterhandler', + name='instrument', + field=models.TextField(max_length=10, primary_key=True, serialize=False), + ), + migrations.AlterField( + model_name='thumbnailfilterinfo', + name='root_name', + field=models.TextField(max_length=300, primary_key=True, serialize=False), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py new file mode 100644 index 000000000..df7bc6965 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0003_auto_20220921_0955.py @@ -0,0 +1,58 @@ +# Generated by Django 3.1.7 on 2022-09-21 14:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0002_auto_20220913_1525'), + ] + + operations = [ + migrations.CreateModel( + name='Archive', + fields=[ + ('instrument', models.CharField(help_text='Instrument name', max_length=7, primary_key=True, serialize=False)), + ], + options={ + 'ordering': ['instrument'], + }, + ), + migrations.CreateModel( + name='Observation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('obsnum', models.CharField(help_text='Observation number, as a 3 digit string', max_length=3)), + ('number_of_files', models.IntegerField(default=0, help_text='Number of files in the proposal')), + ('obsstart', models.FloatField(default=0.0, help_text='Time of the beginning of the observation in MJD')), + ('obsend', models.FloatField(default=0.0, help_text='Time of the end of the observation in MJD')), + ('exptypes', models.CharField(default='', help_text='Comma-separated list of exposure types', max_length=100)), + ], + options={ + 'ordering': ['-obsnum'], + }, + ), + migrations.CreateModel( + name='Proposal', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('prop_id', models.CharField(help_text='5-digit proposal ID string', max_length=5)), + ('thumbnail_path', models.CharField(default='', help_text='Path to the proposal thumbnail', max_length=100)), + ('archive', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.archive')), + ], + options={ + 'ordering': ['-prop_id'], + 'unique_together': {('prop_id', 'archive')}, + }, + ), + migrations.DeleteModel( + name='ImageData', + ), + migrations.AddField( + model_name='observation', + name='proposal', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.proposal'), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py new file mode 100644 index 000000000..998dd2ce4 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0004_auto_20220922_0911.py @@ -0,0 +1,33 @@ +# Generated by Django 3.1.7 on 2022-09-22 14:11 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0003_auto_20220921_0955'), + ] + + operations = [ + migrations.AddField( + model_name='thumbnailfilterinfo', + name='obsnum', + field=models.CharField(default=11, help_text='Observation number, as a 3 digit string', max_length=3), + preserve_default=False, + ), + migrations.CreateModel( + name='RootFileInfo', + fields=[ + ('instrument', models.CharField(help_text='Instrument name', max_length=7)), + ('proposal', models.CharField(help_text='5-digit proposal ID string', max_length=5)), + ('root_name', models.TextField(max_length=300, primary_key=True, serialize=False)), + ('viewed', models.BooleanField(default=False)), + ('obsnum', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jwql.observation')), + ], + options={ + 'ordering': ['-root_name'], + }, + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py new file mode 100644 index 000000000..3c51cbe23 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0005_auto_20220922_1422.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.7 on 2022-09-22 19:22 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0004_auto_20220922_0911'), + ] + + operations = [ + migrations.RemoveField( + model_name='thumbnailfilterinfo', + name='inst_handler', + ), + migrations.DeleteModel( + name='InstrumentFilterHandler', + ), + migrations.DeleteModel( + name='ThumbnailFilterInfo', + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py new file mode 100644 index 000000000..493608df3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0006_auto_20230214_1624.py @@ -0,0 +1,95 @@ +# Generated by Django 3.1.7 on 2023-02-14 21:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0005_auto_20220922_1422'), + ] + + operations = [ + migrations.CreateModel( + name='Anomalies', + fields=[ + ('root_file_info', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='jwql.rootfileinfo')), + ('cosmic_ray_shower', models.BooleanField(default=False)), + ('diffraction_spike', models.BooleanField(default=False)), + ('excessive_saturation', models.BooleanField(default=False)), + ('guidestar_failure', models.BooleanField(default=False)), + ('persistence', models.BooleanField(default=False)), + ('crosstalk', models.BooleanField(default=False)), + ('data_transfer_error', models.BooleanField(default=False)), + ('ghost', models.BooleanField(default=False)), + ('snowball', models.BooleanField(default=False)), + ('column_pull_up', models.BooleanField(default=False)), + ('column_pull_down', models.BooleanField(default=False)), + ('dominant_msa_leakage', models.BooleanField(default=False)), + ('dragons_breath', models.BooleanField(default=False)), + ('mrs_glow', models.BooleanField(default=False)), + ('mrs_zipper', models.BooleanField(default=False)), + ('internal_reflection', models.BooleanField(default=False)), + ('optical_short', models.BooleanField(default=False)), + ('row_pull_up', models.BooleanField(default=False)), + ('row_pull_down', models.BooleanField(default=False)), + ('lrs_contamination', models.BooleanField(default=False)), + ('tree_rings', models.BooleanField(default=False)), + ('scattered_light', models.BooleanField(default=False)), + ('claws', models.BooleanField(default=False)), + ('wisps', models.BooleanField(default=False)), + ('tilt_event', models.BooleanField(default=False)), + ('light_saber', models.BooleanField(default=False)), + ('other', models.BooleanField(default=False)), + ], + options={ + 'ordering': ['-root_file_info'], + }, + ), + migrations.AddField( + model_name='proposal', + name='cat_type', + field=models.CharField(default='', help_text='Category Type', max_length=10), + ), + migrations.AddField( + model_name='rootfileinfo', + name='aperature', + field=models.CharField(default='', help_text='Aperature', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='detector', + field=models.CharField(default='', help_text='Detector', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='filter', + field=models.CharField(default='', help_text='Instrument name', max_length=7), + ), + migrations.AddField( + model_name='rootfileinfo', + name='grating', + field=models.CharField(default='', help_text='Grating', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='pupil', + field=models.CharField(default='', help_text='Pupil', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='read_patt', + field=models.CharField(default='', help_text='Read Pattern', max_length=40), + ), + migrations.AddField( + model_name='rootfileinfo', + name='read_patt_num', + field=models.IntegerField(default=0, help_text='Read Pattern Number'), + ), + migrations.AddField( + model_name='rootfileinfo', + name='subarray', + field=models.CharField(default='', help_text='Subarray', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py new file mode 100644 index 000000000..2d4ffdba3 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0007_auto_20230222_1157.py @@ -0,0 +1,22 @@ +# Generated by Django 3.1.7 on 2023-02-22 16:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0006_auto_20230214_1624'), + ] + + operations = [ + migrations.RemoveField( + model_name='rootfileinfo', + name='aperature', + ), + migrations.AddField( + model_name='rootfileinfo', + name='aperture', + field=models.CharField(default='', help_text='Aperture', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py new file mode 100644 index 000000000..2c67cd955 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0008_rootfileinfo_exp_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.7 on 2023-02-22 17:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0007_auto_20230222_1157'), + ] + + operations = [ + migrations.AddField( + model_name='rootfileinfo', + name='exp_type', + field=models.CharField(default='', help_text='Exposure Type', max_length=40), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py new file mode 100644 index 000000000..d0dbafc76 --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0009_auto_20230303_0930.py @@ -0,0 +1,63 @@ +# Generated by Django 3.1.7 on 2023-03-03 14:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0008_rootfileinfo_exp_type'), + ] + + operations = [ + migrations.RenameField( + model_name='proposal', + old_name='cat_type', + new_name='category', + ), + migrations.AddField( + model_name='rootfileinfo', + name='expstart', + field=models.FloatField(default=0.0, help_text='Exposure Start Time'), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='aperture', + field=models.CharField(blank=True, default='', help_text='Aperture', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='detector', + field=models.CharField(blank=True, default='', help_text='Detector', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='exp_type', + field=models.CharField(blank=True, default='', help_text='Exposure Type', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='filter', + field=models.CharField(blank=True, default='', help_text='Instrument name', max_length=7, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='grating', + field=models.CharField(blank=True, default='', help_text='Grating', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='pupil', + field=models.CharField(blank=True, default='', help_text='Pupil', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='read_patt', + field=models.CharField(blank=True, default='', help_text='Read Pattern', max_length=40, null=True), + ), + migrations.AlterField( + model_name='rootfileinfo', + name='subarray', + field=models.CharField(blank=True, default='', help_text='Subarray', max_length=40, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py new file mode 100644 index 000000000..3d80e95fe --- /dev/null +++ b/jwql/website/apps/jwql/migrations/0010_auto_20230313_1053.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.7 on 2023-03-13 15:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jwql', '0009_auto_20230303_0930'), + ] + + operations = [ + migrations.AddField( + model_name='anomalies', + name='flag_date', + field=models.DateTimeField(blank=True, help_text='flag date', null=True), + ), + migrations.AddField( + model_name='anomalies', + name='user', + field=models.CharField(blank=True, default='', help_text='user', max_length=50, null=True), + ), + ] diff --git a/jwql/website/apps/jwql/migrations/__init__.py b/jwql/website/apps/jwql/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb From a22b9ccc95958ff5a29f6ddbdc35978cbdd64682 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 14:51:41 -0400 Subject: [PATCH 11/26] pep8 fixes --- .../common_monitors/dark_monitor.py | 20 ++++--------- jwql/utils/utils.py | 10 +++++-- .../monitor_pages/monitor_bad_pixel_bokeh.py | 28 +++++-------------- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 467a8bd1c..c8d30c92a 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -275,14 +275,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no # Create figure start_time = Time(float(self.query_start), format='mjd').tt.datetime.strftime("%m/%d/%Y") - end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") + end_time = Time(float(self.query_end), format='mjd').tt.datetime.strftime("%m/%d/%Y") self.plot = figure(title=f'{self.aperture}: {num_files} files. {start_time} to {end_time}', tools='') # tools='pan,box_zoom,reset,wheel_zoom,save') self.plot.x_range.range_padding = self.plot.y_range.range_padding = 0 # Create the color mapper that will be used to scale the image - mapper = LinearColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LinearColorMapper(palette='Viridis256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev))) # Plot image and add color bar imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, @@ -291,13 +291,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no color_bar = ColorBar(color_mapper=mapper, width=8, title='DN/sec') self.plot.add_layout(color_bar, 'right') - # Add hover tool for all pixel values - #hover_tool = HoverTool(tooltips=[('(x, y):', '($x{int}, $y{int})'), - # ('value:', '@image') - # ], - # renderers=[imgplot]) - #self.plot.tools.append(hover_tool) - if (('FULL' in self.aperture) or ('_CEN' in self.aperture)): if hotxy is not None: @@ -337,14 +330,14 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no base_start = Time(float(base_parts[3]), format='mjd').tt.datetime base_end = Time(float(base_parts[5]), format='mjd').tt.datetime base_start_time = base_start.strftime("%m/%d/%Y") - base_end_time = base_end.strftime("%m/%d/%Y") + base_end_time = base_end.strftime("%m/%d/%Y") legend_title = f'Compared to dark from {base_start_time} to {base_end_time}' else: legend_title = 'Compared to previous mean dark' legend = Legend(items=[hot_legend, dead_legend, noisy_legend], location="center", orientation='vertical', - title = legend_title) + title=legend_title) self.plot.add_layout(legend, 'below') @@ -352,7 +345,6 @@ def create_mean_slope_figure(self, image, num_files, hotxy=None, deadxy=None, no save_png(self.plot, filename=output_filename) set_permissions(output_filename) - def get_metadata(self, filename): """Collect basic metadata from a fits file @@ -650,14 +642,14 @@ def overplot_bad_pix(self, pix_type, coords, values): sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0], pixels_y=coords[1] ) - ) + ) # Overplot the bad pixel locations badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y', source=sources[pix_type], color=colors[pix_type]) # Add to the legend - if numpix > 0: + if numpix > 0: if numpix <= DARK_MONITOR_MAX_BADPOINTS_TO_PLOT: text = f"{numpix} pix {adjective[pix_type]} than baseline" else: diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 182c18191..4b8951717 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -158,8 +158,7 @@ def create_png_from_fits(filename, outdir): plot.ygrid.visible = False # Create the color mapper that will be used to scale the image - #mapper = LogColorMapper(palette='Viridis256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) - mapper = LogColorMapper(palette='Greys256', low=(img_med-5*img_dev) ,high=(img_med+5*img_dev)) + mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)),high=(img_med + (5 * img_dev))) # Plot image imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, @@ -170,8 +169,13 @@ def create_png_from_fits(filename, outdir): plot.yaxis.visible = False # Save the plot in a png +<<<<<<< Updated upstream output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) save_png(plot, filename=output_filename) +======= + output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png')) + export_png(plot, filename=output_filename) +>>>>>>> Stashed changes permissions.set_permissions(output_filename) return output_filename else: @@ -750,7 +754,7 @@ def read_png(filename): # Copy the RGBA image into view, flipping it so it comes right-side up # with a lower-left origin - view[:,:,:] = np.flipud(np.asarray(rgba_img)) + view[:, :, :] = np.flipud(np.asarray(rgba_img)) else: view = None # Return the 2D version diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index b172df2d4..385efaa1f 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -74,8 +74,6 @@ def __init__(self, instrument): # Get the relevant database tables self.identify_tables() - #self.detectors = get_unique_values_per_column(self.pixel_table, 'detector') - self.detectors = sorted(DETECTOR_PER_INSTRUMENT[self.instrument]) if self.instrument == 'miri': self.detectors = ['MIRIMAGE'] @@ -128,7 +126,6 @@ def modify_bokeh_saved_html(self): self._html = "".join(newlines) - def run(self): # Right now, the aperture name in the query history table is used as the title of the @@ -163,7 +160,7 @@ def run(self): # Insert into our html template and save template_dir = os.path.join(os.path.dirname(__file__), '../templates') template_file = os.path.join(template_dir, 'bad_pixel_monitor_savefile_basic.html') - temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div':div} + temp_vars = {'inst': self.instrument, 'plot_script': script, 'plot_div': div} self._html = file_html(tabs, CDN, f'{self.instrument} bad pix monitor', template_file, temp_vars) # Modify the html such that our Django-related lines are kept in place, @@ -264,7 +261,6 @@ def __init__(self, pixel_table, instrument, detector): for badtype in self.badtypes: self.get_trending_data(badtype) - def get_most_recent_entry(self): """Get all nedded data from the database tables. """ @@ -312,7 +308,7 @@ def get_trending_data(self, badpix_type): # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), self.pixel_table.obs_mid_time) \ - .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ + .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ .all() # Organize the results @@ -463,7 +459,7 @@ def create_plot(self): self.plot = figure(title=title_text, tools=tools, x_axis_label="Pixel Number", y_axis_label="Pixel Number") else: - self.plot = figure(tools='') #, x_axis_label="Pixel Number", y_axis_label="Pixel Number") + self.plot = figure(tools='') self.plot.toolbar.logo = None self.plot.toolbar_location = None self.plot.min_border = 0 @@ -475,18 +471,9 @@ def create_plot(self): # Plot image if image is not None: ny, nx = image.shape - #imgplot = self.plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") - - - # Shift the figure title slightly right in this case to get it # to align with the axes - #self.plot = figure(title=title, x_range=(0, self._detlen), y_range=(0, self._detlen), width=xdim, height=ydim*, - # tools='pan,box_zoom,reset,wheel_zoom,save', x_axis_label="Pixel Number", y_axis_label="Pixel Number") self.plot.image_rgba(image=[image], x=0, y=0, dw=self._detlen, dh=self._detlen, alpha=0.5) - - - else: # If the background image is not present, manually set the x and y range self.plot.x_range.start = 0 @@ -511,11 +498,10 @@ def create_plot(self): legend = Legend(items=[plot_legend], location="center", orientation='vertical', - title = legend_title) + title=legend_title) self.plot.add_layout(legend, 'below') - def overplot_bad_pix(self): """Add a scatter plot of potential new bad pixels to the plot @@ -537,7 +523,7 @@ def overplot_bad_pix(self): # If there are no new bad pixels, write text within the figure mentioning that txt_source = ColumnDataSource(data=dict(x=[self._detlen / 10], y=[self._detlen / 2], text=[f'No new {self.badpix_type} pixels found'])) - glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value':'20px'}) + glyph = Text(x="x", y="y", text="text", angle=0., text_color="navy", text_font_size={'value': '20px'}) self.plot.add_glyph(txt_source, glyph) # Insert a fake one, in order to get the plot to be made @@ -668,7 +654,7 @@ def create_plot(self): string_time=string_times, value=[self.badpix_type] * len(self.num_pix) ) - ) + ) self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") @@ -696,7 +682,7 @@ def create_plot(self): time_pad = datetime.timedelta(days=1) self.plot.x_range.start = min(self.time) - time_pad self.plot.x_range.end = max(self.time) + time_pad - self.plot.grid.grid_line_color="white" + self.plot.grid.grid_line_color = "white" self.plot.xaxis.axis_label = 'Date' self.plot.yaxis.axis_label = f'Number of {self.badpix_type} pixels' From 7c2cd6618ee81e39cb7b9b09723f96f4beca4c51 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 15:03:00 -0400 Subject: [PATCH 12/26] more pep8 --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 4 ++-- jwql/tests/test_utils.py | 6 +++--- jwql/utils/utils.py | 7 +------ .../apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py | 6 +++--- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index c8d30c92a..c73dd3543 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -641,8 +641,8 @@ def overplot_bad_pix(self, pix_type, coords, values): sources[pix_type] = ColumnDataSource(data=dict(pixels_x=coords[0], pixels_y=coords[1] - ) - ) + ) + ) # Overplot the bad pixel locations badpixplots[pix_type] = self.plot.circle(x=f'pixels_x', y=f'pixels_y', diff --git a/jwql/tests/test_utils.py b/jwql/tests/test_utils.py index 813bb0432..cd50c6a01 100644 --- a/jwql/tests/test_utils.py +++ b/jwql/tests/test_utils.py @@ -485,11 +485,11 @@ def test_filesystem_path(): def test_save_png(): """Test that we can create a png file""" - plot = figure(title='test',tools='') - image = np.zeros((200,200)) + plot = figure(title='test', tools='') + image = np.zeros((200, 200)) image[100:105, 100:105] = 1 ny, nx = image.shape - mapper = LinearColorMapper(palette='Viridis256', low=0 ,high=1.1) + mapper = LinearColorMapper(palette='Viridis256', low=0, high=1.1) imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, color_mapper=mapper, level="image") save_png(plot, filename='test.png') diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index 4b8951717..ca4df70a3 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -169,13 +169,8 @@ def create_png_from_fits(filename, outdir): plot.yaxis.visible = False # Save the plot in a png -<<<<<<< Updated upstream - output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits','png')) - save_png(plot, filename=output_filename) -======= output_filename = os.path.join(outdir, os.path.basename(filename).replace('fits', 'png')) - export_png(plot, filename=output_filename) ->>>>>>> Stashed changes + save_png(plot, filename=output_filename) permissions.set_permissions(output_filename) return output_filename else: diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py index 385efaa1f..5e522f7cd 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_bad_pixel_bokeh.py @@ -308,8 +308,8 @@ def get_trending_data(self, badpix_type): # Query database for all data in the table with a matching detector and bad pixel type all_entries_by_type = session.query(self.pixel_table.type, self.pixel_table.detector, func.array_length(self.pixel_table.x_coord, 1), self.pixel_table.obs_mid_time) \ - .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ - .all() + .filter(and_(self.pixel_table.detector == self.detector, self.pixel_table.type == badpix_type)) \ + .all() # Organize the results num_pix = [] @@ -654,7 +654,7 @@ def create_plot(self): string_time=string_times, value=[self.badpix_type] * len(self.num_pix) ) - ) + ) self.plot = figure(title=f'{self.detector}: New {self.badpix_type} Pixels', tools='pan,box_zoom,reset,wheel_zoom,save', background_fill_color="#fafafa") From 6fc7ad384071ed1bdd884c63a6c7ed22aa33174f Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 28 Sep 2023 15:04:49 -0400 Subject: [PATCH 13/26] pep8, make up your mind --- jwql/utils/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/jwql/utils/utils.py b/jwql/utils/utils.py index ca4df70a3..113dd66c9 100644 --- a/jwql/utils/utils.py +++ b/jwql/utils/utils.py @@ -49,11 +49,11 @@ from jwql.utils import permissions from jwql.utils.constants import FILE_AC_CAR_ID_LEN, FILE_AC_O_ID_LEN, FILE_ACT_LEN, \ - FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ - FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ - FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ - FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ - JWST_INSTRUMENT_NAMES_SHORTHAND + FILE_DATETIME_LEN, FILE_EPOCH_LEN, FILE_GUIDESTAR_ATTMPT_LEN_MIN, \ + FILE_GUIDESTAR_ATTMPT_LEN_MAX, FILE_OBS_LEN, FILE_PARALLEL_SEQ_ID_LEN, \ + FILE_PROG_ID_LEN, FILE_SEG_LEN, FILE_SOURCE_ID_LEN, FILE_SUFFIX_TYPES, \ + FILE_TARG_ID_LEN, FILE_VISIT_GRP_LEN, FILE_VISIT_LEN, FILETYPE_WO_STANDARD_SUFFIX, \ + JWST_INSTRUMENT_NAMES_SHORTHAND __location__ = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) @@ -158,7 +158,7 @@ def create_png_from_fits(filename, outdir): plot.ygrid.visible = False # Create the color mapper that will be used to scale the image - mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)),high=(img_med + (5 * img_dev))) + mapper = LogColorMapper(palette='Greys256', low=(img_med - (5 * img_dev)), high=(img_med + (5 * img_dev))) # Plot image imgplot = plot.image(image=[image], x=0, y=0, dw=nx, dh=ny, From f81de385fa3105aa2470b774014bddcb243d00d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:50:14 +0000 Subject: [PATCH 14/26] Bump scipy from 1.9.3 to 1.10.0 Bumps [scipy](https://github.com/scipy/scipy) from 1.9.3 to 1.10.0. - [Release notes](https://github.com/scipy/scipy/releases) - [Commits](https://github.com/scipy/scipy/compare/v1.9.3...v1.10.0) --- updated-dependencies: - dependency-name: scipy dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d4f3a5616..4270e9d45 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ pyvo==1.4.2 pyyaml==6.0 redis==5.0.0 ruff==0.0.285 -scipy==1.9.3 +scipy==1.10.0 selenium==4.11.2 setuptools==68.0.0 sphinx==6.2.1 From 8c2fd616cdaf51ca280ff0dd77cf0a2dbf922e82 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:07:04 -0400 Subject: [PATCH 15/26] update deps in environment files --- environment_python_3.10.yml | 50 ++++++++++++------------ environment_python_3.9.yml | 46 +++++++++++----------- requirements.txt | 78 ++++++++++++++++++------------------- 3 files changed, 87 insertions(+), 87 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index f61973680..7856cf201 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,49 +22,49 @@ channels: - defaults dependencies: - - astropy=5.3.2 + - astropy=5.3.3 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - - celery=5.3.1 - - cryptography=41.0.3 - - django=4.2.3 + - celery=5.3.4 + - cryptography=41.0.4 + - django=4.2.5 - inflection=0.5.1 - - ipython=8.14.0 + - ipython=8.16.1 - jinja2=3.1.2 - - jsonschema=4.19.0 - - matplotlib=3.7.2 - - nodejs=20.5.1 + - jsonschema=4.19.1 + - matplotlib=3.8.0 + - nodejs>=18.15.0 - numpy=1.25.2 - numpydoc=1.5.0 - - pandas=2.0.3 + - pandas=2.1.1 - pip=23.2.1 - - postgresql=15.4 - - psycopg2=2.9.6 - - pytest=7.4.0 + - postgresql>=15.2 + - psycopg2>=2.9.3 + - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - - python=3.10.12 - - pyyaml=6.0 - - redis - - ruff=0.0.285 + - python=3.10.13 + - pyyaml=6.0.1 + - redis=5.0.1 + - ruff=0.0.292 - scipy=1.9.3 - - setuptools=68.0.0 - - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.20 + - setuptools=68.2.2 + - sphinx=7.2.6 + - sphinx_rtd_theme=1.3.0 + - sqlalchemy=2.0.21 - twine=4.0.2 - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.11.4 + - jwst==1.12.2 - pysiaf==0.20.0 - - pysqlite3==0.5.1 + - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.0 - - selenium==4.11.2 - - stdatamodels==1.7.2 + - redis==5.0.1 + - selenium==4.13.0 + - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index d3724b1d8..554355df3 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -22,49 +22,49 @@ channels: - defaults dependencies: - - astropy=5.3.2 + - astropy=5.3.3 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - - celery=5.3.1 - - cryptography=41.0.3 - - django=4.2.3 + - celery=5.3.4 + - cryptography=41.0.4 + - django=4.2.5 - inflection=0.5.1 - - ipython=8.14.0 + - ipython=8.16.1 - jinja2=3.1.2 - - jsonschema=4.19.0 - - matplotlib=3.7.2 - - nodejs=20.5.1 + - jsonschema=4.19.1 + - matplotlib=3.8.0 + - nodejs=20.8.0 - numpy=1.25.2 - numpydoc=1.5.0 - - pandas=2.0.3 + - pandas=2.1.1 - pip=23.2.1 - postgresql=15.4 - - psycopg2=2.9.6 - - pytest=7.4.0 + - psycopg2=2.9.7 + - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - python=3.9.17 - - pyyaml=6.0 - - redis - - ruff=0.0.285 + - pyyaml=6.0.1 + - redis=5.0.1 + - ruff=0.0.292 - scipy=1.9.3 - - setuptools=68.0.0 - - sphinx=6.2.1 - - sphinx_rtd_theme=1.2.2 - - sqlalchemy=2.0.20 + - setuptools=68.2.2 + - sphinx=7.2.6 + - sphinx_rtd_theme=1.3.0 + - sqlalchemy=2.0.21 - twine=4.0.2 - wtforms=3.0.1 - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.11.4 + - jwst==1.12.2 - pysiaf==0.20.0 - - pysqlite3==0.5.1 + - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.0 - - selenium==4.11.2 - - stdatamodels==1.7.2 + - redis==5.0.1 + - selenium==4.13.0 + - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 - vine==5.0.0 - git+https://github.com/spacetelescope/jwst_reffiles diff --git a/requirements.txt b/requirements.txt index 4270e9d45..6b8d88d41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,40 @@ -astropy==5.3.2 -astroquery==0.4.6 -bandit==1.7.5 -beautifulsoup4==4.12.2 -bokeh==2.4.3 -celery==5.3.1 -cryptography==41.0.4 -django==4.2.3 -inflection==0.5.1 -ipython==8.14.0 -jinja2==3.1.2 -jsonschema==4.19.0 -jwst==1.11.4 -matplotlib==3.7.2 -nodejs==20.5.1 -numpy==1.25.2 -numpydoc==1.5.0 -pandas==2.0.3 -psycopg2-binary==2.9.7 -pysiaf==0.20.0 -pysqlite3==0.5.1 -pytest==7.4.0 -pytest-cov==4.1.0 -pytest-mock==3.11.1 -pyvo==1.4.2 -pyyaml==6.0 -redis==5.0.0 -ruff==0.0.285 -scipy==1.10.0 -selenium==4.11.2 -setuptools==68.0.0 -sphinx==6.2.1 -sphinx_rtd_theme==1.2.2 -sqlalchemy==2.0.20 -stdatamodels==1.7.2 -stsci_rtd_theme==1.0.0 -twine==4.0.2 -vine==5.0.0 -wtforms==3.0.1 +astropy +astroquery +bandit +beautifulsoup4 +bokeh +celery +cryptography +django +inflection +ipython +jinja2 +jsonschema +jwst +matplotlib +nodejs +numpy +numpydoc +pandas +psycopg2-binary +pysiaf +pysqlite3 +pytest +pytest-cov +pytest-mock +pyvo +pyyaml +redis +ruff +scipy +selenium +setuptools +sphinx +sphinx_rtd_theme +sqlalchemy +stdatamodels +stsci_rtd_theme +twine +vine +wtforms git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles From c0de0112325fa00644caa9cbe619d25bb63aa670 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:18:34 -0400 Subject: [PATCH 16/26] update versions in requirements.txt --- requirements.txt | 78 ++++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6b8d88d41..bb289de5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,40 +1,40 @@ -astropy -astroquery -bandit -beautifulsoup4 -bokeh -celery -cryptography -django -inflection -ipython -jinja2 -jsonschema -jwst -matplotlib -nodejs -numpy -numpydoc -pandas -psycopg2-binary -pysiaf -pysqlite3 -pytest -pytest-cov -pytest-mock -pyvo -pyyaml -redis -ruff -scipy -selenium -setuptools -sphinx -sphinx_rtd_theme -sqlalchemy -stdatamodels -stsci_rtd_theme -twine -vine -wtforms +astropy==5.3.3 +astroquery==0.4.6 +bandit==1.7.5 +beautifulsoup4==4.12.2 +bokeh==2.4.3 +celery==5.3.4 +cryptography==41.0.4 +django==4.2.5 +inflection==0.5.1 +ipython==8.16.1 +jinja2==3.1.2 +jsonschema==4.19.1 +jwst==1.12.2 +matplotlib==3.8.0 +nodejs==20.8.0 +numpy==1.25.2 +numpydoc==1.5.0 +pandas==2.1.1 +psycopg2-binary==2.9.7 +pysiaf==0.20.0 +pysqlite3==0.5.2 +pytest==7.4.2 +pytest-cov==4.1.0 +pytest-mock==3.11.1 +pyvo==1.4.2 +pyyaml==6.0.1 +redis==5.0.1 +ruff==0.0.292 +scipy==1.9.3 +selenium==4.13.0 +setuptools==68.2.2 +sphinx==7.2.6 +sphinx_rtd_theme==1.3.0 +sqlalchemy==2.0.21 +stdatamodels==1.8.3 +stsci_rtd_theme==1.0.0 +twine==4.0.2 +vine==5.0.0 +wtforms==3.0.1 git+https://github.com/spacetelescope/jwst_reffiles#egg=jwst_reffiles From 8ae4da424b1cf99c8a6e20cabe504c938285c228 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:21:14 -0400 Subject: [PATCH 17/26] put redis back to 5.0.0 --- environment_python_3.10.yml | 2 +- jwql/utils/logging_functions.py | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 7856cf201..505bdb7c0 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -45,7 +45,7 @@ dependencies: - pytest-mock=3.11.1 - python=3.10.13 - pyyaml=6.0.1 - - redis=5.0.1 + - redis=5.0.0 - ruff=0.0.292 - scipy=1.9.3 - setuptools=68.2.2 diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index abd82a45e..3ef464a40 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -106,7 +106,7 @@ def configure_logging(module): logging.basicConfig(filename=log_file, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S %p', - level=logging.INFO) + level=logging.DEBUG) print('Log file initialized to {}'.format(log_file)) set_permissions(log_file) diff --git a/requirements.txt b/requirements.txt index bb289de5a..d82a3409f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ pytest-cov==4.1.0 pytest-mock==3.11.1 pyvo==1.4.2 pyyaml==6.0.1 -redis==5.0.1 +redis==5.0.0. ruff==0.0.292 scipy==1.9.3 selenium==4.13.0 From 9ef1cd9356335913eab0562282cad68e16f1f579 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Tue, 3 Oct 2023 16:23:10 -0400 Subject: [PATCH 18/26] whoops, accidental logging change --- jwql/utils/logging_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwql/utils/logging_functions.py b/jwql/utils/logging_functions.py index 3ef464a40..abd82a45e 100644 --- a/jwql/utils/logging_functions.py +++ b/jwql/utils/logging_functions.py @@ -106,7 +106,7 @@ def configure_logging(module): logging.basicConfig(filename=log_file, format='%(asctime)s %(levelname)s: %(message)s', datefmt='%m/%d/%Y %H:%M:%S %p', - level=logging.DEBUG) + level=logging.INFO) print('Log file initialized to {}'.format(log_file)) set_permissions(log_file) From b4e4c38e856adfa88e82483dcfc4f81b6e5bbb52 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 11:04:47 -0400 Subject: [PATCH 19/26] revert all redis references to 5.0.0 --- environment_python_3.10.yml | 2 +- environment_python_3.9.yml | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 505bdb7c0..518fbc902 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -62,7 +62,7 @@ dependencies: - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.1 + - redis==5.0.0 - selenium==4.13.0 - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 554355df3..5b70010a8 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -45,7 +45,7 @@ dependencies: - pytest-mock=3.11.1 - python=3.9.17 - pyyaml=6.0.1 - - redis=5.0.1 + - redis=5.0.0 - ruff=0.0.292 - scipy=1.9.3 - setuptools=68.2.2 @@ -62,7 +62,7 @@ dependencies: - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 - - redis==5.0.1 + - redis==5.0.0 - selenium==4.13.0 - stdatamodels==1.8.3 - stsci_rtd_theme==1.0.0 diff --git a/requirements.txt b/requirements.txt index d82a3409f..01aeb4f7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ pytest-cov==4.1.0 pytest-mock==3.11.1 pyvo==1.4.2 pyyaml==6.0.1 -redis==5.0.0. +redis==5.0.0 ruff==0.0.292 scipy==1.9.3 selenium==4.13.0 From b874639b0e45e0a67b8813a9b36a4d12f02af7a1 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 12:51:46 -0400 Subject: [PATCH 20/26] mamba didn't like 3.10.13 --- environment_python_3.10.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index 518fbc902..f60c31d27 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -43,7 +43,7 @@ dependencies: - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 - - python=3.10.13 + - python=3.10.12 - pyyaml=6.0.1 - redis=5.0.0 - ruff=0.0.292 From 6edf1262793467580a131f5c745682e980d8cecb Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 15:32:45 -0400 Subject: [PATCH 21/26] update for 3.10.12 --- environment_python_3.10.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index f60c31d27..fd927ce30 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -22,24 +22,24 @@ channels: - defaults dependencies: - - astropy=5.3.3 + - astropy=5.3.4 - beautifulsoup4=4.12.2 - bokeh=2.4.3 - celery=5.3.4 - cryptography=41.0.4 - - django=4.2.5 + - django=4.2.6 - inflection=0.5.1 - ipython=8.16.1 - jinja2=3.1.2 - jsonschema=4.19.1 - matplotlib=3.8.0 - - nodejs>=18.15.0 + - nodejs=20.8.0 - numpy=1.25.2 - numpydoc=1.5.0 - pandas=2.1.1 - pip=23.2.1 - - postgresql>=15.2 - - psycopg2>=2.9.3 + - postgresql=15.4 + - psycopg2=2.9.7 - pytest=7.4.2 - pytest-cov=4.1.0 - pytest-mock=3.11.1 From 91feef45fdbf47a9e994c28ef8b3921ecae7d68f Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:29:43 -0400 Subject: [PATCH 22/26] update rtd_requirements.txt --- rtd_requirements.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index 39500c2b6..f4f42830e 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -1,16 +1,16 @@ -sphinx_automodapi==0.15.0 +sphinx_automodapi>=0.15.0 bokeh==2.4.3 -celery==5.3.1 -cython==3.0.0 -django==4.2.3 -docutils==0.18.1 -jwst==1.11.4 +celery==5.3.4 +cython>=3.0.0 +django==4.2.5 +docutils>=0.18.1 +jwst==1.12.3 pygments==2.16.1 -pytest==7.4.0 +pytest==7.4.2 redis==5.0.0 -selenium==4.11.2 -sphinx==6.2.1 -sphinx_rtd_theme==1.2.2 +selenium==4.13.0 +sphinx==7.2.6 +sphinx_rtd_theme==1.30 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles From 059310ac90171af382c1c91b130861e20cdb0c6d Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:35:59 -0400 Subject: [PATCH 23/26] jwst changed out from under me --- environment_python_3.10.yml | 2 +- environment_python_3.9.yml | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/environment_python_3.10.yml b/environment_python_3.10.yml index fd927ce30..9f2ed409b 100644 --- a/environment_python_3.10.yml +++ b/environment_python_3.10.yml @@ -58,7 +58,7 @@ dependencies: - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.12.2 + - jwst==1.12.3 - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 diff --git a/environment_python_3.9.yml b/environment_python_3.9.yml index 5b70010a8..4f4159244 100644 --- a/environment_python_3.9.yml +++ b/environment_python_3.9.yml @@ -58,7 +58,7 @@ dependencies: - pip: - astroquery==0.4.6 - bandit==1.7.5 - - jwst==1.12.2 + - jwst==1.12.3 - pysiaf==0.20.0 - pysqlite3==0.5.2 - pyvo==1.4.2 diff --git a/requirements.txt b/requirements.txt index 01aeb4f7f..6a7f670cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ inflection==0.5.1 ipython==8.16.1 jinja2==3.1.2 jsonschema==4.19.1 -jwst==1.12.2 +jwst==1.12.3 matplotlib==3.8.0 nodejs==20.8.0 numpy==1.25.2 From fdfd4b674a9a6db8468fe16e72a235e537894079 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:40:32 -0400 Subject: [PATCH 24/26] update setuptools version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c17d3dc28..133af8c79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ file = "LICENSE" content-type = "text/plain" [build-system] -requires = ["setuptools>=61.2,<=65.5.0", "numpy", "wheel", "setuptools_scm"] +requires = ["setuptools=68.2.2", "numpy", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [tool.setuptools] From f365329f7023f0d18596e59ee6f1979d76d51f31 Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:51:14 -0400 Subject: [PATCH 25/26] setuptools greater than --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 133af8c79..24f701717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ file = "LICENSE" content-type = "text/plain" [build-system] -requires = ["setuptools=68.2.2", "numpy", "wheel", "setuptools_scm"] +requires = ["setuptools>=68.0.0", "numpy", "wheel", "setuptools_scm"] build-backend = "setuptools.build_meta" [tool.setuptools] From f7ecd691e9034e3828a21406ebe00372721ec09a Mon Sep 17 00:00:00 2001 From: Bradley Sappington Date: Fri, 6 Oct 2023 16:54:54 -0400 Subject: [PATCH 26/26] no thank you sphinx_theme. stsci_theme 4ever --- rtd_requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/rtd_requirements.txt b/rtd_requirements.txt index f4f42830e..abac6b128 100644 --- a/rtd_requirements.txt +++ b/rtd_requirements.txt @@ -10,7 +10,6 @@ pytest==7.4.2 redis==5.0.0 selenium==4.13.0 sphinx==7.2.6 -sphinx_rtd_theme==1.30 stsci_rtd_theme==1.0.0 tomli==2.0.1 git+https://github.com/spacetelescope/jwst_reffiles