From e60f971b7dbb8595f3336b68898f307cc4cc5402 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Tue, 28 Apr 2020 11:41:04 -0400 Subject: [PATCH 001/227] Initial commit of test_bokeh_templating --- jwql/tests/test_bokeh_templating.py | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 jwql/tests/test_bokeh_templating.py diff --git a/jwql/tests/test_bokeh_templating.py b/jwql/tests/test_bokeh_templating.py new file mode 100644 index 000000000..9988403d4 --- /dev/null +++ b/jwql/tests/test_bokeh_templating.py @@ -0,0 +1,33 @@ +#! /usr/bin/env python + +""" +""" + +from jwql.bokeh_templating import BokehTemplate + + +class TestTemplate(BokehTemplate): + """ + """ + + def pre_init(self): + """ + """ + + self.interface_file = 'test_interface.yaml' # Some basic plot(s) + + def post_init(self): + """ + """ + +def test_bokeh_templating(): + """ + """ + + test_template = TestTemplate() + + tabs = 'Some collection of plot(s)' + script, div = components(tabs) + + assert script is script obj + assert div is div obj \ No newline at end of file From 2c8a244f6dcf8ea3bf1ba9432d25435822c70e6f Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 24 Jul 2020 13:48:38 -0400 Subject: [PATCH 002/227] Add tests --- jwql/tests/test_badpix_monitor.py | 216 ++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 jwql/tests/test_badpix_monitor.py diff --git a/jwql/tests/test_badpix_monitor.py b/jwql/tests/test_badpix_monitor.py new file mode 100644 index 000000000..d2e3d388d --- /dev/null +++ b/jwql/tests/test_badpix_monitor.py @@ -0,0 +1,216 @@ +#! /usr/bin/env python + +"""Tests for the bad pixel monitor module. + +Authors +------- + + - Bryan Hilbert + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s test_badpix_monitor.py +""" + +import numpy as np +import os +import pytest + +from jwst.datamodels import dqflags + +from jwql.database.database_interface import NIRCamBadPixelQueryHistory, NIRCamBadPixelStats +from jwql.database.database_interface import NIRISSBadPixelQueryHistory, NIRISSBadPixelStats +from jwql.database.database_interface import MIRIBadPixelQueryHistory, MIRIBadPixelStats +from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats +from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats +from jwql.instrument_monitors.common_monitors import bad_pixel_monitor + + +ON_JENKINS = '/home/jenkins' in os.path.expanduser('~') + + +def test_bad_map_to_list(): + """Check that bad pixel locations are correctly retrieved from an image + """ + # Create empty DQ image + image = np.zeros((7, 7), dtype=np.uint16) + + # Set bad pixel locations + do_not_use = ([0, 2, 4, 6], [0, 2, 4, 6]) + hot = ([0, 1, 3, 5], [0, 2, 4, 6]) + rc = ([0, 2, 4, 6], [0, 1, 3, 5]) + + # Populate with bad pixels + image[do_not_use] = dqflags.pixel['DO_NOT_USE'] + image[hot] += dqflags.pixel['HOT'] + image[rc] += dqflags.pixel['RC'] + + # Call the bad pixel monitor + x_do_not_use, y_do_not_use = bad_pixel_monitor.bad_map_to_list(image, 'DO_NOT_USE') + x_hot, y_hot = bad_pixel_monitor.bad_map_to_list(image, 'HOT') + x_rc, y_rc = bad_pixel_monitor.bad_map_to_list(image, 'RC') + + # Test + assert do_not_use == (y_do_not_use, x_do_not_use) + assert hot == (y_hot, x_hot) + assert rc == (y_rc, x_rc) + + +def test_check_for_sufficient_files(): + """Be sure that the file threshold values are being used correctly + """ + input_files = ['file1.fits', 'file2.fits', 'file1.fits', 'file3.fits', 'file1.fits'] + instrument = 'nircam' + aperture = 'NRCA1_FULL' + threshold = 2 + file_type = 'darks' + + files, to_run = bad_pixel_monitor.check_for_sufficient_files(input_files, instrument, aperture, threshold, file_type) + assert files == ['file1.fits', 'file2.fits', 'file3.fits'] + assert to_run is True + + threshold = 4 + files, to_run = bad_pixel_monitor.check_for_sufficient_files(input_files, instrument, aperture, threshold, file_type) + assert files is None + assert to_run is False + + +def test_exclude_crds_mask_pix(): + """Test that bad pixel images are differentiated correctly + """ + common_bad = ([0, 1, 2, 3, 4], [0, 1, 2, 3, 4]) + bad1_only = ([0, 1, 3, 4], [4, 3, 1, 0]) + bad2_only = ([3, 3, 3, 3], [0, 1, 2, 4]) + + bad1 = np.zeros((5, 5), dtype=np.uint8) + bad1[common_bad] = 1 + bad1[bad1_only] = 2 + + bad2 = np.zeros((5, 5), dtype=np.uint8) + bad2[common_bad] = 1 + bad2[bad2_only] = 4 + + # Create a mask to help with indexing + mask = np.zeros(bad1.shape, dtype=bool) + mask[bad1_only] = True + + diff = bad_pixel_monitor.exclude_crds_mask_pix(bad1, bad2) + assert np.all(diff[mask] == 2) + assert np.all(diff[~mask] == 0) + + # Test the reverse case + mask = np.zeros(bad2.shape, dtype=bool) + mask[bad2_only] = True + + diff = bad_pixel_monitor.exclude_crds_mask_pix(bad2, bad1) + assert np.all(diff[mask] == 4) + assert np.all(diff[~mask] == 0) + + +@pytest.mark.skipif(ON_JENKINS, + reason='Requires access to central storage.') +def test_locate_rate_files(): + """Test that rate files are found in filesystem""" + + uncal_files = ['jw00300002001_02102_00001_mirimage_uncal.fits', 'jw00300002001_0210a_00001_mirimage_uncal.fits'] + ratefiles, ratefiles2copy = bad_pixel_monitor.locate_rate_files(uncal_files) + + rates = [os.path.basename(entry) for entry in ratefiles] + rates2copy = [os.path.basename(entry) for entry in ratefiles2copy] + + expected = ['jw00300002001_02102_00001_mirimage_rateints.fits', 'jw00300002001_0210a_00001_mirimage_rateints.fits'] + assert rates == expected + assert rates2copy == expected + + +@pytest.mark.skipif(ON_JENKINS, + reason='Requires access to central storage.') +def test_locate_uncal_files(): + """Test the filesystem search for uncal files + """ + file1 = 'jw00300002001_02102_00001_mirimage_rate.fits' + file2 = 'jw00300010001_02102_00001_mirifushort_uncal.fits' + query_results = [{'filename': file1}, + {'filename': file2}] + + found = bad_pixel_monitor.locate_uncal_files(query_results) + + found_base = [os.path.basename(entry) for entry in found] + assert found_base[0] == file1.replace('rate', 'uncal') + assert found_base[1] == file2 + + +def test_filter_query_results(): + """Test MAST query filtering to extract most common filter/pupil and + acceptable readout patterns + """ + badpix = bad_pixel_monitor.BadPixels() + badpix.instrument = 'nircam' + + dict1 = {'filter': 'F070W', 'pupil': 'CLEAR', 'readpatt': 'RAPID'} + dict2 = {'filter': 'F090W', 'pupil': 'CLEAR', 'readpatt': 'BRIGHT1'} + dict3 = {'filter': 'F070W', 'pupil': 'CLEAR', 'readpatt': 'BRIGHT1'} + query_results = [dict1, dict2, dict1, dict1, dict3, dict1] + + filtered = badpix.filter_query_results(query_results, 'flat') + + assert filtered == [dict1, dict1, dict1, dict1] + + # NIRSpec + badpix.instrument = 'nirspec' + + dict1 = {'filter': 'F070W', 'grating': 'CLEAR', 'readpatt': 'NRSRAPID'} + dict2 = {'filter': 'F070W', 'grating': 'CLEAR', 'readpatt': 'NRSIRS2'} + dict3 = {'filter': 'F070W', 'grating': 'CLEAR', 'readpatt': 'NRSRAPID'} + query_results = [dict1, dict2, dict1, dict1, dict3, dict1] + + filtered = badpix.filter_query_results(query_results, 'flat') + + assert filtered == [dict1, dict1, dict1, dict3, dict1] + + +nrc_list = ['NRCA1_FULL', 'NRCB1_FULL', 'NRCA2_FULL', 'NRCB2_FULL', 'NRCA3_FULL', 'NRCB3_FULL', + 'NRCA4_FULL', 'NRCB4_FULL', 'NRCA5_FULL', 'NRCB5_FULL'] +nis_list = ['NIS_CEN'] +nrs_list = ['NRS1_FULL', 'NRS2_FULL'] +miri_list = [('MIRIMAGE', 'MIRIM_FULL'), ('MIRIFULONG', 'MIRIM_FULL'), ('MIRIFUSHORT', 'MIRIM_FULL')] +fgs_list = ['FGS1_FULL', 'FGS2_FULL'] + + +@pytest.mark.parametrize("instrument,expected_list", [("nircam", nrc_list), ("niriss", nis_list), + ("nirspec", nrs_list), ("miri", miri_list), ("fgs", fgs_list)]) +def test_get_possible_apertures(instrument, expected_list): + """Make sure the correct apertures are returned for the given instrument + """ + badpix = bad_pixel_monitor.BadPixels() + badpix.instrument = instrument + ap_list = badpix.get_possible_apertures() + assert ap_list == expected_list + + +def test_identify_tables(): + """Be sure the correct database tables are identified + """ + badpix = bad_pixel_monitor.BadPixels() + badpix.instrument = 'nircam' + badpix.identify_tables() + assert badpix.query_table == eval('NIRCamBadPixelQueryHistory') + assert badpix.pixel_table == eval('NIRCamBadPixelStats') + + +def test_make_crds_parameter_dict(): + """Test that the dictionary to be used for CRDS queries is properly + created + """ + badpix = bad_pixel_monitor.BadPixels() + badpix.instrument = 'nircam' + badpix.detector = 'nrcalong' + params = badpix.make_crds_parameter_dict() + assert params['INSTRUME'] == 'NIRCAM' + assert params['DETECTOR'] == 'NRCALONG' + assert params['CHANNEL'] == 'LONG' From 72924705077d3b90109f216ec0193198f8e57b9e Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 24 Jul 2020 16:23:50 -0400 Subject: [PATCH 003/227] Add newly-created apertures from pysiaf 0.9.0 --- .../dark_monitor_file_thresholds.txt | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor_file_thresholds.txt b/jwql/instrument_monitors/common_monitors/dark_monitor_file_thresholds.txt index ddb83e174..d423bbbdb 100644 --- a/jwql/instrument_monitors/common_monitors/dark_monitor_file_thresholds.txt +++ b/jwql/instrument_monitors/common_monitors/dark_monitor_file_thresholds.txt @@ -79,6 +79,7 @@ nircam NRCA3_GRISMTS256 30 nircam NRCA3_GRISMTS128 30 nircam NRCA3_GRISMTS64 30 nircam NRCA5_TAGRISMTS32 30 +nircam NRCA5_TAGRISMTS32_F405N 30 nircam NRCA5_TAGRISMTS_SCI_F322W2 30 nircam NRCA5_TAGRISMTS_SCI_F444W 30 nircam NRCA3_DHSPIL 30 @@ -188,6 +189,22 @@ nircam NRCA5_FULL_MASKLWB_F430M 10 nircam NRCA5_FULL_MASKLWB_F460M 10 nircam NRCA5_FULL_MASKLWB_F480M 10 nircam NRCA5_FULL_MASKLWB_F444W 10 +nircam NRCA2_FULL_WEDGE_RND 10 +nircam NRCA4_FULL_WEDGE_BAR 10 +nircam NRCA5_FULL_WEDGE_RND 10 +nircam NRCA5_FULL_WEDGE_BAR 10 +nircam NRCA2_FULL_TAMASK210R 10 +nircam NRCA5_FULL_TAMASK335R 10 +nircam NRCA5_FULL_TAMASK430R 10 +nircam NRCA4_FULL_TAMASKSWB 10 +nircam NRCA5_FULL_TAMASKLWB 10 +nircam NRCA5_FULL_TAMASKLWBL 10 +nircam NRCA4_FULL_TAMASKSWBS 10 +nircam NRCA2_FULL_FSTAMASK210R 10 +nircam NRCA4_FULL_FSTAMASKSWB 10 +nircam NRCA5_FULL_FSTAMASKLWB 10 +nircam NRCA5_FULL_FSTAMASK335R 10 +nircam NRCA5_FULL_FSTAMASK430R 10 niriss NIS_CEN_OSS 10 niriss NIS_CEN 10 niriss NIS_AMI1 30 @@ -220,6 +237,7 @@ niriss NIS_FP4MIMF 30 niriss NIS_FP5MIMF 30 niriss NIS_AMIFULL 10 niriss NIS_SOSSFULL 10 +niriss NIS_WFSS 10 miri MIRIM_FULL_OSS 10 miri MIRIM_FULL 10 miri MIRIM_ILLUM 30 @@ -603,4 +621,4 @@ fgs FGS2_FP1MIMF 30 fgs FGS2_FP2MIMF 30 fgs FGS2_FP3MIMF 30 fgs FGS2_FP4MIMF 30 -fgs FGS2_FP5MIMF 30 +fgs FGS2_FP5MIMF 30 \ No newline at end of file From 7d10f9d89019474bf7481d7e957f8f21ffd49e00 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 24 Jul 2020 15:30:08 -0700 Subject: [PATCH 004/227] Added Bias Monitor, Bad Pixel Monitor, and Readnoise Monitor --- docs/source/common_monitors.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/source/common_monitors.rst b/docs/source/common_monitors.rst index 21d98a7d7..bd5ec2fbb 100644 --- a/docs/source/common_monitors.rst +++ b/docs/source/common_monitors.rst @@ -5,5 +5,23 @@ common_monitors dark_monitor.py --------------- .. automodule:: jwql.instrument_monitors.common_monitors.dark_monitor + :members: + :undoc-members: + +bias_monitor.py +--------------- +.. automodule:: jwql.instrument_monitors.common_monitors.bias_monitor + :members: + :undoc-members: + +bad_pixel_monitor.py +--------------- +.. automodule:: jwql.instrument_monitors.common_monitors.bad_pixel_monitor + :members: + :undoc-members: + +readnoise_monitor.py +--------------- +.. automodule:: jwql.instrument_monitors.common_monitors.readnoise_monitor :members: :undoc-members: \ No newline at end of file From a3f204b49e17de2e34d1d5f72ee33c49691f883c Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 24 Jul 2020 15:33:11 -0700 Subject: [PATCH 005/227] Added sphinx docs for Bias Monitor, Bad Pixel Monitor, and Readnoise Monitor --- docs/source/common_monitors.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/common_monitors.rst b/docs/source/common_monitors.rst index bd5ec2fbb..9d0253d6e 100644 --- a/docs/source/common_monitors.rst +++ b/docs/source/common_monitors.rst @@ -15,13 +15,13 @@ bias_monitor.py :undoc-members: bad_pixel_monitor.py ---------------- +-------------------- .. automodule:: jwql.instrument_monitors.common_monitors.bad_pixel_monitor :members: :undoc-members: readnoise_monitor.py ---------------- +-------------------- .. automodule:: jwql.instrument_monitors.common_monitors.readnoise_monitor :members: :undoc-members: \ No newline at end of file From 84cc16ab691b0383b381298fafa649ec306e549a Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Fri, 24 Jul 2020 23:02:29 -0400 Subject: [PATCH 006/227] Functional env file for 3.7 --- environment_python_3_7.yml | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 environment_python_3_7.yml diff --git a/environment_python_3_7.yml b/environment_python_3_7.yml new file mode 100644 index 000000000..22c2dc24e --- /dev/null +++ b/environment_python_3_7.yml @@ -0,0 +1,40 @@ +channels: +- defaults +- http://ssb.stsci.edu/astroconda +dependencies: +- astroquery=0.4 +- bokeh>=1.0,<1.4 +- django=3.0.3 +- flake8=3.8.3 +- inflection=0.3.1 +- ipython=7.16.1 +- jinja2=2.11.2 +- jsonschema=3.2.0 +- matplotlib=3.2.2 +- nodejs=10.13.0 +- numpy=1.18.5 +- numpydoc=1.1.0 +- pandas=1.0.5 +- pip=20.1.1 +- postgresql=12.2 +- psycopg2=2.8.5 +- pysiaf=0.9.0 +- python=3.7.8 +- pytest=5.4.3 +- pytest-cov=2.10.0 +- scipy=1.5.0 +- setuptools=49.2.0 +- sphinx=3.1.2 +- sqlalchemy=1.3.18 +- twine=2.0.0 +- pip: + - asdf==2.7.0 + - astropy==4.0.1.post1 + - authlib==0.14.3 + - codecov==2.1.8 + - crds==7.5.0.0 + - jwedb==0.0.6 + - pysqlite3==0.4.3 + - stsci_rtd_theme==0.0.2 + - git+https://github.com/spacetelescope/jwst@0.16.2 + - git+https://github.com/spacetelescope/jwst_reffiles From 803a4d7b176e20acc9d436125e4ac4bf1ffa795f Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Sat, 25 Jul 2020 09:25:18 -0400 Subject: [PATCH 007/227] Add python 3.7 to testing matrix --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6b86ed368..7fc95a95d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,7 @@ if (utils.scm_checkout()) return // Establish OS and Python version variables for the matrix matrix_os = ["linux-stable"] // (Note that Jenkins can only be run with Linux, not MacOSX/Windows) -matrix_python = ["3.6"] +matrix_python = ["3.6", "3.7"] // Set up the matrix of builds matrix = [] From 011a791fc4a4cc50b5368dedb1490ee3c29acd4b Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Mon, 27 Jul 2020 11:32:25 -0400 Subject: [PATCH 008/227] Back down to python version 3.7.7 --- environment_python_3_7.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_python_3_7.yml b/environment_python_3_7.yml index 22c2dc24e..3d6196634 100644 --- a/environment_python_3_7.yml +++ b/environment_python_3_7.yml @@ -19,7 +19,7 @@ dependencies: - postgresql=12.2 - psycopg2=2.8.5 - pysiaf=0.9.0 -- python=3.7.8 +- python=3.7.7 - pytest=5.4.3 - pytest-cov=2.10.0 - scipy=1.5.0 From 5470c8501e781fbaf45df1886cf91f2289fc57b0 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 28 Jul 2020 09:31:29 -0700 Subject: [PATCH 009/227] Added bias, readnoise, bad pixel monitor docs --- docs/source/common_monitors.rst | 18 ++++++++++++++++++ jwql/jwql_monitors/monitor_filesystem.py | 1 - jwql/website/apps/jwql/api_views.py | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/source/common_monitors.rst b/docs/source/common_monitors.rst index 21d98a7d7..a667a7fc6 100644 --- a/docs/source/common_monitors.rst +++ b/docs/source/common_monitors.rst @@ -5,5 +5,23 @@ common_monitors dark_monitor.py --------------- .. automodule:: jwql.instrument_monitors.common_monitors.dark_monitor + :members: + :undoc-members: + +bias_monitor.py +--------------- +.. automodule:: jwql.instrument_monitors.common_monitors.bias_monitor + :members: + :undoc-members: + +readnoise_monitor.py +-------------------- +.. automodule:: jwql.instrument_monitors.common_monitors.readnoise_monitor + :members: + :undoc-members: + +bad_pixel_monitor.py +-------------------- +.. automodule:: jwql.instrument_monitors.common_monitors.bad_pixel_monitor :members: :undoc-members: \ No newline at end of file diff --git a/jwql/jwql_monitors/monitor_filesystem.py b/jwql/jwql_monitors/monitor_filesystem.py index e394c5f70..83b27db19 100755 --- a/jwql/jwql_monitors/monitor_filesystem.py +++ b/jwql/jwql_monitors/monitor_filesystem.py @@ -16,7 +16,6 @@ --- This module is intended to be executed from the command line: - :: python monitor_filesystem.py diff --git a/jwql/website/apps/jwql/api_views.py b/jwql/website/apps/jwql/api_views.py index b74b74129..404263f74 100644 --- a/jwql/website/apps/jwql/api_views.py +++ b/jwql/website/apps/jwql/api_views.py @@ -37,7 +37,7 @@ from django.urls import path from . import api_views urlpatterns = [path('web/path/to/view/', - api_views.view_name, name='view_name')] + api_views.view_name, name='view_name')] References ---------- From 24b20535703426e231abaf13dd84b6a765dfde4a Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 28 Jul 2020 09:34:48 -0700 Subject: [PATCH 010/227] Added bias, readnoise, bad pixel monitor docs --- docs/source/common_monitors.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/common_monitors.rst b/docs/source/common_monitors.rst index 9d0253d6e..b67bb5d2e 100644 --- a/docs/source/common_monitors.rst +++ b/docs/source/common_monitors.rst @@ -2,9 +2,9 @@ common_monitors *************** -dark_monitor.py ---------------- -.. automodule:: jwql.instrument_monitors.common_monitors.dark_monitor +bad_pixel_monitor.py +-------------------- +.. automodule:: jwql.instrument_monitors.common_monitors.bad_pixel_monitor :members: :undoc-members: @@ -14,9 +14,9 @@ bias_monitor.py :members: :undoc-members: -bad_pixel_monitor.py --------------------- -.. automodule:: jwql.instrument_monitors.common_monitors.bad_pixel_monitor +dark_monitor.py +--------------- +.. automodule:: jwql.instrument_monitors.common_monitors.dark_monitor :members: :undoc-members: From e07af3359602153675d0966298137b7070b0d968 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Jul 2020 12:59:13 -0700 Subject: [PATCH 011/227] testing removal of jwst_reffiles from requirements.txt for read the docs build --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 92cd248a3..0710ddd25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,5 +26,4 @@ sphinx==3.0.4 sqlalchemy==1.3.17 stsci_rtd_theme==0.0.2 twine==3.1.1 -git+https://github.com/spacetelescope/jwst@stable -git+https://github.com/spacetelescope/jwst_reffiles +git+https://github.com/spacetelescope/jwst@stable \ No newline at end of file From 1caf339cf6070cb7fe55fb9f27312e8cc22a3aa6 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Jul 2020 13:10:36 -0700 Subject: [PATCH 012/227] testing removal of nodejs from requirements.txt for read the docs build --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0710ddd25..27a0e1f39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,6 @@ jinja2==2.11.2 jsonschema==3.2.0 jwedb>=0.0.3 matplotlib==3.2.1 -nodejs==10.13.0 numpy==1.18.4 numpydoc==1.0.0 pandas==1.0.4 @@ -26,4 +25,5 @@ sphinx==3.0.4 sqlalchemy==1.3.17 stsci_rtd_theme==0.0.2 twine==3.1.1 -git+https://github.com/spacetelescope/jwst@stable \ No newline at end of file +git+https://github.com/spacetelescope/jwst@stable +git+https://github.com/spacetelescope/jwst_reffiles \ No newline at end of file From 8e0edb49eceacc3847ccf654ea647f03df1e03bd Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Jul 2020 13:44:56 -0700 Subject: [PATCH 013/227] added pygments requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 27a0e1f39..1e8ce610c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ numpy==1.18.4 numpydoc==1.0.0 pandas==1.0.4 psycopg2==2.8.5 +pygments==2.5.1 pysiaf==0.7.1 pytest==5.4.2 pytest-cov==2.9.0 From 04c4413f35015940b1a7da3617332dbca7aa1d0c Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Jul 2020 13:58:20 -0700 Subject: [PATCH 014/227] testing readthedocs --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1e8ce610c..27a0e1f39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ numpy==1.18.4 numpydoc==1.0.0 pandas==1.0.4 psycopg2==2.8.5 -pygments==2.5.1 pysiaf==0.7.1 pytest==5.4.2 pytest-cov==2.9.0 From 55b46d478223169a96755faa12ee9396c06c8fe1 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Jul 2020 14:01:04 -0700 Subject: [PATCH 015/227] updated requirements.txt --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 27a0e1f39..4d2a7ca89 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ numpy==1.18.4 numpydoc==1.0.0 pandas==1.0.4 psycopg2==2.8.5 +pygments==2.5.1 pysiaf==0.7.1 pytest==5.4.2 pytest-cov==2.9.0 @@ -25,5 +26,4 @@ sphinx==3.0.4 sqlalchemy==1.3.17 stsci_rtd_theme==0.0.2 twine==3.1.1 -git+https://github.com/spacetelescope/jwst@stable -git+https://github.com/spacetelescope/jwst_reffiles \ No newline at end of file +git+https://github.com/spacetelescope/jwst@stable \ No newline at end of file From a7e91669254591f0ec558d392d278c30ca6f93e0 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Jul 2020 14:37:36 -0700 Subject: [PATCH 016/227] took out pygments requirement --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4d2a7ca89..27a0e1f39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,6 @@ numpy==1.18.4 numpydoc==1.0.0 pandas==1.0.4 psycopg2==2.8.5 -pygments==2.5.1 pysiaf==0.7.1 pytest==5.4.2 pytest-cov==2.9.0 @@ -26,4 +25,5 @@ sphinx==3.0.4 sqlalchemy==1.3.17 stsci_rtd_theme==0.0.2 twine==3.1.1 -git+https://github.com/spacetelescope/jwst@stable \ No newline at end of file +git+https://github.com/spacetelescope/jwst@stable +git+https://github.com/spacetelescope/jwst_reffiles \ No newline at end of file From c10ae73dd3e75277cf186d17d5ea16bdc46867a5 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Jul 2020 14:48:10 -0700 Subject: [PATCH 017/227] testing requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 27a0e1f39..07f90010f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ matplotlib==3.2.1 numpy==1.18.4 numpydoc==1.0.0 pandas==1.0.4 +pysegments==2.5.1 psycopg2==2.8.5 pysiaf==0.7.1 pytest==5.4.2 From 41d7e63b8a950f6efc3ebf8e9a26a2f402c894ab Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Jul 2020 14:55:55 -0700 Subject: [PATCH 018/227] adjusted requirements --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 07f90010f..27a0e1f39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,6 @@ matplotlib==3.2.1 numpy==1.18.4 numpydoc==1.0.0 pandas==1.0.4 -pysegments==2.5.1 psycopg2==2.8.5 pysiaf==0.7.1 pytest==5.4.2 From ee6b1abd1e2734661e913ddfe9eaeacdccda9c51 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:04 -0400 Subject: [PATCH 019/227] Update asdf from 2.6.0 to 2.7.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 92cd248a3..117dde5f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -asdf==2.6.0 +asdf==2.7.0 astropy==4.0.1.post1 astroquery==0.4 authlib==0.14.3 From 02eb1c8fdf96a7ac152022c8eae84d9fae4f162d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:05 -0400 Subject: [PATCH 020/227] Update astroquery from 0.4 to 0.4.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 117dde5f0..d621e9010 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ asdf==2.7.0 astropy==4.0.1.post1 -astroquery==0.4 +astroquery==0.4.1 authlib==0.14.3 bokeh==1.3.4 codecov==2.1.3 From 0d9d8fbce3ffbf9544110c4eaabb0bed062b6709 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:06 -0400 Subject: [PATCH 021/227] Update bokeh from 1.3.4 to 2.1.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d621e9010..1af3008ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ asdf==2.7.0 astropy==4.0.1.post1 astroquery==0.4.1 authlib==0.14.3 -bokeh==1.3.4 +bokeh==2.1.1 codecov==2.1.3 crds==7.5.0.0 django==2.2.5 From b381ef1fa2b0ab2c736098d9a69c6e34c618e903 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:07 -0400 Subject: [PATCH 022/227] Update codecov from 2.1.3 to 2.1.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1af3008ea..53866f92b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ astropy==4.0.1.post1 astroquery==0.4.1 authlib==0.14.3 bokeh==2.1.1 -codecov==2.1.3 +codecov==2.1.8 crds==7.5.0.0 django==2.2.5 flake8==3.8.2 From 54bf4f193630c0c888b0ca1fd3ba2d2414720d0b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:08 -0400 Subject: [PATCH 023/227] Update django from 2.2.5 to 3.0.8 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53866f92b..5df3b4668 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ authlib==0.14.3 bokeh==2.1.1 codecov==2.1.8 crds==7.5.0.0 -django==2.2.5 +django==3.0.8 flake8==3.8.2 inflection==0.4.0 ipython==7.15.0 From 46f92f10b13a8024916c1c9dfec7204759e12ee4 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:09 -0400 Subject: [PATCH 024/227] Update flake8 from 3.8.2 to 3.8.3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5df3b4668..53db95244 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ bokeh==2.1.1 codecov==2.1.8 crds==7.5.0.0 django==3.0.8 -flake8==3.8.2 +flake8==3.8.3 inflection==0.4.0 ipython==7.15.0 jinja2==2.11.2 From da11007fd9d2935219278f71c9edadf3bd84d7d8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:10 -0400 Subject: [PATCH 025/227] Update inflection from 0.4.0 to 0.5.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53db95244..3dde0c85c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ codecov==2.1.8 crds==7.5.0.0 django==3.0.8 flake8==3.8.3 -inflection==0.4.0 +inflection==0.5.0 ipython==7.15.0 jinja2==2.11.2 jsonschema==3.2.0 From 896e7af4659209ddf485c6a54cc820bfee662b0a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:10 -0400 Subject: [PATCH 026/227] Update ipython from 7.15.0 to 7.17.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3dde0c85c..b6d8396c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ crds==7.5.0.0 django==3.0.8 flake8==3.8.3 inflection==0.5.0 -ipython==7.15.0 +ipython==7.17.0 jinja2==2.11.2 jsonschema==3.2.0 jwedb>=0.0.3 From b8d24265f770df9e2b3f95cc58be4baeac697e74 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:11 -0400 Subject: [PATCH 027/227] Update matplotlib from 3.2.1 to 3.3.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b6d8396c2..59cc829e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ ipython==7.17.0 jinja2==2.11.2 jsonschema==3.2.0 jwedb>=0.0.3 -matplotlib==3.2.1 +matplotlib==3.3.0 nodejs==10.13.0 numpy==1.18.4 numpydoc==1.0.0 From 2801282807e3eee26846341e09158994f5ee077d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:12 -0400 Subject: [PATCH 028/227] Update numpy from 1.18.4 to 1.19.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 59cc829e5..4764f4593 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ jsonschema==3.2.0 jwedb>=0.0.3 matplotlib==3.3.0 nodejs==10.13.0 -numpy==1.18.4 +numpy==1.19.1 numpydoc==1.0.0 pandas==1.0.4 psycopg2==2.8.5 From 027502cf43185022b46f546255cc64f81d9cc8cf Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:13 -0400 Subject: [PATCH 029/227] Update numpydoc from 1.0.0 to 1.1.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4764f4593..334da8662 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ jwedb>=0.0.3 matplotlib==3.3.0 nodejs==10.13.0 numpy==1.19.1 -numpydoc==1.0.0 +numpydoc==1.1.0 pandas==1.0.4 psycopg2==2.8.5 pysiaf==0.7.1 From ec7811523beeff1a28c6898ccb0ee6fef2d2c587 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:14 -0400 Subject: [PATCH 030/227] Update pandas from 1.0.4 to 1.1.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 334da8662..27f108e07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ matplotlib==3.3.0 nodejs==10.13.0 numpy==1.19.1 numpydoc==1.1.0 -pandas==1.0.4 +pandas==1.1.0 psycopg2==2.8.5 pysiaf==0.7.1 pytest==5.4.2 From 25cb96213022a9b86bdce757f72ed89d941e31d8 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:15 -0400 Subject: [PATCH 031/227] Update pysiaf from 0.7.1 to 0.9.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 27f108e07..ee9a020ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ numpy==1.19.1 numpydoc==1.1.0 pandas==1.1.0 psycopg2==2.8.5 -pysiaf==0.7.1 +pysiaf==0.9.0 pytest==5.4.2 pytest-cov==2.9.0 scipy==1.4.1 From a6ccf07eae5ccdf8d4fd3c53f00c5bc95989722b Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:16 -0400 Subject: [PATCH 032/227] Update pytest from 5.4.2 to 6.0.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ee9a020ae..97c2e8fd6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ numpydoc==1.1.0 pandas==1.1.0 psycopg2==2.8.5 pysiaf==0.9.0 -pytest==5.4.2 +pytest==6.0.1 pytest-cov==2.9.0 scipy==1.4.1 sphinx==3.0.4 From db61481f6f7548cdc32aef8e58212061e7051f05 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:17 -0400 Subject: [PATCH 033/227] Update pytest-cov from 2.9.0 to 2.10.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 97c2e8fd6..910dfb3df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ pandas==1.1.0 psycopg2==2.8.5 pysiaf==0.9.0 pytest==6.0.1 -pytest-cov==2.9.0 +pytest-cov==2.10.0 scipy==1.4.1 sphinx==3.0.4 sqlalchemy==1.3.17 From 5a6ce6143d7e7897604fa17bd7c8a84ae5f53959 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:18 -0400 Subject: [PATCH 034/227] Update scipy from 1.4.1 to 1.5.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 910dfb3df..4eda1fd83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ psycopg2==2.8.5 pysiaf==0.9.0 pytest==6.0.1 pytest-cov==2.10.0 -scipy==1.4.1 +scipy==1.5.2 sphinx==3.0.4 sqlalchemy==1.3.17 stsci_rtd_theme==0.0.2 From e1b790b6553c44a1d34807174ba23345f66f2c71 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:19 -0400 Subject: [PATCH 035/227] Update sphinx from 3.0.4 to 3.1.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4eda1fd83..d16764090 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ pysiaf==0.9.0 pytest==6.0.1 pytest-cov==2.10.0 scipy==1.5.2 -sphinx==3.0.4 +sphinx==3.1.2 sqlalchemy==1.3.17 stsci_rtd_theme==0.0.2 twine==3.1.1 From 0ac3f2537fb7de606c6a449a79a0cf1bd10c337a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:19 -0400 Subject: [PATCH 036/227] Update sqlalchemy from 1.3.17 to 1.3.18 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d16764090..1a0f9f487 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ pytest==6.0.1 pytest-cov==2.10.0 scipy==1.5.2 sphinx==3.1.2 -sqlalchemy==1.3.17 +sqlalchemy==1.3.18 stsci_rtd_theme==0.0.2 twine==3.1.1 git+https://github.com/spacetelescope/jwst@stable From 957b89377a54269c263109f4d7ac8e2a656c12ae Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Sat, 1 Aug 2020 14:31:20 -0400 Subject: [PATCH 037/227] Update twine from 3.1.1 to 3.2.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1a0f9f487..28c17835c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,6 @@ scipy==1.5.2 sphinx==3.1.2 sqlalchemy==1.3.18 stsci_rtd_theme==0.0.2 -twine==3.1.1 +twine==3.2.0 git+https://github.com/spacetelescope/jwst@stable git+https://github.com/spacetelescope/jwst_reffiles From 50b275b7d47174de65e942da319226a75c45a214 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 5 Aug 2020 15:49:36 -0700 Subject: [PATCH 038/227] updates to readthedocs and sphinx documentation of monitors --- environment_python_3_6.yml | 2 +- requirements.txt | 1 + rtd_requirements.txt | 3 +++ setup.py | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 rtd_requirements.txt diff --git a/environment_python_3_6.yml b/environment_python_3_6.yml index 447e0e87f..5d2d19073 100644 --- a/environment_python_3_6.yml +++ b/environment_python_3_6.yml @@ -37,4 +37,4 @@ dependencies: - pysqlite3==0.2.2 - stsci_rtd_theme==0.0.2 - git+https://github.com/spacetelescope/jwst@0.16.2 - - git+https://github.com/spacetelescope/jwst_reffiles + - git+https://github.com/spacetelescope/jwst_reffiles \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 27a0e1f39..ccc038d9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ jinja2==2.11.2 jsonschema==3.2.0 jwedb>=0.0.3 matplotlib==3.2.1 +nodejs==10.13.0 numpy==1.18.4 numpydoc==1.0.0 pandas==1.0.4 diff --git a/rtd_requirements.txt b/rtd_requirements.txt new file mode 100644 index 000000000..00f5c6bf3 --- /dev/null +++ b/rtd_requirements.txt @@ -0,0 +1,3 @@ +pygments==2.5.1 +pytest==5.4.2 +git+https://github.com/spacetelescope/jwst@stable \ No newline at end of file diff --git a/setup.py b/setup.py index 382030c3a..98a87412e 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ 'inflection', 'ipython', 'jinja2', - 'jsonschema==2.6.0', + 'jsonschema>=2.6.0', 'jwedb>=0.0.3', 'matplotlib', 'nodejs', From 6d6cf79f3475a2ec370c8d987126dd2c91a46ba7 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 6 Aug 2020 13:49:24 -0400 Subject: [PATCH 039/227] Adding start of API landing page. --- jwql/website/apps/jwql/templates/base.html | 3 +++ jwql/website/apps/jwql/urls.py | 1 + jwql/website/apps/jwql/views.py | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/jwql/website/apps/jwql/templates/base.html b/jwql/website/apps/jwql/templates/base.html index 592a59b72..b9bfd92ec 100644 --- a/jwql/website/apps/jwql/templates/base.html +++ b/jwql/website/apps/jwql/templates/base.html @@ -134,6 +134,9 @@
  • JWQLDB(current) +
    • +
  • + API(current)
  • Documentation(current) diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 439d0c583..ef19ac736 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -73,6 +73,7 @@ # Main site views path('about/', views.about, name='about'), + path('api/', views.api_landing, name='api'), path('dashboard/', views.dashboard, name='dashboard'), path('edb/', views.engineering_database, name='edb'), path('query_anomaly/', views.query_anomaly, name='query_anomaly'), diff --git a/jwql/website/apps/jwql/views.py b/jwql/website/apps/jwql/views.py index bda007e2f..1ca2c66ad 100644 --- a/jwql/website/apps/jwql/views.py +++ b/jwql/website/apps/jwql/views.py @@ -176,6 +176,26 @@ def about(request): return render(request, template, context) +def api_landing(request): + """Generate the ``api`` page + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage + + Returns + ------- + HttpResponse object + Outgoing response sent to the webpage + """ + + template = 'api_landing.html' + context = {'inst': ''} + + return render(request, template, context) + + @auth_required def archived_proposals(request, user, inst): """Generate the page listing all archived proposals in the database From a7f405915ffaf379ed2095f39a3365652c900676 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Mon, 10 Aug 2020 16:25:21 -0400 Subject: [PATCH 040/227] Adding documentation, examples, and list of services offered --- .../apps/jwql/templates/api_landing.html | 152 ++++++++++++++++++ jwql/website/apps/jwql/templates/base.html | 6 + 2 files changed, 158 insertions(+) create mode 100644 jwql/website/apps/jwql/templates/api_landing.html diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html new file mode 100644 index 000000000..9a57b5e30 --- /dev/null +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -0,0 +1,152 @@ +{% extends "base.html" %} + +{% block preamble %} + + API - JWQL + +{% endblock %} + +{% block content %} + +
    +

    Explore the JWQL API

    +

    +

    +

    +
    + +
    +

    What is the JWQL API?

    +

    + JWQL allows users to access the data available through our programmatic Application Programming Interface (API). In short, an API + accepts requests for services from a system, gathers the items in the request, and returns a response to the user. A popular example used to describe the + functionality of an API is describing the role of a waiter in a restaurant. When you enter a restaurant and sit at your table, you are typically + provided a menu containing all of the drink and food options. But how do you communicate your food and drink order to + the kitchen staff? This is where the waiter (API) comes in, you provide the waiter with your order (request), they will communicate + your order to the kitchen staff (system), and bring your food and drinks when they are prepared (the response). The waiter (API) disguises + all of the steps that are necessary to getting your final product delivered (cooking the food, counting stock, knowing recipes, etc.) but, is the + link to the user and the response they wish to obtain from a system. +

    +
    +
    + +
    +

    Examples

    +

    + Here we showcase a few examples of a few of the APIs that we offer. To see a full list of supported APIs click here. +


    +
    Obtaining Filenames Associated with a Rootname
    +

    + A user is interested in all of the filenames associated with a specific JWST rootname. +

    + 
    +                
+                    from urllib import request
+                    import json
+
+                    # Build the URL https://dljwql.stsci.edu/api/roontname/service/
+                    url = 'https://dljwql.stsci.edu/api/jw93025001001_02102_00001_nrca2/filenames/' 
+                    r = request.urlopen(url) 
+                    filenames = json.loads(r.read().decode())
+
+                    print(filenames)
+
+                    {'filenames': ['jw93025001001_02102_00001_nrca2_calints.fits',
+                                    'jw93025001001_02102_00001_nrca2_rate.fits',
+                                    'jw93025001001_02102_00001_nrca2_rateints.fits',
+                                    'jw93025001001_02102_00001_nrca2_trapsfilled.fits',
+                                    'jw93025001001_02102_00001_nrca2_uncal.fits']}
+                    
+
    + +
    Obtaining Filenames Associated with Proposal
    +

    + A user is interested in all of the thumbnail associated with the Fine Guider Sensors (FGS). +

    + 
    +                    
+                    from urllib import request
+                    import json
+
+                    # Build the URL https://dljwql.stsci.edu/api/instrument/service/
+                    url = 'https://dljwql.stsci.edu/api/fgs/thumbnails/'
+                    r = request.urlopen(url) 
+                    thumbnails = json.loads(r.read().decode())
+
+                    print(thumbnails)
+
+                    {'thumbnails': ['jw97012001003_02101_00001_guider1_trapsfilled_integ2.thumb',
+                                    'jw97012001001_02101_00001_guider1_rateints_integ1.thumb',
+                                    'jw97012001002_02101_00001_guider1_trapsfilled_integ2.thumb',
+                                    'jw97012001004_02101_00001_guider1_rateints_integ0.thumb',
+                                    'jw97012001003_02101_00001_guider1_rateints_integ0.thumb',
+                                    'jw97012001001_02101_00001_guider1_cal_integ0.thumb',
+                                    'jw97012001001_02101_00001_guider1_rateints_integ0.thumb',
+                                    'jw97012001004_02101_00001_guider1_rate_integ0.thumb',
+                                    'jw97012001004_02101_00001_guider1_trapsfilled_integ0.thumb',
+                                    'jw97012001001_02101_00001_guider1_trapsfilled_integ1.thumb',
+                                    'jw97012001004_02101_00001_guider1_trapsfilled_integ1.thumb',
+                                    ...,
+                                    ...]}
+                        
+
    + +
    Obtaining Filenames Associated with Proposal
    +

    + A user is interested in all of the thumbnail associated with the Fine Guider Sensors (FGS). +

    + 
    +                        
+                        from urllib import request
+                        import json
+    
+                        # Build the URL https://dljwql.stsci.edu/api/proposal/service/
+                        url = 'https://dljwql.stsci.edu/api/86600/filenames/'
+                        r = request.urlopen(url) 
+                        filenames = json.loads(r.read().decode())
+    
+                        print(filenames)
+    
+                        {'filenames': ['jw86600001001_02101_00001_guider1_cal.fits',
+                                       'jw86600001001_02101_00001_guider1_rate.fits',
+                                       'jw86600001001_02101_00001_guider1_trapsfilled.fits',
+                                       'jw86600001001_02101_00001_guider1_uncal.fits',
+                                       'jw86600001001_02101_00002_guider1_cal.fits',
+                                       'jw86600001001_02101_00002_guider1_rate.fits',
+                                       'jw86600001001_02101_00002_guider1_trapsfilled.fits',
+                                       'jw86600001001_02101_00002_guider1_uncal.fits',
+                                       'jw86600001001_02101_00003_guider1_cal.fits',
+                                       'jw86600001001_02101_00003_guider1_rate.fits',
+                                       'jw86600001001_02101_00003_guider1_trapsfilled.fits',
+                                       'jw86600001001_02101_00003_guider1_uncal.fits',
+                                       'jw86600001001_02101_00004_guider1_cal.fits',
+                                       'jw86600001001_02101_00004_guider1_rate.fits',
+                                       'jw86600001001_02101_00004_guider1_trapsfilled.fits',
+                                       ...,
+                                       ...]}
+                        
+
    +
    + +
    +

    List of Available Services

    +
      +
    • All Proposals
      • +
    • Instrument Proposals
      • +
    • Preview Images by Instrument
      • +
    • Thumbnails by Instrument
      • +
    • Filenames by Proposal
      • +
    • Preview Images by Proposal
      • +
    • Filenames by Rootname
      • +
    • Preview Images by Rootname
      • +
    • Thumbnails by Rootname
      • +
    +
    +
    + +{% endblock %} \ No newline at end of file diff --git a/jwql/website/apps/jwql/templates/base.html b/jwql/website/apps/jwql/templates/base.html index b9bfd92ec..823a1692e 100644 --- a/jwql/website/apps/jwql/templates/base.html +++ b/jwql/website/apps/jwql/templates/base.html @@ -229,5 +229,11 @@ + + + + + From 619ce4b537671233699f01fee6d107a41f18bcd3 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 11 Aug 2020 11:57:04 -0400 Subject: [PATCH 041/227] Fix failing test due to Travis error --- jwql/instrument_monitors/common_monitors/dark_monitor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py index 77f33cf98..f2e4dc0a5 100755 --- a/jwql/instrument_monitors/common_monitors/dark_monitor.py +++ b/jwql/instrument_monitors/common_monitors/dark_monitor.py @@ -140,8 +140,9 @@ def mast_query_darks(instrument, aperture, start_date, end_date): query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS, add_filters=parameters, return_data=True, caom=False) - if len(query['data']) > 0: - query_results.extend(query['data']) + if 'data' in query.keys(): + if len(query['data']) > 0: + query_results.extend(query['data']) return query_results From 5b4c90359886b06bef9e16f1531196e29cc95cb9 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 11 Aug 2020 13:47:15 -0400 Subject: [PATCH 042/227] Make setup.py and req.txt consistent --- requirements.txt | 36 ++++++++++++++++++------------------ setup.py | 4 ++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/requirements.txt b/requirements.txt index 92cd248a3..f8c34cc82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,30 +1,30 @@ -asdf==2.6.0 +asdf==2.7.0 astropy==4.0.1.post1 astroquery==0.4 authlib==0.14.3 bokeh==1.3.4 -codecov==2.1.3 +codecov==2.1.8 crds==7.5.0.0 django==2.2.5 -flake8==3.8.2 -inflection==0.4.0 -ipython==7.15.0 +flake8==3.8.3 +inflection==0.3.1 +ipython==7.16.1 jinja2==2.11.2 jsonschema==3.2.0 -jwedb>=0.0.3 -matplotlib==3.2.1 +jwedb>=0.0.6 +matplotlib==3.2.2 nodejs==10.13.0 -numpy==1.18.4 -numpydoc==1.0.0 -pandas==1.0.4 +numpy==1.18.5 +numpydoc==1.1.0 +pandas==1.0.5 psycopg2==2.8.5 -pysiaf==0.7.1 -pytest==5.4.2 -pytest-cov==2.9.0 -scipy==1.4.1 -sphinx==3.0.4 -sqlalchemy==1.3.17 +pysiaf==0.9.0 +pytest==5.4.3 +pytest-cov==2.10.0 +scipy==1.5.0 +sphinx==3.1.2 +sqlalchemy==1.3.18 stsci_rtd_theme==0.0.2 -twine==3.1.1 -git+https://github.com/spacetelescope/jwst@stable +twine==2.0.0 +git+https://github.com/spacetelescope/jwst@0.16.2 git+https://github.com/spacetelescope/jwst_reffiles diff --git a/setup.py b/setup.py index 382030c3a..b1967b7f1 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ DESCRIPTION = 'The James Webb Space Telescope Quicklook Project' -DEPENDENCY_LINKS = ['git+https://github.com/spacetelescope/jwst#0.16.2', +DEPENDENCY_LINKS = ['git+https://github.com/spacetelescope/jwst@0.16.2', 'git+https://github.com/spacetelescope/jwst_reffiles' ] REQUIRES = [ @@ -25,7 +25,7 @@ 'inflection', 'ipython', 'jinja2', - 'jsonschema==2.6.0', + 'jsonschema', 'jwedb>=0.0.3', 'matplotlib', 'nodejs', From 10bbb3278424ceda3f5c45b605761c8208c9c6dd Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Tue, 11 Aug 2020 17:31:48 -0400 Subject: [PATCH 043/227] Create all necessary files using dark monitor files as a guide --- .../jwql/monitor_pages/badpixel_monitor.py | 44 ++++ .../monitor_pages/monitor_badpixel_bokeh.py | 214 ++++++++++++++++++ .../yaml/badpixel_monitor_interface.yaml | 109 +++++++++ jwql/website/apps/jwql/monitor_views.py | 32 +++ .../apps/jwql/templates/badpixel_monitor.html | 26 +++ jwql/website/apps/jwql/urls.py | 1 + 6 files changed, 426 insertions(+) create mode 100644 jwql/website/apps/jwql/monitor_pages/badpixel_monitor.py create mode 100755 jwql/website/apps/jwql/monitor_pages/monitor_badpixel_bokeh.py create mode 100755 jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml create mode 100644 jwql/website/apps/jwql/templates/badpixel_monitor.html diff --git a/jwql/website/apps/jwql/monitor_pages/badpixel_monitor.py b/jwql/website/apps/jwql/monitor_pages/badpixel_monitor.py new file mode 100644 index 000000000..5c01a7aa6 --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/badpixel_monitor.py @@ -0,0 +1,44 @@ +import os + +import numpy as np + +from jwql.bokeh_templating import BokehTemplate +from jwql.utils.utils import get_config + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +class BadPixelMonitor(BokehTemplate): + + def pre_init(self): + + +still need to edit below here + + self._embed = True + + #app design + self.format_string = None + self.interface_file = os.path.join(SCRIPT_DIR, "badpixel_monitor_interface.yml") + + self.settings = get_config() + self.output_dir = self.settings['outputs'] + + self.load_data() + self.timestamps = np.arange(10) / 10. + self.dark_current = np.arange(10) + + + def post_init(self): + + self.refs['dark_current_yrange'].start = min(self.dark_current) + self.refs['dark_current_yrange'].end = max(self.dark_current) + + + def load_data(self): + #actually load data: + new_data = np.arange(10) / 20 + + #update columndatasource + self.dark_current = new_data + +DarkMonitor() diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_badpixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_badpixel_bokeh.py new file mode 100755 index 000000000..9ec5f6f8c --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/monitor_badpixel_bokeh.py @@ -0,0 +1,214 @@ +"""This module contains code for the bad pixel monitor Bokeh plots. + +Author +------ + + - Bryan Hilbert + +Use +--- + + This module can be used from the command line as such: + + :: + + from jwql.website.apps.jwql import monitor_pages + monitor_template = monitor_pages.BadPixelMonitor('NIRCam', 'NRCA3_FULL') + script, div = monitor_template.embed("bad_pixel_time_figure") +""" + +import os + +from astropy.io import fits +from astropy.time import Time +from bokeh.models.tickers import LogTicker +import numpy as np + +from jwql.database.database_interface import session +from jwql.database.database_interface import NIRCamBadPixelQueryHistory, NIRCamBadPixelStats +from jwql.database.database_interface import NIRISSBadPixelQueryHistory, NIRISSBadPixelStats +from jwql.database.database_interface import MIRIBadPixelQueryHistory, MIRIBadPixelStats +from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats +from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats +from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE +from jwql.utils.utils import get_config +from jwql.bokeh_templating import BokehTemplate + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class BadPixelMonitor(BokehTemplate): + + # Combine instrument and aperture into a single property because we + # do not want to invoke the setter unless both are updated + @property + def aperture_info(self): + return (self._instrument, self._aperture) + + @aperture_info.setter + def aperture_info(self, info): + self._instrument, self._aperture = info + self.pre_init() + self.post_init() + + def _dark_mean_image(self): + """Update bokeh objects with mean dark image data.""" + + # Open the mean dark current file and get the data + mean_dark_image_file = self.pixel_table[-1].mean_dark_image_file + mean_slope_dir = os.path.join(get_config()['outputs'], 'dark_monitor', 'mean_slope_images') + mean_dark_image_path = os.path.join(mean_slope_dir, mean_dark_image_file) + with fits.open(mean_dark_image_path) as hdulist: + data = hdulist[1].data + + # Update the plot with the data and boundaries + y_size, x_size = np.shape(data) + self.refs["mean_dark_source"].data['image'] = [data] + self.refs["stamp_xr"].end = x_size + self.refs["stamp_yr"].end = y_size + self.refs["mean_dark_source"].data['dw'] = [x_size] + self.refs["mean_dark_source"].data['dh'] = [x_size] + + # Set the image color scale + self.refs["log_mapper"].high = 0 + self.refs["log_mapper"].low = -.2 + + # This should add ticks to the colorbar, but it doesn't + self.refs["mean_dark_cbar"].ticker = LogTicker() + + # Add a title + self.refs['mean_dark_image_figure'].title.text = self._aperture + self.refs['mean_dark_image_figure'].title.align = "center" + self.refs['mean_dark_image_figure'].title.text_font_size = "20px" + + def pre_init(self): + # Start with default values for instrument and aperture because + # BokehTemplate's __init__ method does not allow input arguments + try: + dummy_instrument = self._instrument + dummy_aperture = self._aperture + except AttributeError: + self._instrument = 'NIRCam' + self._aperture = 'NRCA1_FULL' + + self._embed = True + + # Fix aperture/detector name discrepency + if self._aperture in ['NRCA5_FULL', 'NRCB5_FULL']: + self.detector = '{}LONG'.format(self._aperture[0:4]) + else: + self.detector = self._aperture.split('_')[0] + + # App design + self.format_string = None + self.interface_file = os.path.join(SCRIPT_DIR, "yaml", "badpixel_monitor_interface.yaml") + + # Load data tables + self.load_data() + + # Data for mean dark versus time plot + datetime_stamps = [row.obs_mid_time for row in self.badpixel_table] + times = Time(datetime_stamps, format='datetime', scale='utc') # Convert to MJD + self.timestamps = times.mjd + + + replace the line below with a new function? We need the number of bad pixels for + each flavor of bad pixel versus time + self.dark_current = [row.mean for row in self.badpixel_table] + + something like this, where self.bad_pixels is a dictionary with keys equal + to aperture names (e.g. NRCA1_FULL) + self.bad_pixels = XXXX.bad_pixel_trends() + + We also want to retrieve the information for just the most recent entry for each + bad pixel type. We will print the coordinates of the most-recently found bad pixels + in a table, and show an image (or maybe just a blank 2048x2048 grid) with dots at + the locations of the new bad pixels. + + So for each type of bad pixel, that's three panels. + 1) bad pixel count vs time + 2) plot of latest locations + 3) table of latest bad pix - API view, so users can download + + for badpix_type in BAD_PIXEL_TYPES: + entries = [row for row in self.badpixel_table if row.type == badpix_type] + time_stamps = [row.obs_mid_time for row in entries] + latest_timestamp = time_stamps[-1].isoformat() + latest_xcoord = entries[-1].x_coord + latest_ycoord = entries[-1].y_coord + + + + + # Data for dark current histogram plot (full detector) + # Just show the last histogram, which is the one most recently + # added to the database + last_hist_index = -1 + self.last_timestamp = datetime_stamps[last_hist_index].isoformat() + self.full_dark_bin_center = np.array([row.hist_dark_values for + row in self.dark_table])[last_hist_index] + self.full_dark_amplitude = [row.hist_amplitudes for + row in self.dark_table][last_hist_index] + self.full_dark_bottom = np.zeros(len(self.full_dark_amplitude)) + deltas = self.full_dark_bin_center[1:] - self.full_dark_bin_center[0: -1] + self.full_dark_bin_width = np.append(deltas[0], deltas) + + def post_init(self): + + self._update_dark_v_time() + self._update_hist() + self._dark_mean_image() + + def identify_tables(self): + """Determine which dark current database tables as associated with + a given instrument""" + + mixed_case_name = JWST_INSTRUMENT_NAMES_MIXEDCASE[self._instrument.lower()] + self.query_table = eval('{}BadPixelQueryHistory'.format(mixed_case_name)) + self.pixel_table = eval('{}BadPixelStats'.format(mixed_case_name)) + + def load_data(self): + """Query the database tables to get data""" + + # Determine which database tables are needed based on instrument + self.identify_tables() + + # Query database for all data with a matching aperture + self.badpixel_table = session.query(self.pixel_table) \ + .filter(self.pixel_table.detector == self.detector) \ + .all() + + def _update_dark_v_time(self): + + # Define y range of dark current v. time plot + buffer_size = 0.05 * (max(self.dark_current) - min(self.dark_current)) + self.refs['dark_current_yrange'].start = min(self.dark_current) - buffer_size + self.refs['dark_current_yrange'].end = max(self.dark_current) + buffer_size + + # Define x range of dark current v. time plot + horizontal_half_buffer = (max(self.timestamps) - min(self.timestamps)) * 0.05 + if horizontal_half_buffer == 0: + horizontal_half_buffer = 1. # day + self.refs['dark_current_xrange'].start = min(self.timestamps) - horizontal_half_buffer + self.refs['dark_current_xrange'].end = max(self.timestamps) + horizontal_half_buffer + + # Add a title + self.refs['dark_current_time_figure'].title.text = self._aperture + self.refs['dark_current_time_figure'].title.align = "center" + self.refs['dark_current_time_figure'].title.text_font_size = "20px" + + def _update_hist(self): + + # Define y range of dark current histogram + buffer_size = 0.05 * (max(self.full_dark_amplitude) - min(self.full_dark_bottom)) + self.refs['dark_histogram_yrange'].start = min(self.full_dark_bottom) + self.refs['dark_histogram_yrange'].end = max(self.full_dark_amplitude) + buffer_size + + # Define x range of dark current histogram + self.refs['dark_histogram_xrange'].start = min(self.full_dark_bin_center) + self.refs['dark_histogram_xrange'].end = max(self.full_dark_bin_center) + + # Add a title + self.refs['dark_full_histogram_figure'].title.text = self._aperture + self.refs['dark_full_histogram_figure'].title.align = "center" + self.refs['dark_full_histogram_figure'].title.text_font_size = "20px" diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml new file mode 100755 index 000000000..cdadceb33 --- /dev/null +++ b/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml @@ -0,0 +1,109 @@ +# YAML file defining bokeh figures for the dark monitor +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Dark Current v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &dark_current_source + ref: "dark_current_source" + data: + time: !self.timestamps + dark_current: !self.dark_current + +- !Range1d: &dark_current_xrange + ref: "dark_current_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &dark_current_yrange + ref: "dark_current_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [-1, 1] + +- !Figure: &dark_current_time_figure + ref: "dark_current_time_figure" + x_axis_label: "Time (MJD)" + y_axis_label: "Dark current (e-)" + x_range: *dark_current_xrange + y_range: *dark_current_yrange + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'dark_current', line_width: 5, 'source': *dark_current_source} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Dark Histogram Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &dark_full_hist_source + ref: "dark_full_hist_source" + data: + full_dark_bin_center: !self.full_dark_bin_center + full_dark_amplitude: !self.full_dark_amplitude + full_dark_bottom: !self.full_dark_bottom + full_dark_bin_width: !self.full_dark_bin_width + +- !Range1d: &dark_histogram_xrange + ref: "dark_histogram_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &dark_histogram_yrange + ref: "dark_histogram_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [0, 1] + +- !Figure: &dark_full_histogram_figure + ref: "dark_full_histogram_figure" + x_axis_label: "Dark Current (DN/sec)" + y_axis_label: "Number of Pixels" + x_range: *dark_histogram_xrange + y_range: *dark_histogram_yrange + elements: + - {'kind': 'vbar', 'x': 'full_dark_bin_center', 'y': 'full_dark_bin_width', 'top': 'full_dark_amplitude', 'bottom': 'full_dark_bottom', 'source': *dark_full_hist_source} +# - {'kind': 'text', 'x': 0, 'y': 20000, 'id': 1001} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Mean Dark Image Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &mean_dark_source + ref: "mean_dark_source" + data: + dh: [1] + dw: [1] + image: [[[1,0], [0, 1]]] +- !Range1d: &stamp_xr + ref: "stamp_xr" + #start: 0 + #end: 1 + #bounds: !!python/tuple [0, 1] +- !Range1d: &stamp_yr + ref: "stamp_yr" + #start: 0 + #end: 1 + #bounds: !!python/tuple [0, 1] +- !LogColorMapper: &log_mapper + ref: "log_mapper" + palette: "Viridis256" + low: 0. + high: 1. +- !ColorBar: &mean_dark_cbar + ref: "mean_dark_cbar" + color_mapper: *log_mapper + location: !!python/tuple [0, 0] +- !Figure: &mean_dark_image_figure + ref: "mean_dark_image_figure" + x_axis_label: "Col = SIAF det Y" + y_axis_label: "Row = SIAF det X" + x_range: *stamp_xr + y_range: *stamp_yr + tools: "" + height: 250 # Not working + width: 250 # Not working + elements: + - {"kind": "image", "image": "image", "x": 0, "y": 0, "dh": 'dh', "dw": 'dh', "source": *mean_dark_source, "color_mapper": *log_mapper} + - {"kind": "layout", "obj": *mean_dark_cbar, "place": "right"} + +#- !Document: +# - !column: +# - *dark_current_time_figure +# - *dark_full_histogram_figure diff --git a/jwql/website/apps/jwql/monitor_views.py b/jwql/website/apps/jwql/monitor_views.py index 1668072d0..e07d54042 100644 --- a/jwql/website/apps/jwql/monitor_views.py +++ b/jwql/website/apps/jwql/monitor_views.py @@ -38,6 +38,38 @@ FILESYSTEM_DIR = os.path.join(get_config()['jwql_dir'], 'filesystem') +def badpix_monitor(request, inst): + """Generate the dark monitor page for a given instrument + + Parameters + ---------- + request : HttpRequest object + Incoming request from the webpage + inst : str + Name of JWST instrument + + Returns + ------- + HttpResponse object + Outgoing response sent to the webpage + """ + + # Ensure the instrument is correctly capitalized + inst = JWST_INSTRUMENT_NAMES_MIXEDCASE[inst.lower()] + + tabs_components = bokeh_containers.badpixel_monitor_tabs(inst) + + template = "badpixel_monitor.html" + + context = { + 'inst': inst, + 'tabs_components': tabs_components, + } + + # Return a HTTP response with the template and dictionary of variables + return render(request, template, context) + + def dark_monitor(request, inst): """Generate the dark monitor page for a given instrument diff --git a/jwql/website/apps/jwql/templates/badpixel_monitor.html b/jwql/website/apps/jwql/templates/badpixel_monitor.html new file mode 100644 index 000000000..0ba5a1274 --- /dev/null +++ b/jwql/website/apps/jwql/templates/badpixel_monitor.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} + +{% block preamble %} + + {{ inst }} Bad Pixel Monitor- JWQL + +{% endblock %} + +{% block content %} + +
    + +

    {{ inst }} Bad Pixel Monitor

    +
    + +
    + + {{ tabs_components[0] | safe }} +
    + + + {{ tabs_components[1] | safe }} + +
    + +{% endblock %} diff --git a/jwql/website/apps/jwql/urls.py b/jwql/website/apps/jwql/urls.py index 439d0c583..14807e6f3 100644 --- a/jwql/website/apps/jwql/urls.py +++ b/jwql/website/apps/jwql/urls.py @@ -70,6 +70,7 @@ # Common monitor views re_path(r'^(?P({}))/.+_monitor/$'.format(instruments), monitor_views.dark_monitor, name='dark_monitor'), + re_path(r'^(?P({}))/.+_monitor/$'.format(instruments), monitor_views.badpixel_monitor, name='badpixel_monitor'), # Main site views path('about/', views.about, name='about'), From b232b1f9801bfededd6cba081c1b3d6e12cef7c6 Mon Sep 17 00:00:00 2001 From: Ben Sunnquist Date: Wed, 12 Aug 2020 15:23:10 -0400 Subject: [PATCH 044/227] added tests for readnoise monitor --- jwql/tests/test_readnoise_monitor.py | 136 +++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 jwql/tests/test_readnoise_monitor.py diff --git a/jwql/tests/test_readnoise_monitor.py b/jwql/tests/test_readnoise_monitor.py new file mode 100644 index 000000000..4e89868a6 --- /dev/null +++ b/jwql/tests/test_readnoise_monitor.py @@ -0,0 +1,136 @@ +#! /usr/bin/env python + +"""Tests for the ``readnoise_monitor`` module. + +Authors +------- + + - Ben Sunnquist + +Use +--- + + These tests can be run via the command line (omit the ``-s`` to + suppress verbose output to stdout): + :: + + pytest -s test_readnoise_monitor.py +""" + +from collections import OrderedDict +import os +import pytest + +import numpy as np + +from jwql.database.database_interface import NIRCamReadnoiseQueryHistory, NIRCamReadnoiseStats +from jwql.instrument_monitors.common_monitors import readnoise_monitor +from jwql.utils.utils import get_config + + +ON_JENKINS = '/home/jenkins' in os.path.expanduser('~') + + +def test_determine_pipeline_steps(): + """Test the correct pipeline steps are called""" + + monitor = readnoise_monitor.Readnoise() + monitor.instrument = 'NIRCAM' + monitor.read_pattern = 'RAPID' + + # Create dictionary of expected pipeline steps + pipeline_steps_truth = OrderedDict([('group_scale', False), + ('dq_init', True), + ('superbias', True), + ('refpix', True)]) + + # Find the necessary pipeline steps using the readnoise monitor + pipeline_steps = monitor.determine_pipeline_steps() + + assert pipeline_steps == pipeline_steps_truth + + +def test_get_amp_stats(): + """Test amp stat calculations""" + + monitor = readnoise_monitor.Readnoise() + + # Create test data and its corresponding amp stats + data = np.zeros((100, 100)) + amps = {'1': [(0, 100, 1), (0, 100, 1)]} + amp_stats_truth = {'amp1_mean': 0.0, + 'amp1_stddev': 0.0, + 'amp1_n': np.array([10000]), + 'amp1_bin_centers': np.array([0.0])} + + # Find the amp stats using the readnoise monitor + amp_stats = monitor.get_amp_stats(data, amps) + + assert amp_stats == amp_stats_truth + + +@pytest.mark.skipif(ON_JENKINS, + reason='Requires access to central storage.') +def test_get_metadata(): + """Test retrieval of metadata from input file""" + + monitor = readnoise_monitor.Readnoise() + filename = os.path.join(get_config()['test_dir'], 'readnoise_monitor', 'test_image_1.fits') + monitor.get_metadata(filename) + + assert monitor.detector == 'NRCA1' + assert monitor.read_pattern == 'BRIGHT1' + assert monitor.subarray == 'FULL' + assert monitor.nints == 3 + assert monitor.ngroups == 4 + assert monitor.substrt1 == 1 + assert monitor.substrt2 == 1 + assert monitor.subsize1 == 2048 + assert monitor.subsize2 == 2048 + assert monitor.date_obs == '2016-01-18' + assert monitor.time_obs == '04:35:14.523' + assert monitor.expstart == '2016-01-18T04:35:14.523' + + +def test_identify_tables(): + """Be sure the correct database tables are identified""" + + monitor = readnoise_monitor.Readnoise() + monitor.instrument = 'nircam' + monitor.identify_tables() + + assert monitor.query_table == eval('NIRCamReadnoiseQueryHistory') + assert monitor.stats_table == eval('NIRCamReadnoiseStats') + + +def test_make_readnoise_image(): + """Test readnoise image creation""" + + monitor = readnoise_monitor.Readnoise() + + # Create test data and its corresponding readnoise + data = np.arange(200).reshape(1, 8, 5, 5) + readnoise_truth = np.zeros((5, 5)) + + # Find the readnoise in the test data using the readnoise monitor + readnoise = monitor.make_readnoise_image(data) + + assert np.all(readnoise == readnoise_truth) + + +def test_make_histogram(): + """Test histogram creation""" + + monitor = readnoise_monitor.Readnoise() + + # Create test data and its corresponding histogram stats + data = np.zeros((100, 100)) + counts_truth = [10000] + bin_centers_truth = [0.0] + + # Find the histogram stats of the test data using the readnoise monitor + counts, bin_centers = monitor.make_histogram(data) + counts, bin_centers = list(counts), list(bin_centers) + + assert counts == counts_truth + assert bin_centers == bin_centers_truth From 460449ba1ad285a4b267f9957c09009ee29c63fc Mon Sep 17 00:00:00 2001 From: Ben Sunnquist Date: Wed, 12 Aug 2020 15:29:04 -0400 Subject: [PATCH 045/227] minor pep8 edits --- jwql/tests/test_readnoise_monitor.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jwql/tests/test_readnoise_monitor.py b/jwql/tests/test_readnoise_monitor.py index 4e89868a6..37f1c9bd1 100644 --- a/jwql/tests/test_readnoise_monitor.py +++ b/jwql/tests/test_readnoise_monitor.py @@ -40,9 +40,9 @@ def test_determine_pipeline_steps(): # Create dictionary of expected pipeline steps pipeline_steps_truth = OrderedDict([('group_scale', False), - ('dq_init', True), - ('superbias', True), - ('refpix', True)]) + ('dq_init', True), + ('superbias', True), + ('refpix', True)]) # Find the necessary pipeline steps using the readnoise monitor pipeline_steps = monitor.determine_pipeline_steps() @@ -58,9 +58,9 @@ def test_get_amp_stats(): # Create test data and its corresponding amp stats data = np.zeros((100, 100)) amps = {'1': [(0, 100, 1), (0, 100, 1)]} - amp_stats_truth = {'amp1_mean': 0.0, - 'amp1_stddev': 0.0, - 'amp1_n': np.array([10000]), + amp_stats_truth = {'amp1_mean': 0.0, + 'amp1_stddev': 0.0, + 'amp1_n': np.array([10000]), 'amp1_bin_centers': np.array([0.0])} # Find the amp stats using the readnoise monitor @@ -94,7 +94,7 @@ def test_get_metadata(): def test_identify_tables(): """Be sure the correct database tables are identified""" - + monitor = readnoise_monitor.Readnoise() monitor.instrument = 'nircam' monitor.identify_tables() From 867d2450d31fc30d561fc8cf6d6fc553cd191e6f Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Thu, 13 Aug 2020 08:15:00 -0400 Subject: [PATCH 046/227] Updates for 08/2020 pyup --- environment_python_3_6.yml | 6 +++--- requirements.txt | 4 ++-- setup.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/environment_python_3_6.yml b/environment_python_3_6.yml index 447e0e87f..e61b570f6 100644 --- a/environment_python_3_6.yml +++ b/environment_python_3_6.yml @@ -2,7 +2,7 @@ channels: - defaults - http://ssb.stsci.edu/astroconda dependencies: -- astroquery=0.3.10 +- astroquery=0.4 - bokeh>=1.0,<1.4 - django=2.2.5 - flake8=3.7.9 @@ -28,10 +28,10 @@ dependencies: - sqlalchemy=1.3.17 - twine=2.0.0 - pip: - - asdf==2.6.0 + - asdf==2.7.0 - astropy==4.0.1 - authlib==0.14.3 - - codecov==2.1.3 + - codecov==2.1.8 - crds==7.5.0.0 - jwedb>=0.0.3 - pysqlite3==0.2.2 diff --git a/requirements.txt b/requirements.txt index 28c17835c..55a382a4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,10 @@ asdf==2.7.0 astropy==4.0.1.post1 astroquery==0.4.1 authlib==0.14.3 -bokeh==2.1.1 +bokeh==1.3.4 codecov==2.1.8 crds==7.5.0.0 -django==3.0.8 +django==2.2.5 flake8==3.8.3 inflection==0.5.0 ipython==7.17.0 diff --git a/setup.py b/setup.py index 382030c3a..94b57a68f 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ 'bokeh>=1.0,<1.4', 'codecov', 'crds', - 'django>=2.0', + 'django>=2.0,<3.0', 'flake8', 'inflection', 'ipython', From 266e826873afc55cea37c312accf633ae48d38f2 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 13 Aug 2020 10:44:09 -0400 Subject: [PATCH 047/227] First try at defining webpage tabs, one per detector --- jwql/website/apps/jwql/bokeh_containers.py | 72 +++++ .../monitor_pages/monitor_badpixel_bokeh.py | 218 ++++++++++---- .../yaml/badpixel_monitor_interface.yaml | 274 ++++++++++++++++-- 3 files changed, 477 insertions(+), 87 deletions(-) diff --git a/jwql/website/apps/jwql/bokeh_containers.py b/jwql/website/apps/jwql/bokeh_containers.py index b46c7a230..c8d8d6ec5 100644 --- a/jwql/website/apps/jwql/bokeh_containers.py +++ b/jwql/website/apps/jwql/bokeh_containers.py @@ -34,6 +34,78 @@ PACKAGE_DIR = os.path.dirname(__location__.split('website')[0]) REPO_DIR = os.path.split(PACKAGE_DIR)[0] +def badpixel_monitor_tabs(instrument): + """Creates the various tabs of the bad pixel monitor results page. + + Parameters + ---------- + instrument : str + The JWST instrument of interest (e.g. ``nircam``). + + Returns + ------- + div : str + The HTML div to render bad pixel monitor plots + script : str + The JS script to render bad pixel monitor plots + """ + full_apertures = FULL_FRAME_APERTURES[instrument.upper()] + + templates_all_apertures = {} + for aperture in full_apertures: + + # Start with default values for instrument and aperture because + # BokehTemplate's __init__ method does not allow input arguments + monitor_template = monitor_pages.BadPixelMonitor() + + # Set instrument and monitor using DarkMonitor's setters + monitor_template.aperture_info = (instrument, aperture) + templates_all_apertures[aperture] = monitor_template + + #for reference - here are the bad pixel types + #badpix_types_from_flats = ['DEAD', 'LOW_QE', 'OPEN', 'ADJ_OPEN'] + #badpix_types_from_darks = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH'] + + # We loop over detectors here, and create one tab per detector, rather + # than one tab for each plot type, as is done with the dark monitor + all_tabs = [] + for aperture_name, template in templates_all_apertures.items(): + + tab_plots = [] + for badpix_type in BAD_PIXEL_TYPES: + history = template.refs["{}_history_figure".format(badpix_type)] + history.sizing_mode = "scale_width" # Make sure the sizing is adjustable + tab_plots.append(history) + + # Now add the image + image = tmeplate.refs["badpix_image_figure"] + image.sizing_mode = "scale_width" # Make sure the sizing is adjustable + tab_plots.append(image) + + # Let's put three plots per line, but keep the image with the marked + # locations last, on its own line + num_plots = len(tab_plots) - 1 + plots_per_line = 3 + start_index = np.arange(0, num_plots, plots_per_line) + badpix_layout = layout( + tab_plots[start_index[0]: start_index[1]], + tab_plots[start_index[1]: start_index[2]], + tab_plots[start_index[2]: start_index[3]], + [tab_plots[-1]] + ) + + badpix_layout.sizing_mode = "scale_width" # Make sure the sizing is adjustable + badpix_tab = Panel(child=badpix_layout, title=aperture_name) + all_tabs.append(badpix_tab) + + # Build tabs + tabs = Tabs(tabs=all_tabs) + + # Return tab HTML and JavaScript to web app + script, div = components(tabs) + + return div, script + def dark_monitor_tabs(instrument): """Creates the various tabs of the dark monitor results page. diff --git a/jwql/website/apps/jwql/monitor_pages/monitor_badpixel_bokeh.py b/jwql/website/apps/jwql/monitor_pages/monitor_badpixel_bokeh.py index 9ec5f6f8c..8851ed1d1 100755 --- a/jwql/website/apps/jwql/monitor_pages/monitor_badpixel_bokeh.py +++ b/jwql/website/apps/jwql/monitor_pages/monitor_badpixel_bokeh.py @@ -21,7 +21,6 @@ from astropy.io import fits from astropy.time import Time -from bokeh.models.tickers import LogTicker import numpy as np from jwql.database.database_interface import session @@ -31,7 +30,7 @@ from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE -from jwql.utils.utils import get_config +from jwql.utils.utils import get_config, filesystem_path from jwql.bokeh_templating import BokehTemplate SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -51,35 +50,126 @@ def aperture_info(self, info): self.pre_init() self.post_init() - def _dark_mean_image(self): - """Update bokeh objects with mean dark image data.""" + def bad_pixel_history(self, bad_type): + """Use the database to construct information on the total number + of a given type of bad pixels over time + Parameters + ---------- + bad_type : str + The flavor of bad pixel (e.g. 'hot') + + Returns + ------- + num_bad : numpy.ndarray + 1D array of the number of bad pixels + + dates : datetime.datetime + 1D array of dates/times corresponding to num_bad + """ + # Find all the rows corresponding to the requested type of bad pixel + rows = [row for row in self.badpixel_table if row.type == bad_type] + + # Extract the dates and number of bad pixels from each entry + dates = [row.obs_mid_time for row in rows] + num = [len(row.coordinates[0]) for row in rows] + + # If there are no valid entres in the database, return None + if len(dates) == 0: + return None, None + + # Sort by date to make sure everything is in chronological order + chrono = np.argsort(dates) + dates = dates[chrono] + num = num[chrono] + + # Sum the number of bad pixels found from the earliest entry up to + # each new entry + num_bad = [np.sum(num[0:i]) for i in range(1, len(num)+1)] + + return num_bad, dates + + def _badpix_image(self): + """Update bokeh objects with sample image data. + """ # Open the mean dark current file and get the data - mean_dark_image_file = self.pixel_table[-1].mean_dark_image_file - mean_slope_dir = os.path.join(get_config()['outputs'], 'dark_monitor', 'mean_slope_images') - mean_dark_image_path = os.path.join(mean_slope_dir, mean_dark_image_file) - with fits.open(mean_dark_image_path) as hdulist: + with fits.open(self.image_file) as hdulist: data = hdulist[1].data + # Grab only one frame + ndims = len(data.shape) + if ndims == 4: + data = data[0, -1, :, :] + elif ndims == 3: + data = data[-1, :, :] + elif ndims == 2: + pass + else: + raise ValueError('Unrecognized number of dimensions in data file: {}'.format(ndims)) + # Update the plot with the data and boundaries - y_size, x_size = np.shape(data) - self.refs["mean_dark_source"].data['image'] = [data] + y_size, x_size = data.shape + self.refs["bkgd_image"].data['image'] = [data] self.refs["stamp_xr"].end = x_size self.refs["stamp_yr"].end = y_size - self.refs["mean_dark_source"].data['dw'] = [x_size] - self.refs["mean_dark_source"].data['dh'] = [x_size] + self.refs["bkgd_source"].data['dw'] = [x_size] + self.refs["bkgd_source"].data['dh'] = [y_size] # Set the image color scale self.refs["log_mapper"].high = 0 self.refs["log_mapper"].low = -.2 - # This should add ticks to the colorbar, but it doesn't - self.refs["mean_dark_cbar"].ticker = LogTicker() - # Add a title - self.refs['mean_dark_image_figure'].title.text = self._aperture - self.refs['mean_dark_image_figure'].title.align = "center" - self.refs['mean_dark_image_figure'].title.text_font_size = "20px" + self.refs['badpix_map_figure'].title.text = '{}: New Bad Pixels'.format(self._aperture) + self.refs['badpix_map_figure'].title.align = "center" + self.refs['badpix_map_figure'].title.text_font_size = "20px" + + + def most_recent_coords(self, bad_type): + """Return the coordinates of the bad pixels in the most recent + database entry for the given bad pixel type + + Parameters + ---------- + bad_type : str + The flavor of bad pixel (e.g. 'hot') + + Returns + ------- + coords : tup + Tuple containing a list of x coordinates and a list of y + coordinates + """ + # Find all the rows corresponding to the requested type of bad pixel + rows = [row for row in self.badpixel_table if row.type == bad_type] + + # Extract the dates, number of bad pixels, and files used from each entry + dates = [row.obs_mid_time for row in rows] + coords = [row.coordinates for row in rows] + files = [row.source_files[0] for row in rows] + + # If there are no valid entres in the database, return None + if len(dates) == 0: + return None, None + + # Sort by date to make sure everything is in chronological order + chrono = np.argsort(dates) + dates = dates[chrono] + coords = coords[chrono] + files = files[chrono] + + # Keep track of the latest timestamp + self.last_timestamp = dates[-1].isoformat() + + # Grab the name of one of the files used when these bad pixels + # were identified. We'll use this as an image on top of which + # the bad pixels will be noted. Note that these should be + # slope files + self.image_file = filesystem_path(files[-1]) + + # Return the list of coordinates for the most recent entry + return coords[-1] + def pre_init(self): # Start with default values for instrument and aperture because @@ -106,58 +196,37 @@ def pre_init(self): # Load data tables self.load_data() - # Data for mean dark versus time plot - datetime_stamps = [row.obs_mid_time for row in self.badpixel_table] - times = Time(datetime_stamps, format='datetime', scale='utc') # Convert to MJD - self.timestamps = times.mjd + # Populate a dictionary with the number of bad pixels vs time for + # each type of bad pixel. We can't get the full list of bad pixel + # types from the database itself, because if there is a type of bad + # pixel with no found instances, then it won't appear in the database + # Also populate a dictionary containing the locations of all of the + # bad pixels found in the most recent search + self.bad_history = {} + self.bad_latest = {} + for badtype in BAD_PIXEL_TYPES: + num, times = self.bad_pixel_history(badtype) + self.bad_history[badtype] = (times, num) + self.bad_latest[badtype] = self.most_recent_coords(badtype) - replace the line below with a new function? We need the number of bad pixels for - each flavor of bad pixel versus time - self.dark_current = [row.mean for row in self.badpixel_table] - something like this, where self.bad_pixels is a dictionary with keys equal - to aperture names (e.g. NRCA1_FULL) - self.bad_pixels = XXXX.bad_pixel_trends() We also want to retrieve the information for just the most recent entry for each bad pixel type. We will print the coordinates of the most-recently found bad pixels in a table, and show an image (or maybe just a blank 2048x2048 grid) with dots at the locations of the new bad pixels. - So for each type of bad pixel, that's three panels. + for each type of bad pixel, there are three items to show: 1) bad pixel count vs time - 2) plot of latest locations + 2) plot of latest locations (overlain on an image?) 3) table of latest bad pix - API view, so users can download - for badpix_type in BAD_PIXEL_TYPES: - entries = [row for row in self.badpixel_table if row.type == badpix_type] - time_stamps = [row.obs_mid_time for row in entries] - latest_timestamp = time_stamps[-1].isoformat() - latest_xcoord = entries[-1].x_coord - latest_ycoord = entries[-1].y_coord - - - - - # Data for dark current histogram plot (full detector) - # Just show the last histogram, which is the one most recently - # added to the database - last_hist_index = -1 - self.last_timestamp = datetime_stamps[last_hist_index].isoformat() - self.full_dark_bin_center = np.array([row.hist_dark_values for - row in self.dark_table])[last_hist_index] - self.full_dark_amplitude = [row.hist_amplitudes for - row in self.dark_table][last_hist_index] - self.full_dark_bottom = np.zeros(len(self.full_dark_amplitude)) - deltas = self.full_dark_bin_center[1:] - self.full_dark_bin_center[0: -1] - self.full_dark_bin_width = np.append(deltas[0], deltas) def post_init(self): - - self._update_dark_v_time() - self._update_hist() - self._dark_mean_image() + self._update_badpix_v_time() + self._update_badpix_loc_plot() + self._badpix_image() def identify_tables(self): """Determine which dark current database tables as associated with @@ -180,7 +249,7 @@ def load_data(self): def _update_dark_v_time(self): - # Define y range of dark current v. time plot + # Define y range of bad pixel v. time plot buffer_size = 0.05 * (max(self.dark_current) - min(self.dark_current)) self.refs['dark_current_yrange'].start = min(self.dark_current) - buffer_size self.refs['dark_current_yrange'].end = max(self.dark_current) + buffer_size @@ -198,7 +267,6 @@ def _update_dark_v_time(self): self.refs['dark_current_time_figure'].title.text_font_size = "20px" def _update_hist(self): - # Define y range of dark current histogram buffer_size = 0.05 * (max(self.full_dark_amplitude) - min(self.full_dark_bottom)) self.refs['dark_histogram_yrange'].start = min(self.full_dark_bottom) @@ -212,3 +280,37 @@ def _update_hist(self): self.refs['dark_full_histogram_figure'].title.text = self._aperture self.refs['dark_full_histogram_figure'].title.align = "center" self.refs['dark_full_histogram_figure'].title.text_font_size = "20px" + + def _update_badpix_loc_plot(self): + """Update the plot properties for the plot showing the locations + of new bad pixels""" + self.refs['bad_pixel_map_xrange'].start = 0 + self.refs['bad_pixel_map_xrange'].end = 2047 + self.refs['bad_pixel_map_yrange'].start = 0 + self.refs['bad_pixel_map_yrange'].end = 2047 + + self.refs['bad_pixel_map'].title.text = '{}: New Bad Pixel Locations'.format(self._aperture) + self.refs['bad_pixel_map'].title.align = "center" + self.refs['bad_pixel_map'].title.text_font_size = "20px" + + def _update_badpix_v_time(self): + """Update the plot properties for the plots of the number of bad + pixels versus time + """ + for bad_type in BAD_PIXEL_TYPES: + # Define y ranges of bad pixel v. time plot + buffer_size = 0.05 * (max(self.bad_history[badtype][1]) - min(self.bad_history[badtype][1])) + self.refs['{}_history_yrange'.format(bad_type)].start = min(self.bad_history[badtype][1]) - buffer_size + self.refs['{}_history_yrange'.format(bad_type)].end = max(self.bad_history[badtype][1]) + buffer_size + + # Define x range of bad_pixel v. time plot + horizontal_half_buffer = (max(self.bad_history[badtype][0]) - min(self.bad_history[badtype][0])) * 0.05 + if horizontal_half_buffer == 0: + horizontal_half_buffer = 1. # day + self.refs['{}_history_xrange'.format(bad_type)].start = min(self.bad_history[badtype][0]) - horizontal_half_buffer + self.refs['{}_history_xrange'.format(bad_type)].end = max(self.bad_history[badtype][0]) + horizontal_half_buffer + + # Add a title + self.refs['{}_history_figure'].title.text = '{}: {} pixels'.format(self._aperture, bad_type) + self.refs['{}_history_figure'].title.align = "center" + self.refs['{}_history_figure'].title.text_font_size = "20px" diff --git a/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml b/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml index cdadceb33..b3675d422 100755 --- a/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml +++ b/jwql/website/apps/jwql/monitor_pages/yaml/badpixel_monitor_interface.yaml @@ -1,33 +1,247 @@ -# YAML file defining bokeh figures for the dark monitor +# YAML file defining bokeh figures for the bad pixel monitor # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Dark Current v. Time Figure +# Dead Pixels v. Time Figure # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &dark_current_source - ref: "dark_current_source" +- !ColumnDataSource: &dead_history_source + ref: "dead_history_source" data: - time: !self.timestamps - dark_current: !self.dark_current + time: !self.bad_historyxxxxx + dead_pixels: !self.bad_historyxxxxx -- !Range1d: &dark_current_xrange - ref: "dark_current_xrange" +- !Range1d: &dead_history_xrange + ref: "dead_history_xrange" #start: 0 #end: 1 #bounds: 'auto' #!!python/tuple [0, 1] -- !Range1d: &dark_current_yrange - ref: "dark_current_yrange" +- !Range1d: &dead_history_yrange + ref: "dead_history_yrange" #start: 0 #end: 1 #bounds: !!python/tuple [-1, 1] -- !Figure: &dark_current_time_figure - ref: "dark_current_time_figure" +- !Figure: &dead_history_figure + ref: "dead_history_figure" x_axis_label: "Time (MJD)" - y_axis_label: "Dark current (e-)" - x_range: *dark_current_xrange - y_range: *dark_current_yrange + y_axis_label: "Number of DEAD pixels" + x_range: *dead_history_xrange + y_range: *dead_history_yrange elements: - - {'kind': 'circle', 'x': 'time', 'y': 'dark_current', line_width: 5, 'source': *dark_current_source} + - {'kind': 'circle', 'x': 'time', 'y': 'dead_pixels', line_width: 5, 'source': *dead_history_source} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Hot Pixels v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &hot_history_source + ref: "hot_history_source" + data: + time: !self.bad_historyxxxxx + hot_pixels: !self.bad_historyxxxxx + +- !Range1d: &hot_history_xrange + ref: "hot_history_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &hot_history_yrange + ref: "hot_history_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [-1, 1] + +- !Figure: &hot_history_figure + ref: "hot_history_figure" + x_axis_label: "Time (MJD)" + y_axis_label: "Number of HOT pixels" + x_range: *hot_history_xrange + y_range: *hot_history_yrange + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'hot_pixels', line_width: 5, 'source': *hot_history_source} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Low QE Pixels v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &lowqe_history_source + ref: "lowqe_history_source" + data: + time: !self.bad_historyxxxxx + lowqe_pixels: !self.bad_historyxxxxx + +- !Range1d: &lowqe_history_xrange + ref: "lowqe_history_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &lowqe_history_yrange + ref: "lowqe_history_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [-1, 1] + +- !Figure: &lowqe_history_figure + ref: "lowqe_history_figure" + x_axis_label: "Time (MJD)" + y_axis_label: "Number of LOW QE pixels" + x_range: *lowqe_history_xrange + y_range: *lowqe_history_yrange + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'lowqe_pixels', line_width: 5, 'source': *lowqe_history_source} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Open Pixels v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &open_history_source + ref: "open_history_source" + data: + time: !self.bad_historyxxxxx + lowqe_pixels: !self.bad_historyxxxxx + +- !Range1d: &open_history_xrange + ref: "open_history_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &open_history_yrange + ref: "open_history_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [-1, 1] + +- !Figure: &open_history_figure + ref: "open_history_figure" + x_axis_label: "Time (MJD)" + y_axis_label: "Number of OPEN pixels" + x_range: *open_history_xrange + y_range: *open_history_yrange + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'open_pixels', line_width: 5, 'source': *open_history_source} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Open Adj Pixels v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &adj_open_history_source + ref: "adj_open_history_source" + data: + time: !self.bad_historyxxxxx + lowqe_pixels: !self.bad_historyxxxxx + +- !Range1d: &adj_open_history_xrange + ref: "adj_open_history_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &adj_open_history_yrange + ref: "adj_open_history_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [-1, 1] + +- !Figure: &adj_open_history_figure + ref: "adj_open_history_figure" + x_axis_label: "Time (MJD)" + y_axis_label: "Number of OPEN ADJACENT pixels" + x_range: *adj_open_history_xrange + y_range: *adj_open_history_yrange + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'adj_open_pixels', line_width: 5, 'source': *adj_open_history_source} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# RC Pixels v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &rc_history_source + ref: "rc_history_source" + data: + time: !self.bad_historyxxxxx + lowqe_pixels: !self.bad_historyxxxxx + +- !Range1d: &rc_history_xrange + ref: "rc_history_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &rc_history_yrange + ref: "rc_history_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [-1, 1] + +- !Figure: &rc_history_figure + ref: "rc_history_figure" + x_axis_label: "Time (MJD)" + y_axis_label: "Number of RC pixels" + x_range: *rc_history_xrange + y_range: *rc_history_yrange + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'rc_pixels', line_width: 5, 'source': *rc_history_source} + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Other Bad Pixels v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &other_bad_pixel_history_source + ref: "other_bad_pixel_history_source" + data: + time: !self.bad_historyxxxxx + lowqe_pixels: !self.bad_historyxxxxx + +- !Range1d: &other_bad_pixel_history_xrange + ref: "other_bad_pixel_history_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &other_bad_pixel_history_yrange + ref: "other_bad_pixel_history_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [-1, 1] + +- !Figure: &other_bad_pixel_history_figure + ref: "other_bad_pixel_history_figure" + x_axis_label: "Time (MJD)" + y_axis_label: "Number of OTHER BAD pixels" + x_range: *other_bad_pixel_history_xrange + y_range: *other_bad_pixel_history_yrange + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'other_bad_pixel_pixels', line_width: 5, 'source': *other_bad_pixel_history_source} + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# Telegraph Pixels v. Time Figure +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +- !ColumnDataSource: &telegraph_history_source + ref: "telegraph_history_source" + data: + time: !self.bad_historyxxxxx + lowqe_pixels: !self.bad_historyxxxxx + +- !Range1d: &telegraph_history_xrange + ref: "telegraph_history_xrange" + #start: 0 + #end: 1 + #bounds: 'auto' #!!python/tuple [0, 1] + +- !Range1d: &telegraph_history_yrange + ref: "telegraph_history_yrange" + #start: 0 + #end: 1 + #bounds: !!python/tuple [-1, 1] + +- !Figure: &telegraph_history_figure + ref: "telegraph_history_figure" + x_axis_label: "Time (MJD)" + y_axis_label: "Number of TELEGRAPH pixels" + x_range: *telegraph_history_xrange + y_range: *telegraph_history_yrange + elements: + - {'kind': 'circle', 'x': 'time', 'y': 'telegraph_pixels', line_width: 5, 'source': *telegraph_history_source} + + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Dark Histogram Figure @@ -65,12 +279,14 @@ # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Mean Dark Image Figure # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- !ColumnDataSource: &mean_dark_source - ref: "mean_dark_source" +- !ColumnDataSource: &bkgd_image + ref: "bkgd_image" data: dh: [1] dw: [1] image: [[[1,0], [0, 1]]] + dead_pixels_x: !self.bad_latestxxxxx + dead_pixels_y: !self.bad_latestxxxxx - !Range1d: &stamp_xr ref: "stamp_xr" #start: 0 @@ -83,25 +299,25 @@ #bounds: !!python/tuple [0, 1] - !LogColorMapper: &log_mapper ref: "log_mapper" - palette: "Viridis256" + palette: "Greys256" low: 0. high: 1. -- !ColorBar: &mean_dark_cbar - ref: "mean_dark_cbar" - color_mapper: *log_mapper - location: !!python/tuple [0, 0] -- !Figure: &mean_dark_image_figure - ref: "mean_dark_image_figure" +- !Figure: &badpix_image_figure + ref: "badpix_image_figure" x_axis_label: "Col = SIAF det Y" y_axis_label: "Row = SIAF det X" x_range: *stamp_xr y_range: *stamp_yr tools: "" - height: 250 # Not working - width: 250 # Not working + height: 2048 # Not working + width: 2048 # Not working elements: - - {"kind": "image", "image": "image", "x": 0, "y": 0, "dh": 'dh', "dw": 'dh', "source": *mean_dark_source, "color_mapper": *log_mapper} - - {"kind": "layout", "obj": *mean_dark_cbar, "place": "right"} + - {"kind": "image", "image": "image", "x": 0, "y": 0, "dh": 'dh', "dw": 'dw', "source": *bkgd_image, "color_mapper": *log_mapper} + - {'kind': 'circle', 'x': 'dead_pixels_x', 'y': 'dead_pixels_y', line_width: 5, 'source': *other_bad_pixel__history_source} + - {'kind': 'circle', 'x': 'hot_pixels_x', 'y': 'hot_pixels_y', line_width: 5, 'source': *other_bad_pixel__history_source} + - {'kind': 'circle', 'x': 'open_pixels_x', 'y': 'open_pixels_y', line_width: 5, 'source': *other_bad_pixel__history_source} + - {'kind': 'circle', 'x': 'adjopen_pixels_x', 'y': 'adjopen_pixels_y', line_width: 5, 'source': *other_bad_pixel__history_source} + #- !Document: # - !column: From b54ba7c6b6a7863fe22d8c657854ef980c387901 Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Wed, 19 Aug 2020 15:37:20 -0400 Subject: [PATCH 048/227] Making changes suggested by Matt --- .../apps/jwql/templates/api_landing.html | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index 9a57b5e30..29dd18e6e 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -15,7 +15,7 @@

    Explore the JWQL API

  • What is the JWQL API?
  • Examples
  • List of Avaliable Services
    • -
  • Documentation
    • +
  • Documentation


    • @@ -51,8 +51,8 @@
    Obtaining Filenames Associated with a Rootname
    # Build the URL https://dljwql.stsci.edu/api/roontname/service/ url = 'https://dljwql.stsci.edu/api/jw93025001001_02102_00001_nrca2/filenames/' - r = request.urlopen(url) - filenames = json.loads(r.read().decode()) + req = request.urlopen(url) + filenames = json.loads(req.read().decode()) print(filenames) @@ -64,7 +64,7 @@
    Obtaining Filenames Associated with a Rootname
    -
    Obtaining Filenames Associated with Proposal
    +
    Obtaining Thumbnails with Instrument Name

    A user is interested in all of the thumbnail associated with the Fine Guider Sensors (FGS).

    @@ -75,8 +75,8 @@
    Obtaining Filenames Associated with Proposal
    # Build the URL https://dljwql.stsci.edu/api/instrument/service/ url = 'https://dljwql.stsci.edu/api/fgs/thumbnails/' - r = request.urlopen(url) - thumbnails = json.loads(r.read().decode()) + req = request.urlopen(url) + thumbnails = json.loads(req.read().decode()) print(thumbnails) @@ -96,7 +96,7 @@
    Obtaining Filenames Associated with Proposal
    -
    Obtaining Filenames Associated with Proposal
    +
    Obtaining Filenames with Proposal ID

    A user is interested in all of the thumbnail associated with the Fine Guider Sensors (FGS).

    @@ -107,8 +107,8 @@
    Obtaining Filenames Associated with Proposal
    # Build the URL https://dljwql.stsci.edu/api/proposal/service/ url = 'https://dljwql.stsci.edu/api/86600/filenames/' - r = request.urlopen(url) - filenames = json.loads(r.read().decode()) + req = request.urlopen(url) + filenames = json.loads(req.read().decode()) print(filenames) @@ -137,8 +137,8 @@
    Obtaining Filenames Associated with Proposal

    List of Available Services

    • All Proposals
      • -
    • Instrument Proposals
      • -
    • Preview Images by Instrument
      • +
    • Instrument Proposals (https://dljwql.stsci.edu/api/<instrument>/proposals/)
      • +
    • Preview Images by Instrument ()
    • Thumbnails by Instrument
    • Filenames by Proposal
    • Preview Images by Proposal
      • From 27228e729fe30e7c7da35df585e6c3f87bfae6d9 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 19 Aug 2020 18:38:08 -0400 Subject: [PATCH 049/227] Add env file for python 3.8. Add 3.8 to Jenkins --- Jenkinsfile | 2 +- environment_python_3_8.yml | 40 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 environment_python_3_8.yml diff --git a/Jenkinsfile b/Jenkinsfile index 7fc95a95d..180f6303d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,7 +27,7 @@ if (utils.scm_checkout()) return // Establish OS and Python version variables for the matrix matrix_os = ["linux-stable"] // (Note that Jenkins can only be run with Linux, not MacOSX/Windows) -matrix_python = ["3.6", "3.7"] +matrix_python = ["3.6", "3.7", "3.8"] // Set up the matrix of builds matrix = [] diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml new file mode 100644 index 000000000..4e497b142 --- /dev/null +++ b/environment_python_3_8.yml @@ -0,0 +1,40 @@ +channels: +- defaults +- http://ssb.stsci.edu/astroconda +dependencies: +- bokeh>=1.0,<1.4 +- django=3.1.0 +- flake8=3.8.3 +- inflection=0.5.0 +- ipython=7.17.0 +- jinja2=2.11.2 +- jsonschema=3.2.0 +- matplotlib=3.3.1 +- nodejs=14.8.0 +- numpy=1.19.1 +- numpydoc=1.1.0 +- pandas=1.1.0 +- pip=20.2.2 +- postgresql=12.3 +- psycopg2=2.8.5 +- python=3.8.5 +- pytest=6.0.1 +- pytest-cov=2.10.1 +- scipy=1.5.2 +- setuptools=49.6.0 +- sphinx=3.2.1 +- sqlalchemy=1.3.19 +- twine=3.2.0 +- pip: + - asdf==2.7.1 + - astropy==4.0.1.post1 + - astroquery=0.4.1 + - authlib==0.14.3 + - codecov==2.1.8 + - crds==7.5.0.0 + - jwedb==0.0.6 + - pysiaf=0.9.0 + - pysqlite3==0.4.3 + - stsci_rtd_theme==0.0.2 + - git+https://github.com/spacetelescope/jwst@0.16.2 + - git+https://github.com/spacetelescope/jwst_reffiles From fcbdfe16129699a6edfa7732bbeaea774eaf6444 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Wed, 19 Aug 2020 22:08:47 -0400 Subject: [PATCH 050/227] Back down versions that are only in conda-forge --- environment_python_3_8.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index 4e497b142..bbbedcd07 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -3,28 +3,28 @@ channels: - http://ssb.stsci.edu/astroconda dependencies: - bokeh>=1.0,<1.4 -- django=3.1.0 +- django=3.0.3 - flake8=3.8.3 -- inflection=0.5.0 +- inflection=0.3.1 - ipython=7.17.0 - jinja2=2.11.2 - jsonschema=3.2.0 -- matplotlib=3.3.1 -- nodejs=14.8.0 +- matplotlib=3.2.2 +- nodejs=10.13.0 - numpy=1.19.1 - numpydoc=1.1.0 - pandas=1.1.0 - pip=20.2.2 -- postgresql=12.3 +- postgresql=12.2 - psycopg2=2.8.5 - python=3.8.5 - pytest=6.0.1 - pytest-cov=2.10.1 -- scipy=1.5.2 +- scipy=1.5.0 - setuptools=49.6.0 - sphinx=3.2.1 - sqlalchemy=1.3.19 -- twine=3.2.0 +- twine=2.0.0 - pip: - asdf==2.7.1 - astropy==4.0.1.post1 From b49949994bb737bcf6629be3360fa9411b024352 Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 20 Aug 2020 08:22:42 -0400 Subject: [PATCH 051/227] back down ipython and numpy versions --- environment_python_3_8.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index bbbedcd07..02a3f40b0 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -6,12 +6,12 @@ dependencies: - django=3.0.3 - flake8=3.8.3 - inflection=0.3.1 -- ipython=7.17.0 +- ipython=7.16.1 - jinja2=2.11.2 - jsonschema=3.2.0 - matplotlib=3.2.2 - nodejs=10.13.0 -- numpy=1.19.1 +- numpy=1.18.5 - numpydoc=1.1.0 - pandas=1.1.0 - pip=20.2.2 From ea167e545241e6f38f9fe20a2c57e44ae601116d Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 20 Aug 2020 08:56:07 -0400 Subject: [PATCH 052/227] install sqlalchemy via pip --- environment_python_3_8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index 02a3f40b0..3cb69265a 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -23,7 +23,6 @@ dependencies: - scipy=1.5.0 - setuptools=49.6.0 - sphinx=3.2.1 -- sqlalchemy=1.3.19 - twine=2.0.0 - pip: - asdf==2.7.1 @@ -35,6 +34,7 @@ dependencies: - jwedb==0.0.6 - pysiaf=0.9.0 - pysqlite3==0.4.3 + - sqlalchemy=1.3.19 - stsci_rtd_theme==0.0.2 - git+https://github.com/spacetelescope/jwst@0.16.2 - git+https://github.com/spacetelescope/jwst_reffiles From 272f71e01a9d5fd702f94db93acdb85d134574fe Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 20 Aug 2020 10:15:06 -0400 Subject: [PATCH 053/227] Install Bokeh via pip --- environment_python_3_8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index 3cb69265a..673a46638 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -2,7 +2,6 @@ channels: - defaults - http://ssb.stsci.edu/astroconda dependencies: -- bokeh>=1.0,<1.4 - django=3.0.3 - flake8=3.8.3 - inflection=0.3.1 @@ -29,6 +28,7 @@ dependencies: - astropy==4.0.1.post1 - astroquery=0.4.1 - authlib==0.14.3 + - bokeh>=1.0,<1.4 - codecov==2.1.8 - crds==7.5.0.0 - jwedb==0.0.6 From ae701695e7a78bf438712999348aea8350e722ca Mon Sep 17 00:00:00 2001 From: Bryan Hilbert Date: Thu, 20 Aug 2020 10:40:41 -0400 Subject: [PATCH 054/227] correct equal sign typos --- environment_python_3_8.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environment_python_3_8.yml b/environment_python_3_8.yml index 673a46638..18f6c89ae 100644 --- a/environment_python_3_8.yml +++ b/environment_python_3_8.yml @@ -26,15 +26,15 @@ dependencies: - pip: - asdf==2.7.1 - astropy==4.0.1.post1 - - astroquery=0.4.1 + - astroquery==0.4.1 - authlib==0.14.3 - bokeh>=1.0,<1.4 - codecov==2.1.8 - crds==7.5.0.0 - jwedb==0.0.6 - - pysiaf=0.9.0 + - pysiaf==0.9.0 - pysqlite3==0.4.3 - - sqlalchemy=1.3.19 + - sqlalchemy==1.3.19 - stsci_rtd_theme==0.0.2 - git+https://github.com/spacetelescope/jwst@0.16.2 - git+https://github.com/spacetelescope/jwst_reffiles From c25bbb94950d7b25b74fdd111e76f409b458b4ba Mon Sep 17 00:00:00 2001 From: Mees Fix Date: Thu, 20 Aug 2020 12:50:02 -0400 Subject: [PATCH 055/227] Adding api calls to list of available services --- .../apps/jwql/templates/api_landing.html | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index 29dd18e6e..7b72c1da2 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -136,16 +136,17 @@
      Obtaining Filenames with Proposal ID

      List of Available Services

        -
      • All Proposals
        • -
      • Instrument Proposals (https://dljwql.stsci.edu/api/<instrument>/proposals/)
        • -
      • Preview Images by Instrument ()
        • -
      • Thumbnails by Instrument
        • -
      • Filenames by Proposal
        • -
      • Preview Images by Proposal
        • -
      • Filenames by Rootname
        • -
      • Preview Images by Rootname
        • -
      • Thumbnails by Rootname
        • +
      • All Proposals (https://dljwql.stsci.edu/api/proposals/)
        • +
      • Instrument Proposals (https://dljwql.stsci.edu/api/<proposal>/proposals/)
        • +
      • Preview Images by Instrument (https://dljwql.stsci.edu/api/<instrument>/preview_images/)
        • +
      • Thumbnails by Instrument (https://dljwql.stsci.edu/api/<instrument>/thumbnails/)
        • +
      • Filenames by Proposal (https://dljwql.stsci.edu/api/<instrument>/proposals/)
        • +
      • Preview Images by Proposal (https://dljwql.stsci.edu/api/<instrument>/preview_images/)
        • +
      • Filenames by Rootname (https://dljwql.stsci.edu/api/<rootname>/filenames/)
        • +
      • Preview Images by Rootname (https://dljwql.stsci.edu/api/<rootname>/preview_images/)
        • +
      • Thumbnails by Rootname (https://dljwql.stsci.edu/api/<rootname>/thumbnails/)
      +

      Where <instrument>, <rootname>, <proposal> are the values for the instrument name, rootname and proposal ID respectivly

      From 683fea482e0787ac1aeb435d1cc4c6e540ef2185 Mon Sep 17 00:00:00 2001 From: Matthew Bourque Date: Mon, 24 Aug 2020 11:08:35 -0400 Subject: [PATCH 056/227] Fixed "" and "" tag --- .../apps/jwql/templates/api_landing.html | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/jwql/website/apps/jwql/templates/api_landing.html b/jwql/website/apps/jwql/templates/api_landing.html index 7b72c1da2..eae00ccc8 100644 --- a/jwql/website/apps/jwql/templates/api_landing.html +++ b/jwql/website/apps/jwql/templates/api_landing.html @@ -19,7 +19,7 @@

      Explore the JWQL API


    - +

    What is the JWQL API?

    @@ -29,7 +29,7 @@

    What is the JWQL API?

    provided a menu containing all of the drink and food options. But how do you communicate your food and drink order to the kitchen staff? This is where the waiter (API) comes in, you provide the waiter with your order (request), they will communicate your order to the kitchen staff (system), and bring your food and drinks when they are prepared (the response). The waiter (API) disguises - all of the steps that are necessary to getting your final product delivered (cooking the food, counting stock, knowing recipes, etc.) but, is the + all of the steps that are necessary to getting your final product delivered (cooking the food, counting stock, knowing recipes, etc.) but, is the link to the user and the response they wish to obtain from a system.

    @@ -50,8 +50,8 @@
    Obtaining Filenames Associated with a Rootname
    import json # Build the URL https://dljwql.stsci.edu/api/roontname/service/ - url = 'https://dljwql.stsci.edu/api/jw93025001001_02102_00001_nrca2/filenames/' - req = request.urlopen(url) + url = 'https://dljwql.stsci.edu/api/jw93025001001_02102_00001_nrca2/filenames/' + req = request.urlopen(url) filenames = json.loads(req.read().decode()) print(filenames) @@ -66,7 +66,7 @@
    Obtaining Filenames Associated with a Rootname
    Obtaining Thumbnails with Instrument Name

    - A user is interested in all of the thumbnail associated with the Fine Guider Sensors (FGS). + A user is interested in all of the thumbnail associated with the Fine Guider Sensors (FGS). 

                         
@@ -75,7 +75,7 @@ 
    Obtaining Thumbnails with Instrument Name
    
 
                     # Build the URL https://dljwql.stsci.edu/api/instrument/service/
                     url = 'https://dljwql.stsci.edu/api/fgs/thumbnails/'
-                    req = request.urlopen(url) 
+                    req = request.urlopen(url)
                     thumbnails = json.loads(req.read().decode())
 
                     print(thumbnails)
@@ -98,20 +98,20 @@ 
    Obtaining Thumbnails with Instrument Name
    
 
                     
    Obtaining Filenames with Proposal ID 
    
                     
    
-                        A user is interested in all of the thumbnail associated with the Fine Guider Sensors (FGS).  
+                        A user is interested in all of the thumbnail associated with the Fine Guider Sensors (FGS).
                     
    
                     
                             
                         from urllib import request
                         import json
-    
+
                         # Build the URL https://dljwql.stsci.edu/api/proposal/service/
                         url = 'https://dljwql.stsci.edu/api/86600/filenames/'
-                        req = request.urlopen(url) 
+                        req = request.urlopen(url)
                         filenames = json.loads(req.read().decode())
-    
+
                         print(filenames)
-    
+
                         {'filenames': ['jw86600001001_02101_00001_guider1_cal.fits',
                                        'jw86600001001_02101_00001_guider1_rate.fits',
                                        'jw86600001001_02101_00001_guider1_trapsfilled.fits',
@@ -137,11 +137,11 @@ 
    Obtaining Filenames with Proposal ID 
    
             
    List of Available Services