Merge branch 'develop' into dark-mon-use-django-db

.github/workflows/ci.yml

@@ -29,20 +29,21 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.9, "3.10"]
+        python-version: ["3.10"]
 
     steps:
       - uses: actions/checkout@v4
 
-      - uses: mamba-org/provision-with-micromamba@v15
+      - uses: mamba-org/setup-micromamba@v1
         with:
           environment-file: ./environment_python_${{ matrix.python-version }}.yml
-          cache-env: true
+          cache-environment: true
           cache-downloads: true
+          init-shell: bash
 
       - run: pip install -e .[test]
 
-      - run: conda env export
+      - run: micromamba env export
 
       - uses: supercharge/[email protected]
         with:

.readthedocs.yaml

@@ -12,7 +12,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.9"
+    python: "3.11"
   jobs:
     post_install:
       - pip install sqlalchemy==1.4.46

CHANGES.rst

@@ -1,5 +1,75 @@
 ## What's Changed
 
+1.2.8 (2024-04-18)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- final model define for faking by @BradleySappington
+
+
+1.2.7 (2024-04-18)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Import monitor models by @bhilbert4
+
+
+1.2.6 (2024-04-15)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Update NIRCam Background Monitor plots to handle new Claw Monitor columns by @bsunnquist
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- add default to read_patt_num by @BradleySappington
+- migration work for next RC by @BradleySappington
+- Fix database empty fields by @BradleySappington
+
+
+1.2.5 (2024-03-19)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Fix Bokeh `file_html` Call by @mfixstsci
+- Update Bad Pix Exclude Line by @mfixstsci
+- Interactive preview image - updates for Bokeh 3 by @bhilbert4
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Allow creation of pngs from 3D and 4D arrays by @bhilbert4
+- Add max length to charfield by @BradleySappington
+- Header fix by @BradleySappington
+
+
+1.2.4 (2024-03-11)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+- Fix Broken Dashboard - ReImplement Anomaly Dash @BradleySappington
+- Add more info to image- and exposure-level pages @bhilbert4
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- create pull_jwql_branch.sh by @BradleySappington
+- Claw monitor migrate by @york-stsci
+- Update readnoise monitor to use django database models by @york-stsci
+- Migration to django management by @york-stsci
+- Get subarray lists from datamodels schema by @bhilbert4
+- Retrieve program number and category efficiently by @bhilbert4
+
+
 1.2.3 (2024-01-26)
 ==================
 

README.md

@@ -44,7 +44,7 @@ Getting `jwql` up and running on your own computer requires four steps, detailed
 
 ### Prerequisites
 
-It is highly suggested that contributors have a working installation of `anaconda` or `miniconda` for Python 3.9+.  Downloads and installation instructions are  available here:
+It is highly suggested that contributors have a working installation of `anaconda` or `miniconda` for Python 3.10+.  Downloads and installation instructions are  available here:
 
 - [Miniconda](https://conda.io/miniconda.html)
 - [Anaconda](https://www.continuum.io/downloads)
@@ -86,13 +86,13 @@ source activate base/root
 Lastly, create the `jwql` environment via one of the `environment.yml` files (currently `environment_python_3.9.yml`, for python 3.9, and `environment_python_3.10.yml`, for python 3.10, are supported by `jwql`):
 
 ```
-conda env create -f environment_python_3.9.yml
+conda env create -f environment_python_3.10.yml
 ```
 
 or
 
 ```
-conda env create -f environment_python_3.10.yml
+conda env create -f environment_python_3.11.yml
 ```
 
 ### Configuration File

environment_python_3.10.yml

@@ -45,7 +45,8 @@ dependencies:
   - pytest-mock=3.11.1
   - python=3.10.12
   - pyyaml=6.0.1
-  - redis=5.0.0
+  - redis-py=5.0.0
+  - redis-server=7.2.4
   - ruff=0.0.292
   - scipy=1.9.3
   - setuptools=68.2.2
@@ -63,7 +64,6 @@ dependencies:
     - pysiaf==0.20.0
     - pysqlite3==0.5.2
     - pyvo==1.4.2
-    - redis==5.0.0
     - selenium==4.13.0
     - stdatamodels==1.8.3
     - stsci_rtd_theme==1.0.0

jwql/example_config.json

@@ -28,6 +28,7 @@
             "PORT" : ""
         }
     },
+    "django_debug" : false,
     "jwql_dir" : "",
     "jwql_version": "",
     "server_type": "",

jwql/instrument_monitors/nircam_monitors/claw_monitor.py

@@ -192,6 +192,7 @@ def make_background_plots(self, plot_type='bkg'):
                     df = df[df['stddev'] != 0]  # older data has no accurate stddev measures
                     plot_data = df['stddev'].values
                 if plot_type == 'model':
+                    df = df[np.isfinite(df['total_bkg'])]  # the claw monitor did not track model measurements at first
                     plot_data = df['median'].values / df['total_bkg'].values
                 plot_expstarts = df['expstart_mjd'].values
 
@@ -300,7 +301,11 @@ def process(self):
 
                 # Get predicted background level using JWST background tool
                 ra, dec = hdu[1].header['RA_V1'], hdu[1].header['DEC_V1']
-                wv = self.filter_wave[self.fltr.upper()]
+                if ('N' in self.pupil.upper()) | ('M' in self.pupil.upper()):
+                    fltr_wv = self.pupil.upper()
+                else:
+                    fltr_wv = self.fltr.upper()
+                wv = self.filter_wave[fltr_wv]
                 date = hdu[0].header['DATE-BEG']
                 doy = int(Time(date).yday.split(':')[1])
                 try:
@@ -332,7 +337,7 @@ def process(self):
                                  'skyflat_filename': os.path.basename(self.outfile),
                                  'doy': float(doy),
                                  'total_bkg': float(total_bkg),
-                                 'entry_date': datetime.datetime.now()
+                                 'entry_date': datetime.datetime.now(datetime.timezone.utc)
                                  }
                 entry = self.stats_table(**claw_db_entry)
                 entry.save()
@@ -423,11 +428,13 @@ def run(self):
         mast_table = self.query_mast()
         logging.info('{} files found between {} and {}.'.format(len(mast_table), self.query_start_mjd, self.query_end_mjd))
 
-        # Define pivot wavelengths
-        self.filter_wave = {'F070W': 0.704, 'F090W': 0.902, 'F115W': 1.154, 'F150W': 1.501, 'F150W2': 1.659,
-                            'F200W': 1.989, 'F212N': 2.121, 'F250M': 2.503, 'F277W': 2.762, 'F300M': 2.989,
-                            'F322W2': 3.232, 'F356W': 3.568, 'F410M': 4.082, 'F430M': 4.281, 'F444W': 4.408,
-                            'F480M': 4.874}
+        # Define pivot wavelengths - last downloaded March 8 2024 from:
+        # https://jwst-docs.stsci.edu/jwst-near-infrared-camera/nircam-instrumentation/nircam-filters
+        self.filter_wave = {'F070W': 0.704, 'F090W': 0.901, 'F115W': 1.154, 'F140M': 1.404, 'F150W': 1.501, 'F162M': 1.626, 'F164N': 1.644,
+                            'F150W2': 1.671, 'F182M': 1.845, 'F187N': 1.874, 'F200W': 1.99, 'F210M': 2.093, 'F212N': 2.12, 'F250M': 2.503,
+                            'F277W': 2.786, 'F300M': 2.996, 'F322W2': 3.247, 'F323N': 3.237, 'F335M': 3.365, 'F356W': 3.563, 'F360M': 3.621,
+                            'F405N': 4.055, 'F410M': 4.092, 'F430M': 4.28, 'F444W': 4.421, 'F460M': 4.624, 'F466N': 4.654, 'F470N': 4.707,
+                            'F480M': 4.834}
 
         # Create observation-level median stacks for each filter/pupil combo, in pixel-space
         combos = np.array(['{}_{}_{}_{}'.format(str(row['program']), row['observtn'], row['filter'], row['pupil']).lower() for row in mast_table])
@@ -469,7 +476,7 @@ def run(self):
                      'start_time_mjd': self.query_start_mjd,
                      'end_time_mjd': self.query_end_mjd,
                      'run_monitor': monitor_run,
-                     'entry_date': datetime.datetime.now()}
+                     'entry_date': datetime.datetime.now(datetime.timezone.utc)}
         entry = self.query_table(**new_entry)
         entry.save()