Rcal 300 Updates to check for CRDS return value for each ELPP step (#…

…1055)

Co-authored-by: Eddie Schlafly <[email protected]>
Co-authored-by: Zach Burnett <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

CHANGES.rst

@@ -11,11 +11,20 @@ jump detection
 
 - Added uneven ramp-jump detection docs. [#1035]
 
+
 documentation
 -------------
 
+- Update jump step docs [#1035]
+
 - added user documentation for ``roman_static_preview`` script [#1046]
 
+general
+-------
+
+- Update elp steps to check for CRDS not returning a reference file [#1055]
+
+
 0.13.0 (2023-11-28)
 ===================
 

romancal/assign_wcs/assign_wcs_step.py

@@ -32,6 +32,14 @@ def process(self, input):
             for reftype in self.reference_file_types:
                 log.info(f"reftype, {reftype}")
                 reffile = self.get_reference_file(input_model, reftype)
+                # Check for a valid reference file
+                if reffile == "N/A":
+                    self.log.warning("No DISTORTION reference file found")
+                    self.log.warning("Assign WCS step will be skipped")
+                    result = input_model.copy()
+                    result.meta.cal_step.assign_wcs = "SKIPPED"
+                    return result
+
                 reference_file_names[reftype] = reffile if reffile else ""
             log.info("Using reference files: %s for assign_wcs", reference_file_names)
             result = load_wcs(input_model, reference_file_names)

romancal/assign_wcs/tests/test_wcs.py

@@ -1,5 +1,3 @@
-import os
-
 import numpy as np
 import pytest
 from gwcs.wcstools import grid_from_bounding_box
@@ -43,11 +41,6 @@ def load_wcs_step(image, file_name):
         return load_wcs(image, {"distortion": file_name})
 
     def assign_wcs_step(image, file_name):
-        if os.environ.get("CI") == "true":
-            pytest.skip(
-                "Roman CRDS servers are not currently available outside the internal"
-                " network"
-            )
         return AssignWcsStep.call(image, override_distortion=file_name)
 
     return [load_wcs_step, assign_wcs_step]
@@ -95,3 +88,11 @@ def test_wcs(tmpdir, distortion, step):
     ]
     assert_allclose(s_region_alpha_list, l2_wcs.meta.wcsinfo.ra_ref, rtol=1e-1, atol=0)
     assert_allclose(s_region_delta_list, l2_wcs.meta.wcsinfo.dec_ref, rtol=1e-1, atol=0)
+
+
+@pytest.mark.parametrize("step", create_step())
+def test_crds_getbestref(step):
+    l2im = create_image()
+    l2_wcs = AssignWcsStep.call(l2im, override_distortion="N/A")
+
+    assert l2_wcs.meta.cal_step.assign_wcs == "SKIPPED"

romancal/dark_current/dark_current_step.py

@@ -1,6 +1,6 @@
 #! /usr/bin/env python
 
-from roman_datamodels import datamodels as rdd
+from roman_datamodels import datamodels as rdm
 
 from romancal.stpipe import RomanStep
 
@@ -21,13 +21,21 @@ class DarkCurrentStep(RomanStep):
 
     def process(self, input):
         # Open the input data model
-        with rdd.open(input, lazy_load=False) as input_model:
+        with rdm.open(input, lazy_load=False) as input_model:
             # Get the name of the dark reference file to use
             self.dark_name = self.get_reference_file(input_model, "dark")
+            # Check for a valid reference file
+            if self.dark_name == "N/A":
+                self.log.warning("No DARK reference file found")
+                self.log.warning("Dark current step will be skipped")
+                result = input_model
+                result.meta.cal_step.dark = "SKIPPED"
+                return result
+
             self.log.info("Using DARK reference file: %s", self.dark_name)
 
             # Open dark model
-            dark_model = rdd.open(self.dark_name)
+            dark_model = rdm.open(self.dark_name)
 
             # Temporary patch to utilize stcal dark step until MA table support
             # is fully implemented

romancal/dark_current/tests/test_dark.py

@@ -2,7 +2,6 @@
 Unit tests for dark current correction
 """
 
-import os
 
 import numpy as np
 import pytest
@@ -20,12 +19,6 @@
         ("WFI", "WFI_IMAGE"),
     ],
 )
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason=(
-        "Roman CRDS servers are not currently available outside the internal network"
-    ),
-)
 def test_dark_step_interface(instrument, exptype):
     """Test that the basic inferface works for data requiring a DARK reffile"""
 
@@ -57,12 +50,6 @@ def test_dark_step_interface(instrument, exptype):
         ("WFI", "WFI_IMAGE"),
     ],
 )
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason=(
-        "Roman CRDS servers are not currently available outside the internal network"
-    ),
-)
 def test_dark_step_subtraction(instrument, exptype):
     """Test that the values in a dark reference file are properly subtracted"""
 
@@ -97,12 +84,6 @@ def test_dark_step_subtraction(instrument, exptype):
         ("WFI", "WFI_IMAGE"),
     ],
 )
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason=(
-        "Roman CRDS servers are not currently available outside the internal network"
-    ),
-)
 def test_dark_step_output_dark_file(tmpdir, instrument, exptype):
     """Test that the the step can output a proper (optional) dark file"""
     path = str(tmpdir / "dark_out.asdf")
@@ -126,6 +107,39 @@ def test_dark_step_output_dark_file(tmpdir, instrument, exptype):
     assert dark_out_file_model.dq.shape == shape[1:]
 
 
+@pytest.mark.parametrize(
+    "instrument, exptype",
+    [
+        ("WFI", "WFI_IMAGE"),
+    ],
+)
+def test_dark_step_getbestrefs(tmpdir, instrument, exptype):
+    """Test that the the step will skip if CRDS returns N/A for the ref file"""
+    path = str(tmpdir / "dark_out.asdf")
+
+    # Set test size
+    shape = (2, 20, 20)
+
+    # Create test ramp and dark models
+    ramp_model, darkref_model = create_ramp_and_dark(shape, instrument, exptype)
+
+    # Perform Dark Current subtraction step with override = N/A
+    result = DarkCurrentStep.call(ramp_model, override_dark="N/A")
+    assert result.meta.cal_step.dark == "SKIPPED"
+
+    # Perform Dark Current subtraction step
+    DarkCurrentStep.call(ramp_model, override_dark=darkref_model, dark_output=path)
+
+    # Open dark file
+    dark_out_file_model = rdm.open(path)
+
+    # Test dark file results
+    assert type(dark_out_file_model) == DarkRefModel
+    assert dark_out_file_model.validate() is None
+    assert dark_out_file_model.data.shape == shape
+    assert dark_out_file_model.dq.shape == shape[1:]
+
+
 def create_ramp_and_dark(shape, instrument, exptype):
     """Helper function to create test ramp and dark models"""
 

romancal/dq_init/dq_init_step.py

@@ -80,7 +80,7 @@ def process(self, input):
         reference_file_name = self.get_reference_file(output_model, "mask")
 
         # Test for reference file
-        if reference_file_name is not None:
+        if reference_file_name != "N/A" and reference_file_name is not None:
             # If there are mask files, perform dq step
             # Open the relevant reference files as datamodels
             reference_file_model = rdm.open(reference_file_name)

romancal/dq_init/tests/test_dq_init.py

@@ -1,5 +1,3 @@
-import os
-
 import numpy as np
 import pytest
 from astropy import units as u
@@ -191,12 +189,6 @@ def test_dq_add1_groupdq():
         ("WFI", "WFI_IMAGE"),
     ],
 )
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason=(
-        "Roman CRDS servers are not currently available outside the internal network"
-    ),
-)
 def test_dqinit_step_interface(instrument, exptype):
     """Test that the basic inferface works for data requiring a DQ reffile"""
 
@@ -246,12 +238,6 @@ def test_dqinit_step_interface(instrument, exptype):
         ("WFI", "WFI_IMAGE"),
     ],
 )
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason=(
-        "Roman CRDS servers are not currently available outside the internal network"
-    ),
-)
 def test_dqinit_refpix(instrument, exptype):
     """Test that the basic inferface works for data requiring a DQ reffile"""
 
@@ -296,3 +282,33 @@ def test_dqinit_refpix(instrument, exptype):
     assert result.dq_border_ref_pix_left.shape == (20, 4)
     assert result.dq_border_ref_pix_top.shape == (4, 20)
     assert result.dq_border_ref_pix_bottom.shape == (4, 20)
+
+
+@pytest.mark.parametrize(
+    "instrument, exptype",
+    [
+        ("WFI", "WFI_IMAGE"),
+    ],
+)
+def test_dqinit_getbestref(instrument, exptype):
+    """Test that the step is skipped if the CRDS reffile is N/A"""
+
+    # Set test size
+    shape = (2, 20, 20)
+
+    # Create test science raw model
+    wfi_sci_raw = maker_utils.mk_level1_science_raw(shape=shape)
+    wfi_sci_raw.meta.instrument.name = instrument
+    wfi_sci_raw.meta.instrument.detector = "WFI01"
+    wfi_sci_raw.meta.instrument.optical_element = "F158"
+    wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
+    wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
+    wfi_sci_raw.meta.exposure.type = exptype
+    wfi_sci_raw.data = u.Quantity(
+        np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16
+    )
+    wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw)
+
+    # Perform Data Quality application step
+    result = DQInitStep.call(wfi_sci_raw_model, override_mask="N/A")
+    assert result.meta.cal_step.dq_init == "SKIPPED"

romancal/flatfield/tests/test_flatfield.py

@@ -1,5 +1,3 @@
-import os
-
 import numpy as np
 import pytest
 from astropy import units as u
@@ -18,12 +16,6 @@
         ("WFI", "WFI_IMAGE"),
     ],
 )
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason=(
-        "Roman CRDS servers are not currently available outside the internal network"
-    ),
-)
 def test_flatfield_step_interface(instrument, exptype):
     """Test that the basic inferface works for data requiring a FLAT reffile"""
 
@@ -68,19 +60,18 @@ def test_flatfield_step_interface(instrument, exptype):
     assert result.var_flat.shape == shape
     assert result.meta.cal_step.flat_field == "COMPLETE"
 
+    # test that the step is skipped if the reference file is N/A
+    result = FlatFieldStep.call(wfi_image_model, override_flat="N/A")
+
+    assert result.meta.cal_step.flat_field == "SKIPPED"
+
 
 @pytest.mark.parametrize(
     "instrument, exptype",
     [
         ("WFI", "WFI_IMAGE"),
     ],
 )
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason=(
-        "Roman CRDS servers are not currently available outside the internal network"
-    ),
-)
 def test_crds_temporal_match(instrument, exptype):
     """Test that the basic inferface works for data requiring a FLAT reffile"""
 
@@ -119,13 +110,6 @@ def test_crds_temporal_match(instrument, exptype):
         "WFI_PRISM",
     ],
 )
-@pytest.mark.skipif(
-    os.environ.get("CI") == "true",
-    reason=(
-        "Roman CRDS servers are not currently available outside the internal network"
-    ),
-)
-# Test that spectroscopic exposure types will skip flat field step
 def test_spectroscopic_skip(instrument, exptype):
     wfi_image = maker_utils.mk_level2_image()
     wfi_image.meta.instrument.name = instrument