From 27195a38bd730b33f9c701a8e39aabab4ee17df2 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Mon, 7 Oct 2024 11:39:07 -0400
Subject: [PATCH 01/74] Remove units from MOS pipeline.

---
 romancal/flux/flux_step.py                    | 14 +--
 .../outlier_detection/outlier_detection.py    | 23 ++---
 romancal/regtest/test_mos_pipeline.py         | 95 ++++++++++++++++++-
 romancal/resample/gwcs_drizzle.py             |  2 +-
 romancal/resample/resample.py                 | 35 +++----
 romancal/resample/resample_utils.py           |  2 +-
 romancal/skymatch/skymatch.py                 |  8 +-
 romancal/skymatch/skymatch_step.py            |  4 +-
 romancal/skymatch/skystatistics.py            |  6 +-
 romancal/source_catalog/detection.py          |  3 -
 romancal/source_catalog/source_catalog.py     | 43 +++------
 11 files changed, 142 insertions(+), 93 deletions(-)

diff --git a/romancal/flux/flux_step.py b/romancal/flux/flux_step.py
index a4624ce96..ca7fc5cce 100644
--- a/romancal/flux/flux_step.py
+++ b/romancal/flux/flux_step.py
@@ -104,27 +104,19 @@ def apply_flux_correction(model):
     DATA = ("data", "err")
     VARIANCES = ("var_rnoise", "var_poisson", "var_flat")
 
-    if model.data.unit == model.meta.photometry.conversion_megajanskys.unit:
+    if model.meta.cal_step["flux"] == "COMPLETE":
         message = (
-            f"Input data is already in flux units of {model.meta.photometry.conversion_megajanskys.unit}."
+            f"Input data is already in flux units of MJy/sr."
             "\nFlux correction already applied."
         )
         log.info(message)
         return
 
-    if model.data.unit != LV2_UNITS:
-        message = (
-            f"Input data units {model.data.unit} are not in the expected units of {LV2_UNITS}"
-            "\nAborting flux correction"
-        )
-        log.error(message)
-        raise ValueError(message)
-
     # Apply the correction.
     # The end goal in units is to have MJy/sr. The scale is in MJy/sr also.
     # Hence the extra factor of s/DN must be applied to cancel DN/s.
     log.debug("Flux correction being applied")
-    c_mj = model.meta.photometry.conversion_megajanskys / model.data.unit
+    c_mj = model.meta.photometry.conversion_megajanskys
     for data in DATA:
         model[data] = model[data] * c_mj
     for variance in VARIANCES:
diff --git a/romancal/outlier_detection/outlier_detection.py b/romancal/outlier_detection/outlier_detection.py
index 47daeddd5..3a1b9d175 100644
--- a/romancal/outlier_detection/outlier_detection.py
+++ b/romancal/outlier_detection/outlier_detection.py
@@ -107,7 +107,7 @@ def do_detection(self):
             median_wcs = copy.deepcopy(example_model.meta.wcs)
             if pars["save_intermediate_results"]:
                 median_model = example_model.copy()
-                median_model.data = Quantity(median_data, unit=median_model.data.unit)
+                median_model.data = median_data
                 median_model.meta.filename = "drizzled_median.asdf"
                 median_model_output_path = self.make_output_path(
                     basepath=median_model.meta.filename,
@@ -206,15 +206,12 @@ def detect_outliers(self, median_data, median_wcs, resampled):
                 # make blot_data Quantity (same unit as image.data)
                 if resampled:
                     # blot back onto image
-                    blot_data = Quantity(
-                        gwcs_blot(
-                            median_data, median_wcs, image, interp=interp, sinscl=sinscl
-                        ),
-                        unit=image.data.unit,
+                    blot_data = gwcs_blot(
+                        median_data, median_wcs, image, interp=interp, sinscl=sinscl
                     )
                 else:
                     # use median
-                    blot_data = Quantity(median_data, unit=image.data.unit, copy=True)
+                    blot_data = median_data.copy()
                 flag_cr(image, blot_data, **self.outlierpars)
                 self.input_models.shelve(image, i)
 
@@ -272,7 +269,7 @@ def flag_cr(
         subtracted_background = backg
 
     sci_data = sci_image.data
-    blot_deriv = abs_deriv(blot_data.value)
+    blot_deriv = abs_deriv(blot_data)
     err_data = np.nan_to_num(sci_image.err)
 
     # create the outlier mask
@@ -283,8 +280,8 @@ def flag_cr(
         # Create a boolean mask based on a scaled version of
         # the derivative image (dealing with interpolating issues?)
         # and the standard n*sigma above the noise
-        threshold1 = scale1 * blot_deriv + snr1 * err_data.value
-        mask1 = np.greater(diff_noise.value, threshold1)
+        threshold1 = scale1 * blot_deriv + snr1 * err_data
+        mask1 = np.greater(diff_noise, threshold1)
 
         # Smooth the boolean mask with a 3x3 boxcar kernel
         kernel = np.ones((3, 3), dtype=int)
@@ -292,8 +289,8 @@ def flag_cr(
 
         # Create a 2nd boolean mask based on the 2nd set of
         # scale and threshold values
-        threshold2 = scale2 * blot_deriv + snr2 * err_data.value
-        mask2 = np.greater(diff_noise.value, threshold2)
+        threshold2 = scale2 * blot_deriv + snr2 * err_data
+        mask2 = np.greater(diff_noise, threshold2)
 
         # Final boolean mask
         cr_mask = mask1_smoothed & mask2
@@ -303,7 +300,7 @@ def flag_cr(
 
         # straightforward detection of outliers for non-dithered data since
         # err_data includes all noise sources (photon, read, and flat for baseline)
-        cr_mask = np.greater(diff_noise.value, snr1 * err_data.value)
+        cr_mask = np.greater(diff_noise, snr1 * err_data)
 
     # Count existing DO_NOT_USE pixels
     count_existing = np.count_nonzero(sci_image.dq & pixel.DO_NOT_USE)
diff --git a/romancal/regtest/test_mos_pipeline.py b/romancal/regtest/test_mos_pipeline.py
index 247a6256c..62e0149e2 100644
--- a/romancal/regtest/test_mos_pipeline.py
+++ b/romancal/regtest/test_mos_pipeline.py
@@ -5,9 +5,16 @@
 import pytest
 import roman_datamodels as rdm
 
+from romancal.associations.asn_from_list import asn_from_list
 from romancal.pipeline.mosaic_pipeline import MosaicPipeline
 
 from .regtestdata import compare_asdf
+from pathlib import Path
+from astropy.units import Quantity
+import asdf
+
+from ..associations.association_io import json as asn_json
+import json
 
 
 def passfail(bool_expr):
@@ -17,6 +24,86 @@ def passfail(bool_expr):
     return "Fail"
 
 
+class RegtestFileModifier:
+    # TODO: remove this entire class once the units
+    #  have been removed from the regtest files
+
+    def __init__(self, rtdata):
+        self.rtdata = rtdata
+        self.updated_asn_fname = None
+        self.truth_parent = Path(rtdata.truth).parent
+        self.input_parent = Path(rtdata.input).parent
+        self.truth_relative_path = Path(self.truth_parent).relative_to(
+            self.input_parent
+        )
+        self.truth_path = self.truth_relative_path / f"{Path(self.rtdata.truth).name}"
+
+    @staticmethod
+    def create_unitless_file(input_filename: str, output_filename: str) -> None:
+        with asdf.config_context() as cfg:
+            cfg.validate_on_read = False
+            cfg.validate_on_save = False
+            af = asdf.open(input_filename)
+
+            for attr in af.tree["roman"]:
+                item = getattr(af.tree["roman"], attr)
+                if isinstance(item, Quantity):
+                    setattr(af.tree["roman"], attr, item.value)
+
+            for attr in af.tree["roman"].meta.photometry:
+                item = getattr(af.tree["roman"].meta.photometry, attr)
+                if isinstance(item, Quantity):
+                    setattr(af.tree["roman"].meta.photometry, attr, item.value)
+
+            af.write_to(output_filename)
+
+    def create_new_asn_file(self, output_filename_list: list):
+        updated_asn = asn_from_list(
+            output_filename_list,
+            product_name=f"{self.rtdata.asn['products'][0]['name']}_no_units",
+        )
+        updated_asn["target"] = "none"
+
+        current_asn_fname = Path(self.rtdata.input)
+        self.updated_asn_fname = (
+            f"{current_asn_fname.stem}_no_units{current_asn_fname.suffix}"
+        )
+
+        _, serialized_updated_asn = asn_json.dump(updated_asn)
+        with open(self.updated_asn_fname, "w") as f:
+            json.dump(
+                json.loads(serialized_updated_asn), f, indent=4, separators=(",", ": ")
+            )
+
+    def update_rtdata(self):
+        rtdata_root_path = Path(self.rtdata.input).parent
+        self.rtdata.input = f"{rtdata_root_path}/{Path(self.updated_asn_fname)}"
+        # r0099101001001001001_F158_visit_no_units_i2d.asdf
+        self.rtdata.output = f"{rtdata_root_path}/{Path(self.rtdata.output.split('_i2d')[0]).stem}_no_units_i2d{Path(self.rtdata.output).suffix}"
+
+    def prepare_regtest_input_files(self):
+        input_filenames = [
+            x["expname"] for x in self.rtdata.asn["products"][0]["members"]
+        ]
+        input_filenames.append(str(self.truth_path))
+        output_filename_list = []
+        # include truth file
+        for input_filename in input_filenames:
+            fname = Path(input_filename)
+            if str(fname).startswith(str(self.truth_relative_path)):
+                output_filename = Path(
+                    f"{str(fname).split('_i2d.asdf')[0]}_no_units_i2d{fname.suffix}"
+                )
+                self.rtdata.truth = str(self.truth_parent / output_filename.name)
+            else:
+                output_filename = f"{fname.stem}_no_units{fname.suffix}"
+                output_filename_list.append(output_filename)
+            self.create_unitless_file(input_filename, output_filename)
+
+        self.create_new_asn_file(output_filename_list)
+        self.update_rtdata()
+
+
 @pytest.mark.bigdata
 @pytest.mark.soctests
 def test_level3_mos_pipeline(rtdata, ignore_asdf_paths):
@@ -26,12 +113,18 @@ def test_level3_mos_pipeline(rtdata, ignore_asdf_paths):
     # Test Pipeline
     output = "r0099101001001001001_F158_visit_i2d.asdf"
     rtdata.output = output
+
+    rtdata.get_truth(f"truth/WFI/image/{output}")
+
+    fixer = RegtestFileModifier(rtdata)
+    fixer.prepare_regtest_input_files()
+
     args = [
         "roman_mos",
         rtdata.input,
     ]
     MosaicPipeline.from_cmdline(args)
-    rtdata.get_truth(f"truth/WFI/image/{output}")
+
     diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths)
     assert diff.identical, diff.report()
 
diff --git a/romancal/resample/gwcs_drizzle.py b/romancal/resample/gwcs_drizzle.py
index 12cf8b75b..96e1018f8 100644
--- a/romancal/resample/gwcs_drizzle.py
+++ b/romancal/resample/gwcs_drizzle.py
@@ -431,7 +431,7 @@ def dodrizzle(
     log.info(f"Drizzling {insci.shape} --> {outsci.shape}")
 
     _vers, nmiss, nskip = cdrizzle.tdriz(
-        insci.astype(np.float32).value,
+        insci.astype(np.float32),
         inwht.astype(np.float32),
         pixmap,
         outsci,
diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py
index ca6363f1d..6c0da2173 100644
--- a/romancal/resample/resample.py
+++ b/romancal/resample/resample.py
@@ -365,18 +365,15 @@ def resample_many_to_one(self):
         exptime_tot = self.resample_exposure_time(output_model)
 
         # TODO: fix unit here
-        output_model.err = u.Quantity(
-            np.sqrt(
-                np.nansum(
-                    [
-                        output_model.var_rnoise,
-                        output_model.var_poisson,
-                        output_model.var_flat,
-                    ],
-                    axis=0,
-                )
-            ),
-            unit=output_model.err.unit,
+        output_model.err = np.sqrt(
+            np.nansum(
+                [
+                    output_model.var_rnoise,
+                    output_model.var_poisson,
+                    output_model.var_flat,
+                ],
+                axis=0,
+            )
         )
 
         self.update_exposure_times(output_model, exptime_tot)
@@ -395,7 +392,7 @@ def resample_variance_array(self, name, output_model):
         This modifies ``output_model`` in-place.
         """
         output_wcs = self.output_wcs
-        inverse_variance_sum = np.full_like(output_model.data.value, np.nan)
+        inverse_variance_sum = np.full_like(output_model.data, np.nan)
 
         log.info(f"Resampling {name}")
         with self.input_models:
@@ -461,9 +458,7 @@ def resample_variance_array(self, name, output_model):
         # We now have a sum of the inverse resampled variances.  We need the
         # inverse of that to get back to units of variance.
         # TODO: fix unit here
-        output_variance = u.Quantity(
-            np.reciprocal(inverse_variance_sum), unit=u.MJy**2 / u.sr**2
-        )
+        output_variance = np.reciprocal(inverse_variance_sum)
 
         setattr(output_model, name, output_variance)
 
@@ -516,7 +511,7 @@ def resample_exposure_time(self, output_model):
                     ymax=ymax,
                 )
 
-                exptime_tot += resampled_exptime.value
+                exptime_tot += resampled_exptime
                 self.input_models.shelve(model, i, modify=False)
 
         return exptime_tot
@@ -702,11 +697,11 @@ def drizzle_arrays(
         log.info(f"Drizzling {insci.shape} --> {outsci.shape}")
 
         _vers, _nmiss, _nskip = cdrizzle.tdriz(
-            insci.astype(np.float32).value,
+            insci.astype(np.float32),
             inwht,
             pixmap,
-            outsci.value,
-            outwht.value,
+            outsci,
+            outwht,
             outcon,
             uniqid=uniqid,
             xmin=xmin,
diff --git a/romancal/resample/resample_utils.py b/romancal/resample/resample_utils.py
index 0a08460cf..0b399e68b 100644
--- a/romancal/resample/resample_utils.py
+++ b/romancal/resample/resample_utils.py
@@ -151,7 +151,7 @@ def build_driz_weight(
             and model.var_rnoise.shape == model.data.shape
         ):
             with np.errstate(divide="ignore", invalid="ignore"):
-                inv_variance = model.var_rnoise.value**-1
+                inv_variance = model.var_rnoise**-1
             inv_variance[~np.isfinite(inv_variance)] = 1
         else:
             warnings.warn(
diff --git a/romancal/skymatch/skymatch.py b/romancal/skymatch/skymatch.py
index 80a41a3ff..38390b05f 100644
--- a/romancal/skymatch/skymatch.py
+++ b/romancal/skymatch/skymatch.py
@@ -460,8 +460,7 @@ def _overlap_matrix(images, apply_sky=True):
     # TODO: to improve performance, the nested loops could be parallelized
     # since _calc_sky() here can be called independently from previous steps.
     ns = len(images)
-    data_unit = images[0].image.unit
-    A = np.zeros((ns, ns), dtype=float) * data_unit
+    A = np.zeros((ns, ns), dtype=float)
     W = np.zeros((ns, ns), dtype=float)
     for i in range(ns):
         for j in range(i + 1, ns):
@@ -482,7 +481,6 @@ def _overlap_matrix(images, apply_sky=True):
 def _find_optimum_sky_deltas(images, apply_sky=True):
     ns = len(images)
     A, W = _overlap_matrix(images, apply_sky=apply_sky)
-    data_unit = images[0].image.unit
 
     def is_valid(i, j):
         return W[i, j] > 0 and W[j, i] > 0
@@ -514,7 +512,7 @@ def is_valid(i, j):
             if is_valid(i, j):
                 K[ieq, i] = Wm[i, j]
                 K[ieq, j] = -Wm[i, j]
-                F[ieq] = Wm[i, j] * (A[j, i] - A[i, j]).value
+                F[ieq] = Wm[i, j] * (A[j, i] - A[i, j])
                 invalid[i] = False
                 invalid[j] = False
                 ieq += 1
@@ -538,4 +536,4 @@ def is_valid(i, j):
 
     deltas = np.dot(invK, F)
     deltas[np.asarray(invalid, dtype=bool)] = np.nan
-    return deltas * data_unit
+    return deltas
diff --git a/romancal/skymatch/skymatch_step.py b/romancal/skymatch/skymatch_step.py
index 221ac13e7..4087db03c 100644
--- a/romancal/skymatch/skymatch_step.py
+++ b/romancal/skymatch/skymatch_step.py
@@ -173,9 +173,7 @@ def _imodel2skyim(self, image_model):
 
     def _set_sky_background(self, sky_image, step_status):
         image = sky_image.meta["image_model"]
-        sky = sky_image.sky
-        if sky == 0 or sky is None:
-            sky = 0 * image.data.unit
+        sky = sky_image.sky if sky_image.sky is not None else 0
 
         image.meta.background.method = str(self.skymethod)
         image.meta.background.subtracted = self.subtract
diff --git a/romancal/skymatch/skystatistics.py b/romancal/skymatch/skystatistics.py
index a61aca7f3..5032d3291 100644
--- a/romancal/skymatch/skystatistics.py
+++ b/romancal/skymatch/skystatistics.py
@@ -126,14 +126,14 @@ def calc_sky(self, data):
             in `skyvalue`.
 
         """
-        imstat = ImageStats(image=data.value, fields=self._fields, **(self._kwargs))
+        imstat = ImageStats(image=data, fields=self._fields, **(self._kwargs))
         stat = self._skystat(imstat)  # dict or scalar
 
         # re-attach units:
         if hasattr(stat, "__len__"):
-            self.skyval = {k: value * data.unit for k, value in stat.items()}
+            self.skyval = {k: value for k, value in stat.items()}
         else:
-            self.skyval = stat * data.unit
+            self.skyval = stat
 
         self.npix = imstat.npix
         return self.skyval, self.npix
diff --git a/romancal/source_catalog/detection.py b/romancal/source_catalog/detection.py
index 384919511..febc984cc 100644
--- a/romancal/source_catalog/detection.py
+++ b/romancal/source_catalog/detection.py
@@ -40,9 +40,6 @@ def convolve_data(data, kernel_fwhm, size=None, mask=None):
     convolved_data : `numpy.ndarray`
         The convolved data array.
     """
-    if not isinstance(data, Quantity):
-        raise ValueError("Input model must be a Quantity array.")
-
     size = math.ceil(kernel_fwhm * 3)
     size = size + 1 if size % 2 == 0 else size  # make size be odd
     kernel = make_2dgaussian_kernel(kernel_fwhm, size=size)  # normalized to 1
diff --git a/romancal/source_catalog/source_catalog.py b/romancal/source_catalog/source_catalog.py
index e80bc3d07..b0112349f 100644
--- a/romancal/source_catalog/source_catalog.py
+++ b/romancal/source_catalog/source_catalog.py
@@ -103,9 +103,7 @@ def __init__(
 
         self.sb_unit = "MJy/sr"
         self.l2_unit = "DN/s"
-        self.l2_conv_factor = (
-            self.model.meta.photometry.conversion_megajanskys / self.l2_unit
-        )
+        self.l2_conv_factor = self.model.meta.photometry.conversion_megajanskys
 
         if len(ci_star_thresholds) != 2:
             raise ValueError("ci_star_thresholds must contain only 2 items")
@@ -162,7 +160,7 @@ def pixel_area(self):
         pixel_area = self.model.meta.photometry.pixelarea_steradians
         if pixel_area < 0:
             pixel_area = (self._pixel_scale**2).to(u.sr)
-        return pixel_area
+        return pixel_area.value
 
     def convert_l2_to_sb(self):
         """
@@ -205,19 +203,14 @@ def convert_sb_to_flux_density(self):
 
         The flux density unit is defined by self.flux_unit.
         """
-        if self.model.data.unit != self.sb_unit or self.model.err.unit != self.sb_unit:
-            raise ValueError(
-                f"data and err are expected to be in units of {self.sb_unit}"
-            )
 
         # the conversion in done in-place to avoid making copies of the data;
         # use a dictionary to set the value to avoid on-the-fly validation
         self.model["data"] *= self.pixel_area
-        self.model["data"] <<= self.flux_unit
+
         self.model["err"] *= self.pixel_area
-        self.model["err"] <<= self.flux_unit
+
         self.convolved_data *= self.pixel_area
-        self.convolved_data <<= self.flux_unit
 
     def convert_flux_density_to_sb(self):
         """
@@ -226,20 +219,10 @@ def convert_flux_density_to_sb(self):
 
         This is the inverse operation of `convert_sb_to_flux_density`.
         """
-        if (
-            self.model.data.unit != self.flux_unit
-            or self.model.err.unit != self.flux_unit
-        ):
-            raise ValueError(
-                f"data and err are expected to be in units of {self.flux_unit}"
-            )
 
         self.model["data"] /= self.pixel_area
-        self.model["data"] <<= self.sb_unit
         self.model["err"] /= self.pixel_area
-        self.model["err"] <<= self.sb_unit
         self.convolved_data /= self.pixel_area
-        self.convolved_data <<= self.sb_unit
 
     def convert_flux_to_abmag(self, flux, flux_err):
         """
@@ -260,15 +243,14 @@ def convert_flux_to_abmag(self, flux, flux_err):
             warnings.simplefilter("ignore", category=RuntimeWarning)
 
             # exact AB mag zero point
-            flux_zpt = 10 ** (-0.4 * 48.60) * u.erg / u.s / u.cm**2 / u.Hz
-            flux_zpt <<= self.flux_unit
+            flux_zpt = 10 ** (-0.4 * 48.60)
 
-            abmag_zpt = 2.5 * np.log10(flux_zpt.value)
-            abmag = -2.5 * np.log10(flux.value) + abmag_zpt
-            abmag_err = 2.5 * np.log10(1.0 + (flux_err.value / flux.value))
+            abmag_zpt = 2.5 * np.log10(flux_zpt)
+            abmag = -2.5 * np.log10(flux) + abmag_zpt
+            abmag_err = 2.5 * np.log10(1.0 + (flux_err / flux))
 
             # handle negative fluxes
-            idx = flux.value < 0
+            idx = flux < 0
             abmag[idx] = np.nan
             abmag_err[idx] = np.nan
 
@@ -560,7 +542,7 @@ def _aper_local_background(self):
             bkg_median = []
             bkg_std = []
             for mask in bkg_aper_masks:
-                bkg_data = mask.get_values(self.model.data.value)
+                bkg_data = mask.get_values(self.model.data)
                 values = sigclip(bkg_data, masked=False)
                 nvalues.append(values.size)
                 bkg_median.append(np.median(values))
@@ -571,9 +553,6 @@ def _aper_local_background(self):
             # standard error of the median
             bkg_median_err = np.sqrt(np.pi / (2.0 * nvalues)) * np.array(bkg_std)
 
-        bkg_median <<= self.model.data.unit
-        bkg_median_err <<= self.model.data.unit
-
         return bkg_median, bkg_median_err
 
     @lazyproperty
@@ -783,7 +762,7 @@ def _daofind_convolved_data(self):
         The DAOFind convolved data.
         """
         return ndimage.convolve(
-            self.model.data.value, self._daofind_kernel, mode="constant", cval=0.0
+            self.model.data, self._daofind_kernel, mode="constant", cval=0.0
         )
 
     @lazyproperty

From 75cb1550621e37ef14569642def807e3ba4eb60f Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Mon, 7 Oct 2024 12:01:20 -0400
Subject: [PATCH 02/74] Update to point to temp RAD and datamodels
 installation.

---
 pyproject.toml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index bca3c5fd4..1ff281a8c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -23,10 +23,10 @@ dependencies = [
     "photutils >=1.13.0",
     "pyparsing >=2.4.7",
     "requests >=2.26",
-    "rad>=0.21.0,<0.22.0",
-    # "rad @ git+https://github.com/spacetelescope/rad.git",
-    "roman_datamodels>=0.21.0,<0.22.0",
-    # "roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git",
+    # "rad>=0.21.0,<0.22.0",
+    "rad @ git+https://github.com/mairanteodoro/rad.git@RCAL-911-remove-units-from-mosaic-level-pipeline",
+    # "roman_datamodels>=0.21.0,<0.22.0",
+    "roman_datamodels @ git+https://github.com/mairanteodoro/roman_datamodels.git@RCAL-911-remove-units-from-mosaic-level-pipeline",
     "scipy >=1.11",
     "stcal>=1.8.0,<1.9.0",
     # "stcal @ git+https://github.com/spacetelescope/stcal.git@main",

From 953961a8e5cea1400f8fa47fdf7ddf74956217b0 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Thu, 29 Aug 2024 15:02:13 -0400
Subject: [PATCH 03/74] First refactoring of TweakRegStep to use stcal.

---
 romancal/tweakreg/tests/test_tweakreg.py | 214 +--------
 romancal/tweakreg/tweakreg_step.py       | 582 ++++++++---------------
 2 files changed, 197 insertions(+), 599 deletions(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index 9413b6ad1..636da6dfc 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -627,96 +627,6 @@ def test_tweakreg_updates_group_id(tmp_path, base_image):
         res.shelve(model, 0, modify=False)
 
 
-@pytest.mark.parametrize(
-    "shift_1, shift_2, tolerance, is_small_correction",
-    [
-        (0, 0, 5, True),
-        (0, 3, 5, True),
-        (1, 1, 5, True),
-        (5, 5, (5**2 + 5**2) ** 0.5, True),
-        (5, 5, 5, False),
-        (5, 5, 1, False),
-        (5, 5, 3, False),
-        (5, 10, 5, False),
-    ],
-)
-def test_tweakreg_correction_magnitude(
-    shift_1, shift_2, tolerance, is_small_correction, request
-):
-    """
-    Test that TweakReg corrections are within tolerance.
-    All the parametrized values are in arcsec.
-    """
-    img1 = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000)
-    img2 = request.getfixturevalue("base_image")(
-        shift_1=1000 + shift_1 / 0.1, shift_2=1000 + shift_2 / 0.1
-    )
-    img1_wcs = copy.deepcopy(img1.meta.wcs)
-    img2_wcs = copy.deepcopy(img2.meta.wcs)
-
-    step = trs.TweakRegStep()
-    step.tolerance = tolerance / 10.0
-
-    assert step._is_wcs_correction_small(img1_wcs, img2_wcs) == is_small_correction
-
-
-@pytest.mark.parametrize(
-    "filename_list, expected_common_name",
-    (
-        (
-            [
-                "l1-sca1_cal.asdf",
-                "l1-sca2_cal.asdf",
-                "l1-sca3_cal.asdf",
-            ],
-            "l1-sca",
-        ),
-        (
-            [
-                "l1-270-66-gaia-2016-sca1_cal.asdf",
-                "l1-270-66-gaia-2016-sca2_cal.asdf",
-                "l1-270-66-gaia-2016-sca3_cal.asdf",
-            ],
-            "l1-270-66-gaia-2016-sca",
-        ),
-    ),
-)
-def test_tweakreg_common_name(filename_list, expected_common_name, request):
-    """Test that TweakReg raises an error when an invalid input is provided."""
-    img_list = []
-    for filename in filename_list:
-        img = request.getfixturevalue("base_image")()
-        img.meta["filename"] = filename
-        img_list.append(img)
-
-    res = trs._common_name(img_list)
-
-    assert res == expected_common_name
-
-
-@pytest.mark.parametrize(
-    "filename_list",
-    (
-        [
-            "l1-sca1_cal.asdf",
-            "l1-sca2_cal.asdf",
-            "l1-sca3_cal.asdf",
-        ],
-        [
-            "l1-270-66-gaia-2016-sca1_cal.asdf",
-            "l1-270-66-gaia-2016-sca2_cal.asdf",
-            "l1-270-66-gaia-2016-sca3_cal.asdf",
-        ],
-    ),
-)
-def test_tweakreg_common_name_raises_error_on_invalid_input(filename_list):
-    """Test that TweakReg raises an error when an invalid input is provided."""
-    with pytest.raises(Exception) as exec_info:
-        trs._common_name(filename_list)
-
-    assert type(exec_info.value) == TypeError
-
-
 @pytest.mark.parametrize(
     "abs_refcat",
     (
@@ -727,6 +637,8 @@ def test_tweakreg_common_name_raises_error_on_invalid_input(filename_list):
 )
 def test_tweakreg_save_valid_abs_refcat(tmp_path, abs_refcat, request):
     """Test that TweakReg saves the catalog used for absolute astrometry."""
+    os.chdir(tmp_path)
+
     img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000)
     catalog_filename = "ref_catalog.ecsv"
     abs_refcat_filename = f"fit_{abs_refcat.lower()}_ref.ecsv"
@@ -737,9 +649,6 @@ def test_tweakreg_save_valid_abs_refcat(tmp_path, abs_refcat, request):
     )
 
     assert os.path.exists(tmp_path / abs_refcat_filename)
-    # clean up
-    os.remove(tmp_path / abs_refcat_filename)
-    os.remove(tmp_path / catalog_filename)
 
 
 @pytest.mark.parametrize(
@@ -748,6 +657,8 @@ def test_tweakreg_save_valid_abs_refcat(tmp_path, abs_refcat, request):
 )
 def test_tweakreg_defaults_to_valid_abs_refcat(tmp_path, abs_refcat, request):
     """Test that TweakReg defaults to DEFAULT_ABS_REFCAT on invalid values."""
+    os.chdir(tmp_path)
+
     img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000)
     catalog_filename = "ref_catalog.ecsv"
     abs_refcat_filename = f"fit_{trs.DEFAULT_ABS_REFCAT.lower()}_ref.ecsv"
@@ -758,9 +669,6 @@ def test_tweakreg_defaults_to_valid_abs_refcat(tmp_path, abs_refcat, request):
     )
 
     assert os.path.exists(tmp_path / abs_refcat_filename)
-    # clean up
-    os.remove(tmp_path / abs_refcat_filename)
-    os.remove(tmp_path / catalog_filename)
 
 
 def test_tweakreg_raises_error_on_invalid_abs_refcat(tmp_path, base_image):
@@ -771,7 +679,7 @@ def test_tweakreg_raises_error_on_invalid_abs_refcat(tmp_path, base_image):
     with pytest.raises(Exception) as exec_info:
         trs.TweakRegStep.call([img], save_abs_catalog=True, abs_refcat="my_ref_cat")
 
-    assert type(exec_info.value) == ValueError
+    assert type(exec_info.value) == TypeError
 
 
 def test_tweakreg_combine_custom_catalogs_and_asn_file(tmp_path, base_image):
@@ -955,28 +863,6 @@ def test_tweakreg_parses_asn_correctly(tmp_path, base_image):
         [res.shelve(m, i, modify=False) for i, m in enumerate(models)]
 
 
-def test_tweakreg_raises_error_on_connection_error_to_the_vo_service(
-    tmp_path, base_image, monkeypatch
-):
-    """
-    Test that TweakReg raises an error when there is a connection error with
-    the VO API server, which means that an absolute reference catalog cannot be created.
-    """
-
-    img = base_image(shift_1=1000, shift_2=1000)
-    add_tweakreg_catalog_attribute(tmp_path, img)
-
-    monkeypatch.setattr("requests.get", MockConnectionError)
-    res = trs.TweakRegStep.call([img])
-
-    assert type(res) == ModelLibrary
-    assert len(res) == 1
-    with res:
-        model = res.borrow(0)
-        assert model.meta.cal_step.tweakreg.lower() == "skipped"
-        res.shelve(model, 0, modify=False)
-
-
 def test_fit_results_in_meta(tmp_path, base_image):
     """
     Test that the WCS fit results from tweakwcs are available in the meta tree.
@@ -1023,96 +909,6 @@ def test_tweakreg_handles_multiple_groups(tmp_path, base_image):
             res.shelve(r, modify=False)
 
 
-@pytest.mark.parametrize(
-    "column_names",
-    [("x", "y"), ("xcentroid", "ycentroid")],
-)
-def test_imodel2wcsim_valid_column_names(tmp_path, base_image, column_names):
-    """
-    Test that _imodel2wcsim handles different catalog column names.
-    """
-    img_1 = base_image(shift_1=1000, shift_2=1000)
-    img_2 = base_image(shift_1=1030, shift_2=1030)
-    add_tweakreg_catalog_attribute(tmp_path, img_1, catalog_filename="img_1")
-    add_tweakreg_catalog_attribute(tmp_path, img_2, catalog_filename="img_2")
-    # set meta.tweakreg_catalog (this is automatically added by TweakRegStep)
-    catalog_format = "ascii.ecsv"
-    for x in [img_1, img_2]:
-        x.meta["tweakreg_catalog"] = Table.read(
-            x.meta.source_detection.tweakreg_catalog_name,
-            format=catalog_format,
-        )
-        x.meta.tweakreg_catalog.rename_columns(("x", "y"), column_names)
-    xname, yname = column_names
-
-    images = ModelLibrary([img_1, img_2])
-
-    step = trs.TweakRegStep()
-    with images:
-        for i, (m, target) in enumerate(zip(images, [img_1, img_2])):
-            imcat = step._imodel2wcsim(m)
-            assert (
-                imcat.meta["catalog"]["x"] == target.meta.tweakreg_catalog[xname]
-            ).all()
-            assert (
-                imcat.meta["catalog"]["y"] == target.meta.tweakreg_catalog[yname]
-            ).all()
-            images.shelve(m, i, modify=False)
-
-
-@pytest.mark.parametrize(
-    "column_names",
-    [
-        ("x_centroid", "y_centroid"),
-        ("x_cen", "y_cen"),
-    ],
-)
-def test_imodel2wcsim_error_invalid_column_names(tmp_path, base_image, column_names):
-    """
-    Test that _imodel2wcsim raises a ValueError on invalid catalog column names.
-    """
-    img_1 = base_image(shift_1=1000, shift_2=1000)
-    img_2 = base_image(shift_1=1030, shift_2=1030)
-    add_tweakreg_catalog_attribute(tmp_path, img_1, catalog_filename="img_1")
-    add_tweakreg_catalog_attribute(tmp_path, img_2, catalog_filename="img_2")
-    # set meta.tweakreg_catalog (this is automatically added by TweakRegStep)
-    catalog_format = "ascii.ecsv"
-    for x in [img_1, img_2]:
-        x.meta["tweakreg_catalog"] = Table.read(
-            x.meta.source_detection.tweakreg_catalog_name,
-            format=catalog_format,
-        )
-        x.meta.tweakreg_catalog.rename_columns(("x", "y"), column_names)
-
-    images = ModelLibrary([img_1, img_2])
-
-    step = trs.TweakRegStep()
-    with pytest.raises(ValueError):
-        with images:
-            for i, model in enumerate(images):
-                images.shelve(model, i, modify=False)
-                step._imodel2wcsim(model)
-
-
-def test_imodel2wcsim_error_invalid_catalog(tmp_path, base_image):
-    """
-    Test that _imodel2wcsim raises an error on invalid catalog format.
-    """
-    img_1 = base_image(shift_1=1000, shift_2=1000)
-    add_tweakreg_catalog_attribute(tmp_path, img_1, catalog_filename="img_1")
-    # set meta.tweakreg_catalog (this is automatically added by TweakRegStep)
-    img_1.meta["tweakreg_catalog"] = "nonsense"
-
-    images = ModelLibrary([img_1])
-
-    step = trs.TweakRegStep()
-    with pytest.raises(AttributeError):
-        with images:
-            for i, model in enumerate(images):
-                images.shelve(model, i, modify=False)
-                step._imodel2wcsim(model)
-
-
 def test_parse_catfile_valid_catalog(tmp_path, base_image):
     """
     Test that _parse_catfile can parse a custom catalog with valid format.
diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index 080486d99..a370539d7 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -6,18 +6,13 @@
 from pathlib import Path
 
 import numpy as np
-from astropy import units as u
-from astropy.coordinates import SkyCoord
 from astropy.table import Table
 from roman_datamodels import datamodels as rdm
-from tweakwcs.correctors import JWSTWCSCorrector
-from tweakwcs.imalign import align_wcs
-from tweakwcs.matchutils import XYXYMatch
+from stcal.tweakreg import tweakreg
 
 # LOCAL
 from ..datamodels import ModelLibrary
 from ..stpipe import RomanStep
-from . import astrometric_utils as amutils
 
 
 def _oxford_or_str_join(str_list):
@@ -84,55 +79,13 @@ class TweakRegStep(RomanStep):
 
     def process(self, input):
 
-        use_custom_catalogs = self.use_custom_catalogs
+        # properly handle input
+        images = self.handle_input(step_input=input)
 
-        if use_custom_catalogs:
-            catdict = _parse_catfile(self.catfile)
-            # if user requested the use of custom catalogs and provided a
-            # valid 'catfile' file name that has no custom catalogs,
-            # turn off the use of custom catalogs:
-            if catdict is not None and not catdict:
-                self.log.warning(
-                    "'use_custom_catalogs' is set to True but 'catfile' "
-                    "contains no user catalogs."
-                )
-                use_custom_catalogs = False
+        catdict = _parse_catfile(self.catfile)
 
-        try:
-            if isinstance(input, rdm.DataModel):
-                images = ModelLibrary([input])
-            elif str(input).endswith(".asdf"):
-                images = ModelLibrary([rdm.open(input)])
-            elif isinstance(input, ModelLibrary):
-                images = input
-            else:
-                images = ModelLibrary(input)
-        except TypeError as e:
-            e.args = (
-                "Input to tweakreg must be a list of DataModels, an "
-                "association, or an already open ModelLibrary "
-                "containing one or more DataModels.",
-            ) + e.args[1:]
-            raise e
-
-        if use_custom_catalogs and catdict:
-            with images:
-                for i, member in enumerate(images.asn["products"][0]["members"]):
-                    filename = member["expname"]
-                    if filename in catdict:
-                        # FIXME: I'm not sure if this captures all the possible combinations
-                        # for example, meta.tweakreg_catalog is set by the container (when
-                        # it's present in the association). However the code in this step
-                        # checks meta.source_catalog.tweakreg_catalog. I think this means
-                        # that setting a catalog via an association does not work. Is this
-                        # intended? If so, the container can be updated to not support that.
-                        model = images.borrow(i)
-                        model.meta["source_detection"] = {
-                            "tweakreg_catalog_name": catdict[filename],
-                        }
-                        images.shelve(model, i)
-                    else:
-                        images.shelve(model, i, modify=False)
+        if self.use_custom_catalogs:
+            self.validate_custom_catalogs(catdict, images)
 
         if len(self.catalog_path) == 0:
             self.catalog_path = os.getcwd()
@@ -152,107 +105,7 @@ def process(self, input):
             raise ValueError("Input must contain at least one image model.")
 
         # Build the catalogs for input images
-        with images:
-            for i, image_model in enumerate(images):
-                if image_model.meta.exposure.type != "WFI_IMAGE":
-                    # Check to see if attempt to run tweakreg on non-Image data
-                    self.log.info("Skipping TweakReg for spectral exposure.")
-                    # Uncomment below once rad & input data have the cal_step tweakreg
-                    image_model.meta.cal_step.tweakreg = "SKIPPED"
-                    images.shelve(image_model)
-                    return image_model
-
-                if hasattr(image_model.meta, "source_detection"):
-                    is_tweakreg_catalog_present = hasattr(
-                        image_model.meta.source_detection, "tweakreg_catalog"
-                    )
-                    is_tweakreg_catalog_name_present = hasattr(
-                        image_model.meta.source_detection, "tweakreg_catalog_name"
-                    )
-                    if is_tweakreg_catalog_present:
-                        # read catalog from structured array
-                        catalog = Table(
-                            np.asarray(
-                                image_model.meta.source_detection.tweakreg_catalog
-                            )
-                        )
-                    elif is_tweakreg_catalog_name_present:
-                        catalog = self.read_catalog(
-                            image_model.meta.source_detection.tweakreg_catalog_name
-                        )
-                    else:
-                        images.shelve(image_model, i, modify=False)
-                        raise AttributeError(
-                            "Attribute 'meta.source_detection.tweakreg_catalog' is missing."
-                            "Please either run SourceDetectionStep or provide a"
-                            "custom source catalog."
-                        )
-                    # remove 4D numpy array from meta.source_detection
-                    if is_tweakreg_catalog_present:
-                        del image_model.meta.source_detection["tweakreg_catalog"]
-                else:
-                    images.shelve(image_model, i, modify=False)
-                    raise AttributeError(
-                        "Attribute 'meta.source_detection' is missing."
-                        "Please either run SourceDetectionStep or provide a"
-                        "custom source catalog."
-                    )
-
-                for axis in ["x", "y"]:
-                    if axis not in catalog.colnames:
-                        long_axis = axis + "centroid"
-                        if long_axis in catalog.colnames:
-                            catalog.rename_column(long_axis, axis)
-                        else:
-                            images.shelve(image_model, i, modify=False)
-                            raise ValueError(
-                                "'tweakreg' source catalogs must contain a header with "
-                                "columns named either 'x' and 'y' or "
-                                "'xcentroid' and 'ycentroid'."
-                            )
-
-                filename = image_model.meta.filename
-
-                # filter out sources outside the WCS bounding box
-                bb = image_model.meta.wcs.bounding_box
-                x = catalog["x"]
-                y = catalog["y"]
-                if bb is None:
-                    r, d = image_model.meta.wcs(x, y)
-                    mask = np.isfinite(r) & np.isfinite(d)
-                    catalog = catalog[mask]
-
-                    n_removed_src = np.sum(np.logical_not(mask))
-                    if n_removed_src:
-                        self.log.info(
-                            f"Removed {n_removed_src} sources from {filename}'s "
-                            "catalog whose image coordinates could not be "
-                            "converted to world coordinates."
-                        )
-                else:
-                    # assume image coordinates of all sources within a bounding box
-                    # can be converted to world coordinates.
-                    ((xmin, xmax), (ymin, ymax)) = bb
-                    mask = (x > xmin) & (x < xmax) & (y > ymin) & (y < ymax)
-                    catalog = catalog[mask]
-
-                    n_removed_src = np.sum(np.logical_not(mask))
-                    if n_removed_src:
-                        self.log.info(
-                            f"Removed {n_removed_src} sources from {filename}'s "
-                            "catalog that were outside of the bounding box."
-                        )
-
-                # set meta.tweakreg_catalog
-                image_model.meta["tweakreg_catalog"] = catalog.as_array()
-
-                nsources = len(catalog)
-                if nsources == 0:
-                    self.log.warning(f"No sources found in {filename}.")
-                else:
-                    self.log.info(f"Detected {len(catalog)} sources in {filename}.")
-
-                images.shelve(image_model, i)
+        self.set_tweakreg_catalog_attribute(images)
 
         # group images by their "group id":
         group_indices = images.group_indices
@@ -261,197 +114,57 @@ def process(self, input):
         self.log.info(f"Number of image groups to be aligned: {len(group_indices):d}.")
         self.log.info("Image groups:")
 
-        imcats = []
-        with images:
-            for i, m in enumerate(images):
-                imcats.append(self._imodel2wcsim(m))
-                images.shelve(m, i, modify=False)
+        imcats = self.build_image_catalogs(images)
 
         if len(group_indices) > 1:
             # local align images:
-            xyxymatch = XYXYMatch(
+            tweakreg.relative_align(
+                imcats,
                 searchrad=self.searchrad,
                 separation=self.separation,
                 use2dhist=self.use2dhist,
                 tolerance=self.tolerance,
                 xoffset=0,
                 yoffset=0,
+                enforce_user_order=self.enforce_user_order,
+                expand_refcat=self.expand_refcat,
+                minobj=self.minobj,
+                fitgeometry=self.fitgeometry,
+                nclip=self.nclip,
+                sigma=self.sigma,
             )
 
-            try:
-                align_wcs(
-                    imcats,
-                    refcat=None or self.refcat,
-                    enforce_user_order=self.enforce_user_order,
-                    expand_refcat=self.expand_refcat,
-                    minobj=self.minobj,
-                    match=xyxymatch,
-                    fitgeom=self.fitgeometry,
-                    nclip=self.nclip,
-                    sigma=(self.sigma, "rmse"),
-                    clip_accum=True,
-                )
-
-            except ValueError as e:
-                msg = e.args[0]
-                if (
-                    msg == "Too few input images (or groups of images) with non-empty"
-                    " catalogs."
-                ):
-                    # we need at least two exposures to perform image alignment
-                    self.log.warning(msg)
-                    self.log.warning(
-                        "At least two exposures are required for relative image alignment."
-                    )
-                else:
-                    raise e
-
-            except RuntimeError as e:
-                msg = e.args[0]
-                if msg.startswith("Number of output coordinates exceeded allocation"):
-                    # we need at least two exposures to perform image alignment
-                    self.log.error(msg)
-                    self.log.error(
-                        "Multiple sources within specified tolerance "
-                        "matched to a single reference source. Try to "
-                        "adjust 'tolerance' and/or 'separation' parameters."
-                    )
-                    self.log.warning("Skipping 'TweakRegStep'...")
-                    self.skip = True
-                    with images:
-                        for i, model in enumerate(images):
-                            model.meta.cal_step.tweakreg = "SKIPPED"
-                            images.shelve(model, i)
-                    return images
-                else:
-                    raise e
-
-            with images:
-                for i, imcat in enumerate(imcats):
-                    model = images.borrow(i)
-                    if model.meta.cal_step.get("tweakreg") == "SKIPPED":
-                        continue
-                    wcs = model.meta.wcs
-                    twcs = imcat.wcs
-                    small_correction = self._is_wcs_correction_small(wcs, twcs)
-                    images.shelve(model, i, modify=False)
-                    if not small_correction:
-                        # Large corrections are typically a result of source
-                        # mis-matching or poorly-conditioned fit. Skip such models.
-                        self.log.warning(
-                            "WCS has been tweaked by more than"
-                            f" {10 * self.tolerance} arcsec"
-                        )
-
-                        self.log.warning("Skipping relative alignment (stage 1)...")
-
-        # Get catalog of GAIA sources for the field
-        #
-        # NOTE:  If desired, the pipeline can write out the reference
-        #        catalog as a separate product with a name based on
-        #        whatever convention is determined by the JWST Cal Working
-        #        Group.
-        if self.save_abs_catalog:
-            output_name = os.path.join(
-                self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
-            )
-        else:
-            output_name = None
-
-        # initial shift to be used with absolute astrometry
-        self.abs_xoffset = 0
-        self.abs_yoffset = 0
-
         self.abs_refcat = self.abs_refcat.strip()
         gaia_cat_name = self.abs_refcat.upper()
 
         if gaia_cat_name in SINGLE_GROUP_REFCAT:
             with images:
-                models = list(images)
-
-                try:
-                    # FIXME: astrometric_utils expects all models in memory
-                    ref_cat = amutils.create_astrometric_catalog(
-                        models,
-                        gaia_cat_name,
-                        output=output_name,
-                    )
-                except Exception as e:
-                    self.log.warning(
-                        "TweakRegStep cannot proceed because of an error that "
-                        "occurred while fetching data from the VO server. "
-                        f"Returned error message: '{e}'"
-                    )
-                    self.log.warning("Skipping 'TweakRegStep'...")
-                    self.skip = True
-                    for model in models:
-                        model.meta.cal_step["tweakreg"] = "SKIPPED"
-                    [images.shelve(m, i, modify=False) for i, m in enumerate(models)]
-                    return images
-                [images.shelve(m, i, modify=False) for i, m in enumerate(models)]
-
-        elif os.path.isfile(self.abs_refcat):
-            ref_cat = Table.read(self.abs_refcat)
+                ref_image = images.borrow(0)
+                images.shelve(ref_image, 0, modify=False)
 
-        else:
-            raise ValueError(
-                "'abs_refcat' must be a path to an "
-                "existing file name or one of the supported "
-                f"reference catalogs: {_SINGLE_GROUP_REFCAT_STR}."
+            tweakreg.absolute_align(
+                imcats,
+                self.abs_refcat,
+                ref_wcs=ref_image.meta.wcs,
+                ref_wcsinfo=ref_image.meta.wcsinfo,
+                epoch=ref_image.meta.exposure.mid_time.decimalyear,
+                abs_minobj=self.abs_minobj,
+                abs_fitgeometry=self.abs_fitgeometry,
+                abs_nclip=self.abs_nclip,
+                abs_sigma=self.abs_sigma,
+                abs_searchrad=self.abs_searchrad,
+                abs_use2dhist=self.abs_use2dhist,
+                abs_separation=self.abs_separation,
+                abs_tolerance=self.abs_tolerance,
+                save_abs_catalog=self.save_abs_catalog,
+                abs_catalog_output_dir=self.output_dir,
             )
 
-        # Check that there are enough GAIA sources for a reliable/valid fit
-        num_ref = len(ref_cat)
-        if num_ref < self.abs_minobj:
-            # Raise Exception here to avoid rest of code in this try block
-            self.log.warning(
-                f"Not enough sources ({num_ref}) in the reference catalog "
-                "for the single-group alignment step to perform a fit. "
-                f"Skipping alignment to the {self.abs_refcat} reference "
-                "catalog!"
-            )
-        else:
-            # align images:
-            # Update to separation needed to prevent confusion of sources
-            # from overlapping images where centering is not consistent or
-            # for the possibility that errors still exist in relative overlap.
-            xyxymatch_gaia = XYXYMatch(
-                searchrad=self.abs_searchrad,
-                separation=self.abs_separation,
-                use2dhist=self.abs_use2dhist,
-                tolerance=self.abs_tolerance,
-                xoffset=self.abs_xoffset,
-                yoffset=self.abs_yoffset,
-            )
+        self.finalize_step(images, imcats)
 
-            # Set group_id to same value so all get fit as one observation
-            # The assigned value, 987654, has been hard-coded to make it
-            # easy to recognize when alignment to GAIA was being performed
-            # as opposed to the group_id values used for relative alignment
-            # earlier in this step.
-            for imcat in imcats:
-                imcat.meta["group_id"] = 987654
-                if (
-                    "fit_info" in imcat.meta
-                    and "REFERENCE" in imcat.meta["fit_info"]["status"]
-                ):
-                    del imcat.meta["fit_info"]
-
-            # Perform fit
-            align_wcs(
-                imcats,
-                refcat=ref_cat,
-                enforce_user_order=True,
-                expand_refcat=False,
-                minobj=self.abs_minobj,
-                match=xyxymatch_gaia,
-                fitgeom=self.abs_fitgeometry,
-                nclip=self.abs_nclip,
-                sigma=(self.abs_sigma, "rmse"),
-                ref_tpwcs=imcats[0],
-                clip_accum=True,
-            )
+        return images
 
+    def finalize_step(self, images, imcats):
         with images:
             for i, imcat in enumerate(imcats):
                 image_model = images.borrow(i)
@@ -497,8 +210,48 @@ def process(self, input):
                     image_model.meta.wcs = imcat.wcs
                 images.shelve(image_model, i)
 
+    @staticmethod
+    def handle_input(step_input):
+        try:
+            if isinstance(step_input, rdm.DataModel):
+                images = ModelLibrary([step_input])
+            elif str(step_input).endswith(".asdf"):
+                images = ModelLibrary([rdm.open(step_input)])
+            elif isinstance(step_input, ModelLibrary):
+                images = step_input
+            else:
+                images = ModelLibrary(step_input)
+        except TypeError as e:
+            e.args = (
+                "Input to tweakreg must be a list of DataModels, an "
+                "association, or an already open ModelLibrary "
+                "containing one or more DataModels.",
+            ) + e.args[1:]
+            raise e
         return images
 
+    @staticmethod
+    def build_image_catalogs(images):
+        imcats = []
+        with images:
+            for i, m in enumerate(images):
+                # catalog name
+                catalog_name = os.path.splitext(m.meta.filename)[0].strip("_- ")
+                # catalog data
+                catalog_table = Table(m.meta.tweakreg_catalog)
+                catalog_table.meta["name"] = catalog_name
+
+                imcats.append(
+                    tweakreg.construct_wcs_corrector(
+                        wcs=m.meta.wcs,
+                        refang=m.meta.wcsinfo,
+                        catalog=catalog_table,
+                        group_id=m.meta.group_id,
+                    )
+                )
+                images.shelve(m, i, modify=False)
+        return imcats
+
     def read_catalog(self, catalog_name):
         if catalog_name.endswith("asdf"):
             with rdm.open(catalog_name) as source_catalog_model:
@@ -507,84 +260,133 @@ def read_catalog(self, catalog_name):
             catalog = Table.read(catalog_name, format=self.catalog_format)
         return catalog
 
-    def _is_wcs_correction_small(self, wcs, twcs):
-        """Check that the newly tweaked wcs hasn't gone off the rails"""
-        tolerance = 10.0 * self.tolerance * u.arcsec
-
-        ra, dec = wcs.footprint(axis_type="spatial").T
-        tra, tdec = twcs.footprint(axis_type="spatial").T
-        skycoord = SkyCoord(ra=ra, dec=dec, unit="deg")
-        tskycoord = SkyCoord(ra=tra, dec=tdec, unit="deg")
+    def save_abs_ref_catalog(self, catalog_table: Table):
+        output_name = os.path.join(
+            self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
+        )
+        catalog_table.write(output_name, format=self.catalog_format, overwrite=True)
 
-        separation = skycoord.separation(tskycoord)
+    def validate_custom_catalogs(self, catdict, images):
+        use_custom_catalogs = self.use_custom_catalogs
+        # if user requested the use of custom catalogs and provided a
+        # valid 'catfile' file name that has no custom catalogs,
+        # turn off the use of custom catalogs:
+        if catdict is not None and not catdict:
+            self.log.warning(
+                "'use_custom_catalogs' is set to True but 'catfile' "
+                "contains no user catalogs."
+            )
+            use_custom_catalogs = False
 
-        return (separation < tolerance).all()
+        if use_custom_catalogs and catdict:
+            with images:
+                for i, member in enumerate(images.asn["products"][0]["members"]):
+                    filename = member["expname"]
+                    if filename in catdict:
+                        # FIXME: I'm not sure if this captures all the possible combinations
+                        # for example, meta.tweakreg_catalog is set by the container (when
+                        # it's present in the association). However the code in this step
+                        # checks meta.source_catalog.tweakreg_catalog. I think this means
+                        # that setting a catalog via an association does not work. Is this
+                        # intended? If so, the container can be updated to not support that.
+                        model = images.borrow(i)
+                        model.meta["source_detection"] = {
+                            "tweakreg_catalog_name": catdict[filename],
+                        }
+                        images.shelve(model, i)
+                    else:
+                        images.shelve(model, i, modify=False)
 
-    def _imodel2wcsim(self, image_model):
-        catalog = image_model.meta.tweakreg_catalog
-        model_name = os.path.splitext(image_model.meta.filename)[0].strip("_- ")
+    def get_tweakreg_catalog(self, source_detection, image_model, index):
+        """Retrieve the tweakreg catalog from source detection."""
+        if hasattr(source_detection, "tweakreg_catalog"):
+            tweakreg_catalog = Table(np.asarray(source_detection.tweakreg_catalog))
+            del image_model.meta.source_detection["tweakreg_catalog"]
+            return tweakreg_catalog
+        elif hasattr(source_detection, "tweakreg_catalog_name"):
+            return self.read_catalog(source_detection.tweakreg_catalog_name)
+        else:
+            images.shelve(image_model, index, modify=False)
+            raise AttributeError(
+                "Attribute 'meta.source_detection.tweakreg_catalog' is missing. "
+                "Please either run SourceDetectionStep or provide a custom source catalog."
+            )
 
-        try:
-            if self.use_custom_catalogs:
-                catalog_format = self.catalog_format
+    @staticmethod
+    def validate_catalog_columns(catalog, axis, image_model, index):
+        """Validate the presence of required columns in the catalog."""
+        if axis not in catalog.colnames:
+            long_axis = f"{axis}centroid"
+            if long_axis in catalog.colnames:
+                catalog.rename_column(long_axis, axis)
             else:
-                catalog_format = "ascii.ecsv"
+                images.shelve(image_model, index, modify=False)
+                raise ValueError(
+                    "'tweakreg' source catalogs must contain a header with "
+                    "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
+                )
 
-            if isinstance(catalog, str):
-                # a string with the name of the catalog was provided
-                catalog = Table.read(catalog, format=catalog_format)
-            else:
-                # catalog is a structured array, convert to astropy table:
-                catalog = Table(catalog)
+    def filter_catalog_by_wcs(self, catalog, image_model, filename):
+        """Filter sources in the catalog based on WCS bounding box."""
+        bb = image_model.meta.wcs.bounding_box
+        x, y = catalog["x"], catalog["y"]
 
-            catalog.meta["name"] = (
-                str(catalog) if isinstance(catalog, str) else model_name
+        if bb is None:
+            r, d = image_model.meta.wcs(x, y)
+            mask = np.isfinite(r) & np.isfinite(d)
+        else:
+            ((xmin, xmax), (ymin, ymax)) = bb
+            mask = (x > xmin) & (x < xmax) & (y > ymin) & (y < ymax)
+
+        catalog = catalog[mask]
+        n_removed_src = np.sum(np.logical_not(mask))
+        if n_removed_src:
+            self.log.info(
+                f"Removed {n_removed_src} sources from {filename}'s "
+                "catalog that were outside of the bounding box."
+                if bb
+                else f"catalog whose image coordinates could not be converted to world coordinates."
             )
-        except OSError:
-            self.log.error(f"Cannot read catalog {catalog}")
-
-        # make sure catalog has 'x' and 'y' columns
-        for axis in ["x", "y"]:
-            if axis not in catalog.colnames:
-                long_axis = axis + "centroid"
-                if long_axis in catalog.colnames:
-                    catalog.rename_column(long_axis, axis)
-                else:
-                    raise ValueError(
-                        "'tweakreg' source catalogs must contain either columns 'x' and"
-                        " 'y' or 'xcentroid' and 'ycentroid'."
+
+        return catalog
+
+    def set_tweakreg_catalog_attribute(self, images):
+        with images:
+            for i, image_model in enumerate(images):
+                exposure_type = image_model.meta.exposure.type
+                if exposure_type != "WFI_IMAGE":
+                    self.log.info("Skipping TweakReg for spectral exposure.")
+                    image_model.meta.cal_step.tweakreg = "SKIPPED"
+                    images.shelve(image_model)
+                    return image_model
+
+                source_detection = getattr(image_model.meta, "source_detection", None)
+                if source_detection is None:
+                    images.shelve(image_model, i, modify=False)
+                    raise AttributeError(
+                        "Attribute 'meta.source_detection' is missing. "
+                        "Please either run SourceDetectionStep or provide a custom source catalog."
                     )
 
-        # create WCSImageCatalog object:
-        refang = image_model.meta.wcsinfo
-        # TODO: create RSTWCSCorrector in tweakwcs
-        im = JWSTWCSCorrector(
-            wcs=image_model.meta.wcs,
-            wcsinfo={
-                "roll_ref": refang["roll_ref"],
-                "v2_ref": refang["v2_ref"],
-                "v3_ref": refang["v3_ref"],
-            },
-            meta={
-                "catalog": catalog,
-                "group_id": image_model.meta.group_id,
-                "name": model_name,
-            },
-        )
+                catalog = self.get_tweakreg_catalog(source_detection, image_model, i)
 
-        return im
+                for axis in ["x", "y"]:
+                    self.validate_catalog_columns(catalog, axis, image_model, i)
 
+                filename = image_model.meta.filename
+                catalog = self.filter_catalog_by_wcs(catalog, image_model, filename)
 
-def _common_name(group):
-    file_names = []
-    for im in group:
-        if isinstance(im, rdm.DataModel):
-            file_names.append(os.path.splitext(im.meta.filename)[0].strip("_- "))
-        else:
-            raise TypeError("Input must be a list of datamodels list.")
+                if self.save_abs_catalog:
+                    self.save_abs_ref_catalog(catalog)
 
-    cn = os.path.commonprefix(file_names)
-    return cn
+                image_model.meta["tweakreg_catalog"] = catalog.as_array()
+                nsources = len(catalog)
+                self.log.info(
+                    f"Detected {nsources} sources in {filename}."
+                    if nsources
+                    else f"No sources found in {filename}."
+                )
+                images.shelve(image_model, i)
 
 
 def _parse_catfile(catfile):
@@ -596,7 +398,7 @@ def _parse_catfile(catfile):
     with open(catfile) as f:
         catfile_dir = os.path.dirname(catfile)
 
-        for line in f.readlines():
+        for line in f:
             sline = line.strip()
             if not sline or sline[0] == "#":
                 continue
@@ -605,7 +407,7 @@ def _parse_catfile(catfile):
             catalog = list(map(str.strip, catalog))
             if len(catalog) == 1:
                 catdict[data_model] = os.path.join(catfile_dir, catalog[0])
-            elif len(catalog) == 0:
+            elif not catalog:
                 catdict[data_model] = None
             else:
                 raise ValueError("'catfile' can contain at most two columns.")

From 89bd44e038d5db73fd9fb1d9e142eb7f1b48814d Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Fri, 30 Aug 2024 16:39:06 -0400
Subject: [PATCH 04/74] Code cleanup/refactoring.

---
 romancal/tweakreg/tweakreg_step.py | 564 +++++++++++++++++++++--------
 1 file changed, 409 insertions(+), 155 deletions(-)

diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index a370539d7..f7dc7c865 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -9,6 +9,7 @@
 from astropy.table import Table
 from roman_datamodels import datamodels as rdm
 from stcal.tweakreg import tweakreg
+from typing import List
 
 # LOCAL
 from ..datamodels import ModelLibrary
@@ -75,48 +76,218 @@ class TweakRegStep(RomanStep):
     """  # noqa: E501
 
     reference_file_types = []
-    refcat = None
 
     def process(self, input):
 
         # properly handle input
-        images = self.handle_input(step_input=input)
+        images = self.parse_input(step_input=input)
 
         catdict = _parse_catfile(self.catfile)
 
         if self.use_custom_catalogs:
             self.validate_custom_catalogs(catdict, images)
 
+        # set path where the source catalog will be saved to
+        self.set_catalog_path()
+
+        self.set_reference_catalog()
+
+        # Build the catalogs for input images
+        self.set_tweakreg_catalog_attribute(images)
+
+        imcats = _build_image_catalogs(images)
+
+        self.do_relative_alignment(images, imcats)
+
+        self.do_absolute_alignment(images, imcats)
+
+        self.finalize_step(images, imcats)
+
+        return images
+
+    def parse_input(self, step_input) -> ModelLibrary:
+        """
+        Parse the input for the class.
+
+        This method handles various types of input, including single DataModels,
+        file paths to ASDF files, and collections of DataModels. It ensures that
+        the input is converted into a ModelLibrary for further processing.
+
+        Parameters
+        ----------
+        step_input : Union[rdm.DataModel, str, ModelLibrary, list]
+            The input to be parsed, which can be a DataModel, a file path,
+            a ModelLibrary, or a list of DataModels.
+
+        Returns
+        -------
+        ModelLibrary
+            A ModelLibrary containing the parsed images.
+
+        Raises
+        ------
+        TypeError
+            If the input is not a valid type for processing.
+        """
+        try:
+            if isinstance(step_input, rdm.DataModel):
+                images = ModelLibrary([step_input])
+            elif str(step_input).endswith(".asdf"):
+                images = ModelLibrary([rdm.open(step_input)])
+            elif isinstance(step_input, ModelLibrary):
+                images = step_input
+            else:
+                images = ModelLibrary(step_input)
+        except TypeError as e:
+            e.args = (
+                "Input to tweakreg must be a list of DataModels, an "
+                "association, or an already open ModelLibrary "
+                "containing one or more DataModels.",
+            ) + e.args[1:]
+            raise e
+
+        if len(images) == 0:
+            raise ValueError("Input must contain at least one image model.")
+
+        self.log.info("")
+        self.log.info(
+            f"Number of image groups to be aligned: {len(images.group_indices):d}."
+        )
+        self.log.info("Image groups:")
+
+        return images
+
+    def validate_custom_catalogs(self, catdict, images):
+        """
+        Validate and apply custom catalogs for the tweak registration step.
+
+        This method checks if the user has requested the use of custom catalogs
+        and whether the provided catalog file contains valid entries. If valid
+        catalogs are found, it updates the image models with the corresponding
+        catalog names.
+
+        Parameters
+        ----------
+        catdict : dict
+            A dictionary mapping image filenames to custom catalog file paths.
+        images : ModelLibrary
+            A collection of image models to be updated with custom catalog information.
+
+        Returns
+        -------
+        None
+        """
+        use_custom_catalogs = self.use_custom_catalogs
+        # if user requested the use of custom catalogs and provided a
+        # valid 'catfile' file name that has no custom catalogs,
+        # turn off the use of custom catalogs:
+        if catdict is not None and not catdict:
+            self.log.warning(
+                "'use_custom_catalogs' is set to True but 'catfile' "
+                "contains no user catalogs."
+            )
+            use_custom_catalogs = False
+
+        if use_custom_catalogs and catdict:
+            with images:
+                for i, member in enumerate(images.asn["products"][0]["members"]):
+                    filename = member["expname"]
+                    if filename in catdict:
+                        # FIXME: I'm not sure if this captures all the possible combinations
+                        # for example, meta.tweakreg_catalog is set by the container (when
+                        # it's present in the association). However the code in this step
+                        # checks meta.source_catalog.tweakreg_catalog. I think this means
+                        # that setting a catalog via an association does not work. Is this
+                        # intended? If so, the container can be updated to not support that.
+                        model = images.borrow(i)
+                        model.meta["source_detection"] = {
+                            "tweakreg_catalog_name": catdict[filename],
+                        }
+                        images.shelve(model, i)
+                    else:
+                        images.shelve(model, i, modify=False)
+
+    def set_catalog_path(self):
         if len(self.catalog_path) == 0:
             self.catalog_path = os.getcwd()
-
         self.catalog_path = Path(self.catalog_path).as_posix()
         self.log.info(f"All source catalogs will be saved to: {self.catalog_path}")
 
-        if self.abs_refcat is None or len(self.abs_refcat.strip()) == 0:
-            self.abs_refcat = DEFAULT_ABS_REFCAT
-
+    def set_reference_catalog(self):
+        if self.abs_refcat is None or len(self.abs_refcat) == 0:
+            self.abs_refcat = DEFAULT_ABS_REFCAT.strip().upper()
         if self.abs_refcat != DEFAULT_ABS_REFCAT:
             # Set expand_refcat to True to eliminate possibility of duplicate
             # entries when aligning to absolute astrometric reference catalog
             self.expand_refcat = True
 
-        if len(images) == 0:
-            raise ValueError("Input must contain at least one image model.")
+    def set_tweakreg_catalog_attribute(self, images):
+        """
+        Set the tweakreg catalog attribute for each image model.
 
-        # Build the catalogs for input images
-        self.set_tweakreg_catalog_attribute(images)
+        This method iterates through the provided image models, checking the
+        exposure type and ensuring that the necessary source detection metadata
+        is present. It retrieves the tweak registration catalog, validates its
+        columns, filters it based on WCS, and updates the image model's metadata.
 
-        # group images by their "group id":
-        group_indices = images.group_indices
+        Parameters
+        ----------
+        images : ModelLibrary
+            A collection of image models to be updated with tweak registration catalogs.
 
-        self.log.info("")
-        self.log.info(f"Number of image groups to be aligned: {len(group_indices):d}.")
-        self.log.info("Image groups:")
+        Returns
+        -------
+        None
 
-        imcats = self.build_image_catalogs(images)
+        Raises
+        ------
+        AttributeError
+            If the required source detection metadata is missing from an image model.
 
-        if len(group_indices) > 1:
+        Logs
+        -----
+        Information about the number of detected sources is logged for each image model.
+        """
+
+        with images:
+            for i, image_model in enumerate(images):
+                exposure_type = image_model.meta.exposure.type
+                if exposure_type != "WFI_IMAGE":
+                    self.log.info("Skipping TweakReg for spectral exposure.")
+                    image_model.meta.cal_step.tweakreg = "SKIPPED"
+                    images.shelve(image_model)
+                    return image_model
+
+                source_detection = getattr(image_model.meta, "source_detection", None)
+                if source_detection is None:
+                    images.shelve(image_model, i, modify=False)
+                    raise AttributeError(
+                        "Attribute 'meta.source_detection' is missing. "
+                        "Please either run SourceDetectionStep or provide a custom source catalog."
+                    )
+
+                catalog = self.get_tweakreg_catalog(source_detection, image_model, i)
+
+                for axis in ["x", "y"]:
+                    _validate_catalog_columns(catalog, axis, image_model, i)
+
+                filename = image_model.meta.filename
+                catalog = self.filter_catalog_by_wcs(catalog, image_model, filename)
+
+                if self.save_abs_catalog:
+                    self.save_abs_ref_catalog(catalog)
+
+                image_model.meta["tweakreg_catalog"] = catalog.as_array()
+                nsources = len(catalog)
+                self.log.info(
+                    f"Detected {nsources} sources in {filename}."
+                    if nsources
+                    else f"No sources found in {filename}."
+                )
+                images.shelve(image_model, i)
+
+    def do_relative_alignment(self, images, imcats):
+        if len(images.group_indices) > 1:
             # local align images:
             tweakreg.relative_align(
                 imcats,
@@ -134,13 +305,9 @@ def process(self, input):
                 sigma=self.sigma,
             )
 
-        self.abs_refcat = self.abs_refcat.strip()
-        gaia_cat_name = self.abs_refcat.upper()
-
-        if gaia_cat_name in SINGLE_GROUP_REFCAT:
-            with images:
-                ref_image = images.borrow(0)
-                images.shelve(ref_image, 0, modify=False)
+    def do_absolute_alignment(self, images, imcats):
+        if self.abs_refcat in SINGLE_GROUP_REFCAT:
+            ref_image = _get_reference_image(images)
 
             tweakreg.absolute_align(
                 imcats,
@@ -160,11 +327,25 @@ def process(self, input):
                 abs_catalog_output_dir=self.output_dir,
             )
 
-        self.finalize_step(images, imcats)
-
-        return images
-
     def finalize_step(self, images, imcats):
+        """
+        Finalize the tweak registration step by updating image metadata and WCS information.
+
+        This method iterates through the provided image catalogs, marking TweakRegStep as complete,
+        removing the source catalog, and updating the WCS if the fit was successful.
+        It also serializes fit results for storage in the image model's metadata.
+
+        Parameters
+        ----------
+        images : ModelLibrary
+            A collection of image models to be updated.
+        imcats : list
+            A collection of image catalogs containing fit information.
+
+        Returns
+        -------
+        None
+        """
         with images:
             for i, imcat in enumerate(imcats):
                 image_model = images.borrow(i)
@@ -210,49 +391,29 @@ def finalize_step(self, images, imcats):
                     image_model.meta.wcs = imcat.wcs
                 images.shelve(image_model, i)
 
-    @staticmethod
-    def handle_input(step_input):
-        try:
-            if isinstance(step_input, rdm.DataModel):
-                images = ModelLibrary([step_input])
-            elif str(step_input).endswith(".asdf"):
-                images = ModelLibrary([rdm.open(step_input)])
-            elif isinstance(step_input, ModelLibrary):
-                images = step_input
-            else:
-                images = ModelLibrary(step_input)
-        except TypeError as e:
-            e.args = (
-                "Input to tweakreg must be a list of DataModels, an "
-                "association, or an already open ModelLibrary "
-                "containing one or more DataModels.",
-            ) + e.args[1:]
-            raise e
-        return images
-
-    @staticmethod
-    def build_image_catalogs(images):
-        imcats = []
-        with images:
-            for i, m in enumerate(images):
-                # catalog name
-                catalog_name = os.path.splitext(m.meta.filename)[0].strip("_- ")
-                # catalog data
-                catalog_table = Table(m.meta.tweakreg_catalog)
-                catalog_table.meta["name"] = catalog_name
-
-                imcats.append(
-                    tweakreg.construct_wcs_corrector(
-                        wcs=m.meta.wcs,
-                        refang=m.meta.wcsinfo,
-                        catalog=catalog_table,
-                        group_id=m.meta.group_id,
-                    )
-                )
-                images.shelve(m, i, modify=False)
-        return imcats
-
     def read_catalog(self, catalog_name):
+        """
+        Reads a source catalog from a specified file.
+
+        This function determines the format of the catalog based on the file extension.
+        If the file ends with "asdf", it uses a specific method to open and read the catalog;
+        otherwise, it reads the catalog using a standard table format.
+
+        Parameters
+        ----------
+        catalog_name : str
+            The name of the catalog file to read.
+
+        Returns
+        -------
+        Table
+            The read catalog as a Table object.
+
+        Raises
+        ------
+        ValueError
+            If the catalog format is unsupported.
+        """
         if catalog_name.endswith("asdf"):
             with rdm.open(catalog_name) as source_catalog_model:
                 catalog = source_catalog_model.source_catalog
@@ -261,44 +422,54 @@ def read_catalog(self, catalog_name):
         return catalog
 
     def save_abs_ref_catalog(self, catalog_table: Table):
+        """
+        Save the absolute reference catalog to a specified file.
+
+        This method writes the provided catalog table to a file in the specified
+        format and location, using a naming convention based on the absolute
+        reference catalog.
+
+        Parameters
+        ----------
+        catalog_table : Table
+            The catalog table to be saved as an output file.
+
+        Returns
+        -------
+        None
+        """
         output_name = os.path.join(
             self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
         )
         catalog_table.write(output_name, format=self.catalog_format, overwrite=True)
 
-    def validate_custom_catalogs(self, catdict, images):
-        use_custom_catalogs = self.use_custom_catalogs
-        # if user requested the use of custom catalogs and provided a
-        # valid 'catfile' file name that has no custom catalogs,
-        # turn off the use of custom catalogs:
-        if catdict is not None and not catdict:
-            self.log.warning(
-                "'use_custom_catalogs' is set to True but 'catfile' "
-                "contains no user catalogs."
-            )
-            use_custom_catalogs = False
-
-        if use_custom_catalogs and catdict:
-            with images:
-                for i, member in enumerate(images.asn["products"][0]["members"]):
-                    filename = member["expname"]
-                    if filename in catdict:
-                        # FIXME: I'm not sure if this captures all the possible combinations
-                        # for example, meta.tweakreg_catalog is set by the container (when
-                        # it's present in the association). However the code in this step
-                        # checks meta.source_catalog.tweakreg_catalog. I think this means
-                        # that setting a catalog via an association does not work. Is this
-                        # intended? If so, the container can be updated to not support that.
-                        model = images.borrow(i)
-                        model.meta["source_detection"] = {
-                            "tweakreg_catalog_name": catdict[filename],
-                        }
-                        images.shelve(model, i)
-                    else:
-                        images.shelve(model, i, modify=False)
-
     def get_tweakreg_catalog(self, source_detection, image_model, index):
-        """Retrieve the tweakreg catalog from source detection."""
+        """
+        Retrieve the tweakreg catalog from source detection.
+
+        This method checks the source detection metadata for the presence of a
+        tweakreg catalog data or a string with its name. It returns the catalog
+        as a Table object if either is found, or raises an error if neither is available.
+
+        Parameters
+        ----------
+        source_detection : object
+            The source detection metadata containing catalog information.
+        image_model : DataModel
+            The image model associated with the source detection.
+        index : int
+            The index of the image model in the collection.
+
+        Returns
+        -------
+        Table
+            The retrieved tweakreg catalog as a Table object.
+
+        Raises
+        ------
+        AttributeError
+            If the required catalog information is missing from the source detection.
+        """
         if hasattr(source_detection, "tweakreg_catalog"):
             tweakreg_catalog = Table(np.asarray(source_detection.tweakreg_catalog))
             del image_model.meta.source_detection["tweakreg_catalog"]
@@ -312,22 +483,32 @@ def get_tweakreg_catalog(self, source_detection, image_model, index):
                 "Please either run SourceDetectionStep or provide a custom source catalog."
             )
 
-    @staticmethod
-    def validate_catalog_columns(catalog, axis, image_model, index):
-        """Validate the presence of required columns in the catalog."""
-        if axis not in catalog.colnames:
-            long_axis = f"{axis}centroid"
-            if long_axis in catalog.colnames:
-                catalog.rename_column(long_axis, axis)
-            else:
-                images.shelve(image_model, index, modify=False)
-                raise ValueError(
-                    "'tweakreg' source catalogs must contain a header with "
-                    "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
-                )
-
     def filter_catalog_by_wcs(self, catalog, image_model, filename):
-        """Filter sources in the catalog based on WCS bounding box."""
+        """
+        Filter sources in the catalog based on WCS bounding box.
+
+        This method removes sources from the catalog that fall outside the
+        specified WCS bounding box. If no bounding box is defined, it checks
+        the validity of the sources' world coordinates and filters accordingly.
+
+        Parameters
+        ----------
+        catalog : Table
+            The catalog containing source information to be filtered.
+        image_model : DataModel
+            The image model associated with the catalog, used to access WCS information.
+        filename : str
+            The name of the file associated with the catalog, used for logging.
+
+        Returns
+        -------
+        Table
+            The filtered catalog containing only sources within the bounding box.
+
+        Logs
+        -----
+        Information about the number of sources removed from the catalog is logged.
+        """
         bb = image_model.meta.wcs.bounding_box
         x, y = catalog["x"], catalog["y"]
 
@@ -350,46 +531,33 @@ def filter_catalog_by_wcs(self, catalog, image_model, filename):
 
         return catalog
 
-    def set_tweakreg_catalog_attribute(self, images):
-        with images:
-            for i, image_model in enumerate(images):
-                exposure_type = image_model.meta.exposure.type
-                if exposure_type != "WFI_IMAGE":
-                    self.log.info("Skipping TweakReg for spectral exposure.")
-                    image_model.meta.cal_step.tweakreg = "SKIPPED"
-                    images.shelve(image_model)
-                    return image_model
-
-                source_detection = getattr(image_model.meta, "source_detection", None)
-                if source_detection is None:
-                    images.shelve(image_model, i, modify=False)
-                    raise AttributeError(
-                        "Attribute 'meta.source_detection' is missing. "
-                        "Please either run SourceDetectionStep or provide a custom source catalog."
-                    )
-
-                catalog = self.get_tweakreg_catalog(source_detection, image_model, i)
-
-                for axis in ["x", "y"]:
-                    self.validate_catalog_columns(catalog, axis, image_model, i)
-
-                filename = image_model.meta.filename
-                catalog = self.filter_catalog_by_wcs(catalog, image_model, filename)
-
-                if self.save_abs_catalog:
-                    self.save_abs_ref_catalog(catalog)
-
-                image_model.meta["tweakreg_catalog"] = catalog.as_array()
-                nsources = len(catalog)
-                self.log.info(
-                    f"Detected {nsources} sources in {filename}."
-                    if nsources
-                    else f"No sources found in {filename}."
-                )
-                images.shelve(image_model, i)
-
 
 def _parse_catfile(catfile):
+    """
+    Parse a catalog file and return a dictionary mapping data models to catalog paths.
+
+    This function reads a specified catalog file, extracting data model names and
+    their associated catalog paths. It supports a format where each line contains
+    a data model followed by an optional catalog path, and it ensures that the
+    file adheres to the expected structure.
+
+    Parameters
+    ----------
+    catfile : str
+        The path to the catalog file to be parsed.
+
+    Returns
+    -------
+    dict or None
+        A dictionary mapping data model names to catalog paths, or None if the
+        input file is empty or invalid.
+
+    Raises
+    ------
+    ValueError
+        If the catalog file contains more than two columns per line.
+    """
+
     if catfile is None or not catfile.strip():
         return None
 
@@ -413,3 +581,89 @@ def _parse_catfile(catfile):
                 raise ValueError("'catfile' can contain at most two columns.")
 
     return catdict
+
+
+def _build_image_catalogs(images) -> List:
+    """
+    Build image catalogs from the provided images.
+
+    This method constructs a list of image catalogs by extracting the necessary
+    metadata from each image model. It creates a WCS corrector for each image
+    based on its associated catalog and metadata.
+
+    Parameters
+    ----------
+    images : ModelLibrary
+        A collection of image models from which to build catalogs.
+
+    Returns
+    -------
+    imcats : list
+        A list of image catalogs constructed from the input images.
+    """
+    imcats = []
+    with images:
+        for i, m in enumerate(images):
+            # catalog name
+            catalog_name = os.path.splitext(m.meta.filename)[0].strip("_- ")
+            # catalog data
+            catalog_table = Table(m.meta.tweakreg_catalog)
+            catalog_table.meta["name"] = catalog_name
+
+            imcats.append(
+                tweakreg.construct_wcs_corrector(
+                    wcs=m.meta.wcs,
+                    refang=m.meta.wcsinfo,
+                    catalog=catalog_table,
+                    group_id=m.meta.group_id,
+                )
+            )
+            images.shelve(m, i, modify=False)
+    return imcats
+
+
+def _get_reference_image(images):
+    with images:
+        ref_image = images.borrow(0)
+        images.shelve(ref_image, 0, modify=False)
+    return ref_image
+
+
+def _validate_catalog_columns(catalog, axis, image_model, index):
+    """
+    Validate the presence of required columns in the catalog.
+
+    This method checks if the specified axis column exists in the catalog.
+    If the axis is not found, it looks for a corresponding centroid column
+    and renames it if present. If neither is found, it raises an error.
+
+    Parameters
+    ----------
+    catalog : Table
+        The catalog to validate, which should contain source information.
+    axis : str
+        The axis to check for in the catalog (e.g., 'x' or 'y').
+    image_model : DataModel
+        The image model associated with the catalog.
+    index : int
+        The index of the image model in the collection.
+
+    Returns
+    -------
+    None
+
+    Raises
+    ------
+    ValueError
+        If the required columns are missing from the catalog.
+    """
+    if axis not in catalog.colnames:
+        long_axis = f"{axis}centroid"
+        if long_axis in catalog.colnames:
+            catalog.rename_column(long_axis, axis)
+        else:
+            images.shelve(image_model, index, modify=False)
+            raise ValueError(
+                "'tweakreg' source catalogs must contain a header with "
+                "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
+            )

From c033702e971965732081ebc70a221af6502ccc79 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Fri, 30 Aug 2024 16:48:51 -0400
Subject: [PATCH 05/74] Replace local method with stcal's.

---
 romancal/tweakreg/tweakreg_step.py | 52 ++----------------------------
 1 file changed, 3 insertions(+), 49 deletions(-)

diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index f7dc7c865..c998cc888 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -272,7 +272,9 @@ def set_tweakreg_catalog_attribute(self, images):
                     _validate_catalog_columns(catalog, axis, image_model, i)
 
                 filename = image_model.meta.filename
-                catalog = self.filter_catalog_by_wcs(catalog, image_model, filename)
+                catalog = tweakreg.filter_catalog_by_bounding_box(
+                    catalog, image_model.meta.wcs.bounding_box
+                )
 
                 if self.save_abs_catalog:
                     self.save_abs_ref_catalog(catalog)
@@ -483,54 +485,6 @@ def get_tweakreg_catalog(self, source_detection, image_model, index):
                 "Please either run SourceDetectionStep or provide a custom source catalog."
             )
 
-    def filter_catalog_by_wcs(self, catalog, image_model, filename):
-        """
-        Filter sources in the catalog based on WCS bounding box.
-
-        This method removes sources from the catalog that fall outside the
-        specified WCS bounding box. If no bounding box is defined, it checks
-        the validity of the sources' world coordinates and filters accordingly.
-
-        Parameters
-        ----------
-        catalog : Table
-            The catalog containing source information to be filtered.
-        image_model : DataModel
-            The image model associated with the catalog, used to access WCS information.
-        filename : str
-            The name of the file associated with the catalog, used for logging.
-
-        Returns
-        -------
-        Table
-            The filtered catalog containing only sources within the bounding box.
-
-        Logs
-        -----
-        Information about the number of sources removed from the catalog is logged.
-        """
-        bb = image_model.meta.wcs.bounding_box
-        x, y = catalog["x"], catalog["y"]
-
-        if bb is None:
-            r, d = image_model.meta.wcs(x, y)
-            mask = np.isfinite(r) & np.isfinite(d)
-        else:
-            ((xmin, xmax), (ymin, ymax)) = bb
-            mask = (x > xmin) & (x < xmax) & (y > ymin) & (y < ymax)
-
-        catalog = catalog[mask]
-        n_removed_src = np.sum(np.logical_not(mask))
-        if n_removed_src:
-            self.log.info(
-                f"Removed {n_removed_src} sources from {filename}'s "
-                "catalog that were outside of the bounding box."
-                if bb
-                else f"catalog whose image coordinates could not be converted to world coordinates."
-            )
-
-        return catalog
-
 
 def _parse_catfile(catfile):
     """

From f015feb85d31de81817a799fd9155c362e0a2916 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 3 Sep 2024 21:03:18 -0400
Subject: [PATCH 06/74] Further code refactoring.

---
 romancal/tweakreg/tweakreg_step.py | 354 +++++++++++++++++------------
 1 file changed, 209 insertions(+), 145 deletions(-)

diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index c998cc888..66a56595c 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -81,6 +81,8 @@ def process(self, input):
 
         # properly handle input
         images = self.parse_input(step_input=input)
+        # set the first image as reference
+        ref_image = _set_reference_image(images)
 
         catdict = _parse_catfile(self.catfile)
 
@@ -90,16 +92,18 @@ def process(self, input):
         # set path where the source catalog will be saved to
         self.set_catalog_path()
 
-        self.set_reference_catalog()
+        self.set_reference_catalog_name()
 
         # Build the catalogs for input images
         self.set_tweakreg_catalog_attribute(images)
 
         imcats = _build_image_catalogs(images)
 
-        self.do_relative_alignment(images, imcats)
+        if getattr(images, "group_indices", None) and len(images.group_indices) > 1:
+            self.do_relative_alignment(imcats)
 
-        self.do_absolute_alignment(images, imcats)
+        if self.abs_refcat in SINGLE_GROUP_REFCAT:
+            self.do_absolute_alignment(ref_image, imcats)
 
         self.finalize_step(images, imcats)
 
@@ -146,7 +150,7 @@ def parse_input(self, step_input) -> ModelLibrary:
             ) + e.args[1:]
             raise e
 
-        if len(images) == 0:
+        if not images:
             raise ValueError("Input must contain at least one image model.")
 
         self.log.info("")
@@ -208,19 +212,131 @@ def validate_custom_catalogs(self, catdict, images):
                         images.shelve(model, i, modify=False)
 
     def set_catalog_path(self):
+        """
+        Set the path for saving source catalogs.
+
+        This method checks if the catalog path (where all source catalogs will be saved)
+        is empty and, if so, sets it to the current working directory.
+
+        Returns
+        -------
+        None
+        """
         if len(self.catalog_path) == 0:
             self.catalog_path = os.getcwd()
         self.catalog_path = Path(self.catalog_path).as_posix()
         self.log.info(f"All source catalogs will be saved to: {self.catalog_path}")
 
-    def set_reference_catalog(self):
-        if self.abs_refcat is None or len(self.abs_refcat) == 0:
+    def set_reference_catalog_name(self):
+        """
+        Set the name of the absolute reference catalog.
+
+        This method checks if the absolute reference catalog name is not set or
+        is empty, and if so, assigns it a default value. If the absolute reference
+        catalog name is different from the default, it enables the expansion of
+        the reference catalog to avoid duplicate entries during alignment.
+
+        Returns
+        -------
+        None
+        """
+        if not self.abs_refcat:
             self.abs_refcat = DEFAULT_ABS_REFCAT.strip().upper()
         if self.abs_refcat != DEFAULT_ABS_REFCAT:
-            # Set expand_refcat to True to eliminate possibility of duplicate
-            # entries when aligning to absolute astrometric reference catalog
             self.expand_refcat = True
 
+    def read_catalog(self, catalog_name):
+        """
+        Reads a source catalog from a specified file.
+
+        This function determines the format of the catalog based on the file extension.
+        If the file ends with "asdf", it uses a specific method to open and read the catalog;
+        otherwise, it reads the catalog using a standard table format.
+
+        Parameters
+        ----------
+        catalog_name : str
+            The name of the catalog file to read.
+
+        Returns
+        -------
+        Table
+            The read catalog as a Table object.
+
+        Raises
+        ------
+        ValueError
+            If the catalog format is unsupported.
+        """
+        if catalog_name.endswith("asdf"):
+            with rdm.open(catalog_name) as source_catalog_model:
+                catalog = source_catalog_model.source_catalog
+        else:
+            catalog = Table.read(catalog_name, format=self.catalog_format)
+        return catalog
+
+    def save_abs_ref_catalog(self, catalog_table: Table):
+        """
+        Save the absolute reference catalog to a specified file.
+
+        This method writes the provided catalog table to a file in the specified
+        format and location, using a naming convention based on the absolute
+        reference catalog.
+
+        Parameters
+        ----------
+        catalog_table : Table
+            The catalog table to be saved as an output file.
+
+        Returns
+        -------
+        None
+        """
+        output_name = os.path.join(
+            self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
+        )
+        catalog_table.write(output_name, format=self.catalog_format, overwrite=True)
+
+    def get_tweakreg_catalog(self, source_detection, image_model, index):
+        """
+        Retrieve the tweakreg catalog from source detection.
+
+        This method checks the source detection metadata for the presence of a
+        tweakreg catalog data or a string with its name. It returns the catalog
+        as a Table object if either is found, or raises an error if neither is available.
+
+        Parameters
+        ----------
+        source_detection : object
+            The source detection metadata containing catalog information.
+        image_model : DataModel
+            The image model associated with the source detection.
+        index : int
+            The index of the image model in the collection.
+
+        Returns
+        -------
+        Table
+            The retrieved tweakreg catalog as a Table object.
+
+        Raises
+        ------
+        AttributeError
+            If the required catalog information is missing from the source detection.
+        """
+        if getattr(source_detection, "tweakreg_catalog", None):
+            tweakreg_catalog = Table(np.asarray(source_detection.tweakreg_catalog))
+            del image_model.meta.source_detection["tweakreg_catalog"]
+            return tweakreg_catalog
+
+        if getattr(source_detection, "tweakreg_catalog_name", None):
+            return self.read_catalog(source_detection.tweakreg_catalog_name)
+
+        raise AttributeError(
+            "Attribute 'meta.source_detection.tweakreg_catalog' is missing. "
+            "Please either run SourceDetectionStep or provide a custom source catalog."
+        )
+
     def set_tweakreg_catalog_attribute(self, images):
         """
         Set the tweakreg catalog attribute for each image model.
@@ -228,7 +344,7 @@ def set_tweakreg_catalog_attribute(self, images):
         This method iterates through the provided image models, checking the
         exposure type and ensuring that the necessary source detection metadata
         is present. It retrieves the tweak registration catalog, validates its
-        columns, filters it based on WCS, and updates the image model's metadata.
+        columns, filters it based on bounding box, and updates the image model's metadata.
 
         Parameters
         ----------
@@ -266,10 +382,22 @@ def set_tweakreg_catalog_attribute(self, images):
                         "Please either run SourceDetectionStep or provide a custom source catalog."
                     )
 
-                catalog = self.get_tweakreg_catalog(source_detection, image_model, i)
+                try:
+                    catalog = self.get_tweakreg_catalog(
+                        source_detection, image_model, i
+                    )
+                except AttributeError as e:
+                    self.log.error(f"Failed to retrieve tweakreg_catalog: {e}")
+                    images.shelve(image_model, i, modify=False)
+                    raise AttributeError() from e
 
-                for axis in ["x", "y"]:
-                    _validate_catalog_columns(catalog, axis, image_model, i)
+                try:
+                    for axis in ["x", "y"]:
+                        _validate_catalog_columns(catalog, axis, image_model, i)
+                except ValueError as e:
+                    self.log.error(f"Failed to validate catalog columns: {e}")
+                    images.shelve(image_model, i, modify=False)
+                    raise ValueError() from e
 
                 filename = image_model.meta.filename
                 catalog = tweakreg.filter_catalog_by_bounding_box(
@@ -288,46 +416,75 @@ def set_tweakreg_catalog_attribute(self, images):
                 )
                 images.shelve(image_model, i)
 
-    def do_relative_alignment(self, images, imcats):
-        if len(images.group_indices) > 1:
-            # local align images:
-            tweakreg.relative_align(
-                imcats,
-                searchrad=self.searchrad,
-                separation=self.separation,
-                use2dhist=self.use2dhist,
-                tolerance=self.tolerance,
-                xoffset=0,
-                yoffset=0,
-                enforce_user_order=self.enforce_user_order,
-                expand_refcat=self.expand_refcat,
-                minobj=self.minobj,
-                fitgeometry=self.fitgeometry,
-                nclip=self.nclip,
-                sigma=self.sigma,
-            )
+    def do_relative_alignment(self, imcats):
+        """
+        Perform relative alignment of images.
 
-    def do_absolute_alignment(self, images, imcats):
-        if self.abs_refcat in SINGLE_GROUP_REFCAT:
-            ref_image = _get_reference_image(images)
-
-            tweakreg.absolute_align(
-                imcats,
-                self.abs_refcat,
-                ref_wcs=ref_image.meta.wcs,
-                ref_wcsinfo=ref_image.meta.wcsinfo,
-                epoch=ref_image.meta.exposure.mid_time.decimalyear,
-                abs_minobj=self.abs_minobj,
-                abs_fitgeometry=self.abs_fitgeometry,
-                abs_nclip=self.abs_nclip,
-                abs_sigma=self.abs_sigma,
-                abs_searchrad=self.abs_searchrad,
-                abs_use2dhist=self.abs_use2dhist,
-                abs_separation=self.abs_separation,
-                abs_tolerance=self.abs_tolerance,
-                save_abs_catalog=self.save_abs_catalog,
-                abs_catalog_output_dir=self.output_dir,
-            )
+        This method performs relative alignment with the specified parameters,
+        including search radius, separation, and fitting geometry.
+
+        Parameters
+        ----------
+        imcats : list
+            A list of image catalogs containing source information for alignment.
+
+        Returns
+        -------
+        None
+        """
+        tweakreg.relative_align(
+            imcats,
+            searchrad=self.searchrad,
+            separation=self.separation,
+            use2dhist=self.use2dhist,
+            tolerance=self.tolerance,
+            xoffset=0,
+            yoffset=0,
+            enforce_user_order=self.enforce_user_order,
+            expand_refcat=self.expand_refcat,
+            minobj=self.minobj,
+            fitgeometry=self.fitgeometry,
+            nclip=self.nclip,
+            sigma=self.sigma,
+        )
+
+    def do_absolute_alignment(self, ref_image, imcats):
+        """
+        Perform absolute alignment of images.
+
+        This method retrieves a reference image and performs absolute alignment
+        using the specified parameters, including reference WCS information and
+        catalog details. It aligns the provided image catalogs to the absolute
+        reference catalog.
+
+        Parameters
+        ----------
+        ref_image : DataModel
+            The reference image used for alignment, which contains WCS information.
+        imcats : list
+            A list of image catalogs containing source information for alignment.
+
+        Returns
+        -------
+        None
+        """
+        tweakreg.absolute_align(
+            imcats,
+            self.abs_refcat,
+            ref_wcs=ref_image.meta.wcs,
+            ref_wcsinfo=ref_image.meta.wcsinfo,
+            epoch=ref_image.meta.exposure.mid_time.decimalyear,
+            abs_minobj=self.abs_minobj,
+            abs_fitgeometry=self.abs_fitgeometry,
+            abs_nclip=self.abs_nclip,
+            abs_sigma=self.abs_sigma,
+            abs_searchrad=self.abs_searchrad,
+            abs_use2dhist=self.abs_use2dhist,
+            abs_separation=self.abs_separation,
+            abs_tolerance=self.abs_tolerance,
+            save_abs_catalog=self.save_abs_catalog,
+            abs_catalog_output_dir=self.output_dir,
+        )
 
     def finalize_step(self, images, imcats):
         """
@@ -393,98 +550,6 @@ def finalize_step(self, images, imcats):
                     image_model.meta.wcs = imcat.wcs
                 images.shelve(image_model, i)
 
-    def read_catalog(self, catalog_name):
-        """
-        Reads a source catalog from a specified file.
-
-        This function determines the format of the catalog based on the file extension.
-        If the file ends with "asdf", it uses a specific method to open and read the catalog;
-        otherwise, it reads the catalog using a standard table format.
-
-        Parameters
-        ----------
-        catalog_name : str
-            The name of the catalog file to read.
-
-        Returns
-        -------
-        Table
-            The read catalog as a Table object.
-
-        Raises
-        ------
-        ValueError
-            If the catalog format is unsupported.
-        """
-        if catalog_name.endswith("asdf"):
-            with rdm.open(catalog_name) as source_catalog_model:
-                catalog = source_catalog_model.source_catalog
-        else:
-            catalog = Table.read(catalog_name, format=self.catalog_format)
-        return catalog
-
-    def save_abs_ref_catalog(self, catalog_table: Table):
-        """
-        Save the absolute reference catalog to a specified file.
-
-        This method writes the provided catalog table to a file in the specified
-        format and location, using a naming convention based on the absolute
-        reference catalog.
-
-        Parameters
-        ----------
-        catalog_table : Table
-            The catalog table to be saved as an output file.
-
-        Returns
-        -------
-        None
-        """
-        output_name = os.path.join(
-            self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
-        )
-        catalog_table.write(output_name, format=self.catalog_format, overwrite=True)
-
-    def get_tweakreg_catalog(self, source_detection, image_model, index):
-        """
-        Retrieve the tweakreg catalog from source detection.
-
-        This method checks the source detection metadata for the presence of a
-        tweakreg catalog data or a string with its name. It returns the catalog
-        as a Table object if either is found, or raises an error if neither is available.
-
-        Parameters
-        ----------
-        source_detection : object
-            The source detection metadata containing catalog information.
-        image_model : DataModel
-            The image model associated with the source detection.
-        index : int
-            The index of the image model in the collection.
-
-        Returns
-        -------
-        Table
-            The retrieved tweakreg catalog as a Table object.
-
-        Raises
-        ------
-        AttributeError
-            If the required catalog information is missing from the source detection.
-        """
-        if hasattr(source_detection, "tweakreg_catalog"):
-            tweakreg_catalog = Table(np.asarray(source_detection.tweakreg_catalog))
-            del image_model.meta.source_detection["tweakreg_catalog"]
-            return tweakreg_catalog
-        elif hasattr(source_detection, "tweakreg_catalog_name"):
-            return self.read_catalog(source_detection.tweakreg_catalog_name)
-        else:
-            images.shelve(image_model, index, modify=False)
-            raise AttributeError(
-                "Attribute 'meta.source_detection.tweakreg_catalog' is missing. "
-                "Please either run SourceDetectionStep or provide a custom source catalog."
-            )
-
 
 def _parse_catfile(catfile):
     """
@@ -576,7 +641,7 @@ def _build_image_catalogs(images) -> List:
     return imcats
 
 
-def _get_reference_image(images):
+def _set_reference_image(images):
     with images:
         ref_image = images.borrow(0)
         images.shelve(ref_image, 0, modify=False)
@@ -616,7 +681,6 @@ def _validate_catalog_columns(catalog, axis, image_model, index):
         if long_axis in catalog.colnames:
             catalog.rename_column(long_axis, axis)
         else:
-            images.shelve(image_model, index, modify=False)
             raise ValueError(
                 "'tweakreg' source catalogs must contain a header with "
                 "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."

From a99761c3f5e2d23a03037156a0fd948a3ec2f8a9 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 4 Sep 2024 01:07:13 +0000
Subject: [PATCH 07/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/tweakreg/tests/test_tweakreg.py | 1 -
 romancal/tweakreg/tweakreg_step.py       | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index 636da6dfc..bb2a80f09 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -13,7 +13,6 @@
 from astropy import units as u
 from astropy.modeling import models
 from astropy.modeling.models import RotationSequence3D, Scale, Shift
-from astropy.table import Table
 from astropy.time import Time
 from gwcs import coordinate_frames as cf
 from gwcs import wcs
diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index 66a56595c..62119ae9b 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -4,12 +4,12 @@
 
 import os
 from pathlib import Path
+from typing import List
 
 import numpy as np
 from astropy.table import Table
 from roman_datamodels import datamodels as rdm
 from stcal.tweakreg import tweakreg
-from typing import List
 
 # LOCAL
 from ..datamodels import ModelLibrary

From 79b667c01dcdedadae315c3526e29852523738d0 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Wed, 4 Sep 2024 13:05:47 -0400
Subject: [PATCH 08/74] Utilize stcal's get_catalog method.

---
 romancal/tweakreg/tests/test_tweakreg.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index bb2a80f09..f67d6a89a 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -22,7 +22,7 @@
 
 from romancal.datamodels import ModelLibrary
 from romancal.tweakreg import tweakreg_step as trs
-from romancal.tweakreg.astrometric_utils import get_catalog
+from stcal.tweakreg.astrometric_utils import get_catalog
 
 
 class MockConnectionError:
@@ -339,7 +339,9 @@ def create_wcs_for_tweakreg_pipeline(input_dm, shift_1=0, shift_2=0):
 
 
 def get_catalog_data(input_dm):
-    gaia_cat = get_catalog(ra=270, dec=66, sr=100 / 3600)
+    gaia_cat = get_catalog(
+        right_ascension=270, declination=66, search_radius=100 / 3600
+    )
     gaia_source_coords = [(ra, dec) for ra, dec in zip(gaia_cat["ra"], gaia_cat["dec"])]
     catalog_data = np.array(
         [input_dm.meta.wcs.world_to_pixel(ra, dec) for ra, dec in gaia_source_coords]
@@ -761,7 +763,9 @@ def test_tweakreg_rotated_plane(tmp_path, theta, offset_x, offset_y, request):
     """
     Test that TweakReg returns accurate results.
     """
-    gaia_cat = get_catalog(ra=270, dec=66, sr=100 / 3600)
+    gaia_cat = get_catalog(
+        right_ascension=270, declination=66, search_radius=100 / 3600
+    )
     gaia_source_coords = [(ra, dec) for ra, dec in zip(gaia_cat["ra"], gaia_cat["dec"])]
 
     img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000)

From 42fef2c738a1434e630b958e0ba7f5358975cf2c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 4 Sep 2024 17:06:48 +0000
Subject: [PATCH 09/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/tweakreg/tests/test_tweakreg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index f67d6a89a..4c3dcbe92 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -19,10 +19,10 @@
 from gwcs.geometry import CartesianToSpherical, SphericalToCartesian
 from roman_datamodels import datamodels as rdm
 from roman_datamodels import maker_utils
+from stcal.tweakreg.astrometric_utils import get_catalog
 
 from romancal.datamodels import ModelLibrary
 from romancal.tweakreg import tweakreg_step as trs
-from stcal.tweakreg.astrometric_utils import get_catalog
 
 
 class MockConnectionError:

From 8991f9fcc1e939618689ed4c7101df952ae16646 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Thu, 5 Sep 2024 13:14:45 -0400
Subject: [PATCH 10/74] Add clip_accum=True to call to alignment methods.

---
 romancal/tweakreg/tweakreg_step.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index 62119ae9b..b6ebd9db4 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -446,6 +446,7 @@ def do_relative_alignment(self, imcats):
             fitgeometry=self.fitgeometry,
             nclip=self.nclip,
             sigma=self.sigma,
+            clip_accum=True,
         )
 
     def do_absolute_alignment(self, ref_image, imcats):
@@ -484,6 +485,7 @@ def do_absolute_alignment(self, ref_image, imcats):
             abs_tolerance=self.abs_tolerance,
             save_abs_catalog=self.save_abs_catalog,
             abs_catalog_output_dir=self.output_dir,
+            clip_accum=True,
         )
 
     def finalize_step(self, images, imcats):

From 76eb339ee81928929c9f56bc01870c9b4baaa6ff Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 10 Sep 2024 16:00:19 -0400
Subject: [PATCH 11/74] Point stcal installation to main branch.

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 1ff281a8c..89185451a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,8 +28,8 @@ dependencies = [
     # "roman_datamodels>=0.21.0,<0.22.0",
     "roman_datamodels @ git+https://github.com/mairanteodoro/roman_datamodels.git@RCAL-911-remove-units-from-mosaic-level-pipeline",
     "scipy >=1.11",
-    "stcal>=1.8.0,<1.9.0",
-    # "stcal @ git+https://github.com/spacetelescope/stcal.git@main",
+#    "stcal>=1.8.0,<1.9.0",
+    "stcal @ git+https://github.com/spacetelescope/stcal.git@main",
     "stpipe >=0.7.0,<0.8.0",
     # "stpipe @ git+https://github.com/spacetelescope/stpipe.git@main",
     "tweakwcs >=0.8.8",

From 70914337e6dad691df46a14d8b11f1d80fde97c2 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Wed, 11 Sep 2024 11:41:20 -0400
Subject: [PATCH 12/74] Revert some refactoring.

---
 romancal/tweakreg/tweakreg_step.py | 509 +++++++++--------------------
 1 file changed, 154 insertions(+), 355 deletions(-)

diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index b6ebd9db4..6c6b207ba 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -80,68 +80,15 @@ class TweakRegStep(RomanStep):
     def process(self, input):
 
         # properly handle input
-        images = self.parse_input(step_input=input)
-        # set the first image as reference
-        ref_image = _set_reference_image(images)
-
-        catdict = _parse_catfile(self.catfile)
-
-        if self.use_custom_catalogs:
-            self.validate_custom_catalogs(catdict, images)
-
-        # set path where the source catalog will be saved to
-        self.set_catalog_path()
-
-        self.set_reference_catalog_name()
-
-        # Build the catalogs for input images
-        self.set_tweakreg_catalog_attribute(images)
-
-        imcats = _build_image_catalogs(images)
-
-        if getattr(images, "group_indices", None) and len(images.group_indices) > 1:
-            self.do_relative_alignment(imcats)
-
-        if self.abs_refcat in SINGLE_GROUP_REFCAT:
-            self.do_absolute_alignment(ref_image, imcats)
-
-        self.finalize_step(images, imcats)
-
-        return images
-
-    def parse_input(self, step_input) -> ModelLibrary:
-        """
-        Parse the input for the class.
-
-        This method handles various types of input, including single DataModels,
-        file paths to ASDF files, and collections of DataModels. It ensures that
-        the input is converted into a ModelLibrary for further processing.
-
-        Parameters
-        ----------
-        step_input : Union[rdm.DataModel, str, ModelLibrary, list]
-            The input to be parsed, which can be a DataModel, a file path,
-            a ModelLibrary, or a list of DataModels.
-
-        Returns
-        -------
-        ModelLibrary
-            A ModelLibrary containing the parsed images.
-
-        Raises
-        ------
-        TypeError
-            If the input is not a valid type for processing.
-        """
         try:
-            if isinstance(step_input, rdm.DataModel):
-                images = ModelLibrary([step_input])
-            elif str(step_input).endswith(".asdf"):
-                images = ModelLibrary([rdm.open(step_input)])
-            elif isinstance(step_input, ModelLibrary):
-                images = step_input
+            if isinstance(input, rdm.DataModel):
+                images = ModelLibrary([input])
+            elif str(input).endswith(".asdf"):
+                images = ModelLibrary([rdm.open(input)])
+            elif isinstance(input, ModelLibrary):
+                images = input
             else:
-                images = ModelLibrary(step_input)
+                images = ModelLibrary(input)
         except TypeError as e:
             e.args = (
                 "Input to tweakreg must be a list of DataModels, an "
@@ -158,29 +105,13 @@ def parse_input(self, step_input) -> ModelLibrary:
             f"Number of image groups to be aligned: {len(images.group_indices):d}."
         )
         self.log.info("Image groups:")
+        # set the first image as reference
+        with images:
+            ref_image = images.borrow(0)
+            images.shelve(ref_image, 0, modify=False)
 
-        return images
-
-    def validate_custom_catalogs(self, catdict, images):
-        """
-        Validate and apply custom catalogs for the tweak registration step.
-
-        This method checks if the user has requested the use of custom catalogs
-        and whether the provided catalog file contains valid entries. If valid
-        catalogs are found, it updates the image models with the corresponding
-        catalog names.
-
-        Parameters
-        ----------
-        catdict : dict
-            A dictionary mapping image filenames to custom catalog file paths.
-        images : ModelLibrary
-            A collection of image models to be updated with custom catalog information.
+        catdict = _parse_catfile(self.catfile)
 
-        Returns
-        -------
-        None
-        """
         use_custom_catalogs = self.use_custom_catalogs
         # if user requested the use of custom catalogs and provided a
         # valid 'catfile' file name that has no custom catalogs,
@@ -211,40 +142,158 @@ def validate_custom_catalogs(self, catdict, images):
                     else:
                         images.shelve(model, i, modify=False)
 
-    def set_catalog_path(self):
-        """
-        Set the path for saving source catalogs.
-
-        This method checks if the catalog path (where all source catalogs will be saved)
-        is empty and, if so, sets it to the current working directory.
-
-        Returns
-        -------
-        None
-        """
+        # set path where the source catalog will be saved to
         if len(self.catalog_path) == 0:
             self.catalog_path = os.getcwd()
         self.catalog_path = Path(self.catalog_path).as_posix()
         self.log.info(f"All source catalogs will be saved to: {self.catalog_path}")
 
-    def set_reference_catalog_name(self):
-        """
-        Set the name of the absolute reference catalog.
-
-        This method checks if the absolute reference catalog name is not set or
-        is empty, and if so, assigns it a default value. If the absolute reference
-        catalog name is different from the default, it enables the expansion of
-        the reference catalog to avoid duplicate entries during alignment.
-
-        Returns
-        -------
-        None
-        """
+        # set reference catalog name
         if not self.abs_refcat:
             self.abs_refcat = DEFAULT_ABS_REFCAT.strip().upper()
         if self.abs_refcat != DEFAULT_ABS_REFCAT:
             self.expand_refcat = True
 
+        # build the catalogs for input images
+        with images:
+            for i, image_model in enumerate(images):
+                exposure_type = image_model.meta.exposure.type
+                if exposure_type != "WFI_IMAGE":
+                    self.log.info("Skipping TweakReg for spectral exposure.")
+                    image_model.meta.cal_step.tweakreg = "SKIPPED"
+                    images.shelve(image_model)
+                    return image_model
+
+                source_detection = getattr(image_model.meta, "source_detection", None)
+                if source_detection is None:
+                    images.shelve(image_model, i, modify=False)
+                    raise AttributeError(
+                        "Attribute 'meta.source_detection' is missing. "
+                        "Please either run SourceDetectionStep or provide a custom source catalog."
+                    )
+
+                try:
+                    catalog = self.get_tweakreg_catalog(
+                        source_detection, image_model, i
+                    )
+                except AttributeError as e:
+                    self.log.error(f"Failed to retrieve tweakreg_catalog: {e}")
+                    images.shelve(image_model, i, modify=False)
+                    raise AttributeError() from e
+
+                try:
+                    for axis in ["x", "y"]:
+                        # validate catalog columns
+                        if axis not in catalog.colnames:
+                            long_axis = f"{axis}centroid"
+                            if long_axis in catalog.colnames:
+                                catalog.rename_column(long_axis, axis)
+                            else:
+                                raise ValueError(
+                                    "'tweakreg' source catalogs must contain a header with "
+                                    "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
+                                )
+                except ValueError as e:
+                    self.log.error(f"Failed to validate catalog columns: {e}")
+                    images.shelve(image_model, i, modify=False)
+                    raise ValueError() from e
+
+                filename = image_model.meta.filename
+                catalog = tweakreg.filter_catalog_by_bounding_box(
+                    catalog, image_model.meta.wcs.bounding_box
+                )
+
+                if self.save_abs_catalog:
+                    output_name = os.path.join(
+                        self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
+                    )
+                    catalog.write(
+                        output_name, format=self.catalog_format, overwrite=True
+                    )
+
+                image_model.meta["tweakreg_catalog"] = catalog.as_array()
+                nsources = len(catalog)
+                self.log.info(
+                    f"Detected {nsources} sources in {filename}."
+                    if nsources
+                    else f"No sources found in {filename}."
+                )
+                images.shelve(image_model, i)
+
+        # build image catalogs
+        imcats = []
+        with images:
+            for i, m in enumerate(images):
+                # catalog name
+                catalog_name = os.path.splitext(m.meta.filename)[0].strip("_- ")
+                # catalog data
+                catalog_table = Table(m.meta.tweakreg_catalog)
+                catalog_table.meta["name"] = catalog_name
+
+                imcats.append(
+                    tweakreg.construct_wcs_corrector(
+                        wcs=m.meta.wcs,
+                        refang=m.meta.wcsinfo,
+                        catalog=catalog_table,
+                        group_id=m.meta.group_id,
+                    )
+                )
+                images.shelve(m, i, modify=False)
+
+        if getattr(images, "group_indices", None) and len(images.group_indices) > 1:
+            self.do_relative_alignment(imcats)
+
+        if self.abs_refcat in SINGLE_GROUP_REFCAT:
+            self.do_absolute_alignment(ref_image, imcats)
+
+        # finalize step
+        with images:
+            for i, imcat in enumerate(imcats):
+                image_model = images.borrow(i)
+                image_model.meta.cal_step["tweakreg"] = "COMPLETE"
+                # remove source catalog
+                del image_model.meta["tweakreg_catalog"]
+
+                # retrieve fit status and update wcs if fit is successful:
+                if "SUCCESS" in imcat.meta.get("fit_info")["status"]:
+                    # Update/create the WCS .name attribute with information
+                    # on this astrometric fit as the only record that it was
+                    # successful:
+
+                    # NOTE: This .name attrib agreed upon by the JWST Cal
+                    #       Working Group.
+                    #       Current value is merely a place-holder based
+                    #       on HST conventions. This value should also be
+                    #       translated to the FITS WCSNAME keyword
+                    #       IF that is what gets recorded in the archive
+                    #       for end-user searches.
+                    imcat.wcs.name = f"FIT-LVL2-{self.abs_refcat}"
+
+                    # serialize object from tweakwcs
+                    # (typecasting numpy objects to python types so that it doesn't cause an
+                    # issue when saving datamodel to ASDF)
+                    wcs_fit_results = {
+                        k: v.tolist() if isinstance(v, (np.ndarray, np.bool_)) else v
+                        for k, v in imcat.meta["fit_info"].items()
+                    }
+                    # add fit results and new WCS to datamodel
+                    image_model.meta["wcs_fit_results"] = wcs_fit_results
+                    # remove unwanted keys from WCS fit results
+                    for k in [
+                        "eff_minobj",
+                        "matched_ref_idx",
+                        "matched_input_idx",
+                        "fit_RA",
+                        "fit_DEC",
+                        "fitmask",
+                    ]:
+                        del image_model.meta["wcs_fit_results"][k]
+
+                    image_model.meta.wcs = imcat.wcs
+                images.shelve(image_model, i)
+
+        return images
+
     def read_catalog(self, catalog_name):
         """
         Reads a source catalog from a specified file.
@@ -275,28 +324,6 @@ def read_catalog(self, catalog_name):
             catalog = Table.read(catalog_name, format=self.catalog_format)
         return catalog
 
-    def save_abs_ref_catalog(self, catalog_table: Table):
-        """
-        Save the absolute reference catalog to a specified file.
-
-        This method writes the provided catalog table to a file in the specified
-        format and location, using a naming convention based on the absolute
-        reference catalog.
-
-        Parameters
-        ----------
-        catalog_table : Table
-            The catalog table to be saved as an output file.
-
-        Returns
-        -------
-        None
-        """
-        output_name = os.path.join(
-            self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
-        )
-        catalog_table.write(output_name, format=self.catalog_format, overwrite=True)
-
     def get_tweakreg_catalog(self, source_detection, image_model, index):
         """
         Retrieve the tweakreg catalog from source detection.
@@ -337,85 +364,6 @@ def get_tweakreg_catalog(self, source_detection, image_model, index):
             "Please either run SourceDetectionStep or provide a custom source catalog."
         )
 
-    def set_tweakreg_catalog_attribute(self, images):
-        """
-        Set the tweakreg catalog attribute for each image model.
-
-        This method iterates through the provided image models, checking the
-        exposure type and ensuring that the necessary source detection metadata
-        is present. It retrieves the tweak registration catalog, validates its
-        columns, filters it based on bounding box, and updates the image model's metadata.
-
-        Parameters
-        ----------
-        images : ModelLibrary
-            A collection of image models to be updated with tweak registration catalogs.
-
-        Returns
-        -------
-        None
-
-        Raises
-        ------
-        AttributeError
-            If the required source detection metadata is missing from an image model.
-
-        Logs
-        -----
-        Information about the number of detected sources is logged for each image model.
-        """
-
-        with images:
-            for i, image_model in enumerate(images):
-                exposure_type = image_model.meta.exposure.type
-                if exposure_type != "WFI_IMAGE":
-                    self.log.info("Skipping TweakReg for spectral exposure.")
-                    image_model.meta.cal_step.tweakreg = "SKIPPED"
-                    images.shelve(image_model)
-                    return image_model
-
-                source_detection = getattr(image_model.meta, "source_detection", None)
-                if source_detection is None:
-                    images.shelve(image_model, i, modify=False)
-                    raise AttributeError(
-                        "Attribute 'meta.source_detection' is missing. "
-                        "Please either run SourceDetectionStep or provide a custom source catalog."
-                    )
-
-                try:
-                    catalog = self.get_tweakreg_catalog(
-                        source_detection, image_model, i
-                    )
-                except AttributeError as e:
-                    self.log.error(f"Failed to retrieve tweakreg_catalog: {e}")
-                    images.shelve(image_model, i, modify=False)
-                    raise AttributeError() from e
-
-                try:
-                    for axis in ["x", "y"]:
-                        _validate_catalog_columns(catalog, axis, image_model, i)
-                except ValueError as e:
-                    self.log.error(f"Failed to validate catalog columns: {e}")
-                    images.shelve(image_model, i, modify=False)
-                    raise ValueError() from e
-
-                filename = image_model.meta.filename
-                catalog = tweakreg.filter_catalog_by_bounding_box(
-                    catalog, image_model.meta.wcs.bounding_box
-                )
-
-                if self.save_abs_catalog:
-                    self.save_abs_ref_catalog(catalog)
-
-                image_model.meta["tweakreg_catalog"] = catalog.as_array()
-                nsources = len(catalog)
-                self.log.info(
-                    f"Detected {nsources} sources in {filename}."
-                    if nsources
-                    else f"No sources found in {filename}."
-                )
-                images.shelve(image_model, i)
-
     def do_relative_alignment(self, imcats):
         """
         Perform relative alignment of images.
@@ -488,70 +436,6 @@ def do_absolute_alignment(self, ref_image, imcats):
             clip_accum=True,
         )
 
-    def finalize_step(self, images, imcats):
-        """
-        Finalize the tweak registration step by updating image metadata and WCS information.
-
-        This method iterates through the provided image catalogs, marking TweakRegStep as complete,
-        removing the source catalog, and updating the WCS if the fit was successful.
-        It also serializes fit results for storage in the image model's metadata.
-
-        Parameters
-        ----------
-        images : ModelLibrary
-            A collection of image models to be updated.
-        imcats : list
-            A collection of image catalogs containing fit information.
-
-        Returns
-        -------
-        None
-        """
-        with images:
-            for i, imcat in enumerate(imcats):
-                image_model = images.borrow(i)
-                image_model.meta.cal_step["tweakreg"] = "COMPLETE"
-                # remove source catalog
-                del image_model.meta["tweakreg_catalog"]
-
-                # retrieve fit status and update wcs if fit is successful:
-                if "SUCCESS" in imcat.meta.get("fit_info")["status"]:
-                    # Update/create the WCS .name attribute with information
-                    # on this astrometric fit as the only record that it was
-                    # successful:
-
-                    # NOTE: This .name attrib agreed upon by the JWST Cal
-                    #       Working Group.
-                    #       Current value is merely a place-holder based
-                    #       on HST conventions. This value should also be
-                    #       translated to the FITS WCSNAME keyword
-                    #       IF that is what gets recorded in the archive
-                    #       for end-user searches.
-                    imcat.wcs.name = f"FIT-LVL2-{self.abs_refcat}"
-
-                    # serialize object from tweakwcs
-                    # (typecasting numpy objects to python types so that it doesn't cause an
-                    # issue when saving datamodel to ASDF)
-                    wcs_fit_results = {
-                        k: v.tolist() if isinstance(v, (np.ndarray, np.bool_)) else v
-                        for k, v in imcat.meta["fit_info"].items()
-                    }
-                    # add fit results and new WCS to datamodel
-                    image_model.meta["wcs_fit_results"] = wcs_fit_results
-                    # remove unwanted keys from WCS fit results
-                    for k in [
-                        "eff_minobj",
-                        "matched_ref_idx",
-                        "matched_input_idx",
-                        "fit_RA",
-                        "fit_DEC",
-                        "fitmask",
-                    ]:
-                        del image_model.meta["wcs_fit_results"][k]
-
-                    image_model.meta.wcs = imcat.wcs
-                images.shelve(image_model, i)
-
 
 def _parse_catfile(catfile):
     """
@@ -602,88 +486,3 @@ def _parse_catfile(catfile):
                 raise ValueError("'catfile' can contain at most two columns.")
 
     return catdict
-
-
-def _build_image_catalogs(images) -> List:
-    """
-    Build image catalogs from the provided images.
-
-    This method constructs a list of image catalogs by extracting the necessary
-    metadata from each image model. It creates a WCS corrector for each image
-    based on its associated catalog and metadata.
-
-    Parameters
-    ----------
-    images : ModelLibrary
-        A collection of image models from which to build catalogs.
-
-    Returns
-    -------
-    imcats : list
-        A list of image catalogs constructed from the input images.
-    """
-    imcats = []
-    with images:
-        for i, m in enumerate(images):
-            # catalog name
-            catalog_name = os.path.splitext(m.meta.filename)[0].strip("_- ")
-            # catalog data
-            catalog_table = Table(m.meta.tweakreg_catalog)
-            catalog_table.meta["name"] = catalog_name
-
-            imcats.append(
-                tweakreg.construct_wcs_corrector(
-                    wcs=m.meta.wcs,
-                    refang=m.meta.wcsinfo,
-                    catalog=catalog_table,
-                    group_id=m.meta.group_id,
-                )
-            )
-            images.shelve(m, i, modify=False)
-    return imcats
-
-
-def _set_reference_image(images):
-    with images:
-        ref_image = images.borrow(0)
-        images.shelve(ref_image, 0, modify=False)
-    return ref_image
-
-
-def _validate_catalog_columns(catalog, axis, image_model, index):
-    """
-    Validate the presence of required columns in the catalog.
-
-    This method checks if the specified axis column exists in the catalog.
-    If the axis is not found, it looks for a corresponding centroid column
-    and renames it if present. If neither is found, it raises an error.
-
-    Parameters
-    ----------
-    catalog : Table
-        The catalog to validate, which should contain source information.
-    axis : str
-        The axis to check for in the catalog (e.g., 'x' or 'y').
-    image_model : DataModel
-        The image model associated with the catalog.
-    index : int
-        The index of the image model in the collection.
-
-    Returns
-    -------
-    None
-
-    Raises
-    ------
-    ValueError
-        If the required columns are missing from the catalog.
-    """
-    if axis not in catalog.colnames:
-        long_axis = f"{axis}centroid"
-        if long_axis in catalog.colnames:
-            catalog.rename_column(long_axis, axis)
-        else:
-            raise ValueError(
-                "'tweakreg' source catalogs must contain a header with "
-                "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
-            )

From 2313559328f59c95b3b263f15d72d7fc0b67f928 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Mon, 16 Sep 2024 11:03:38 -0400
Subject: [PATCH 13/74] Refactor handling of invalid exposure types.

---
 romancal/tweakreg/tests/test_tweakreg.py |  22 +-
 romancal/tweakreg/tweakreg_step.py       | 246 ++++++++++++-----------
 2 files changed, 146 insertions(+), 122 deletions(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index 4c3dcbe92..75adbb83b 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -11,7 +11,7 @@
 from astropy import coordinates as coord
 from astropy import table
 from astropy import units as u
-from astropy.modeling import models
+from astropy.modeling import models, Model
 from astropy.modeling.models import RotationSequence3D, Scale, Shift
 from astropy.time import Time
 from gwcs import coordinate_frames as cf
@@ -984,3 +984,23 @@ def test_parse_catfile_raises_error_on_invalid_content(tmp_path, catfile_line_co
         trs._parse_catfile(catfile)
 
     assert type(exec_info.value) == ValueError
+
+
+@pytest.mark.parametrize(
+    "exposure_type",
+    ["WFI_GRISM", "WFI_PRISM", "WFI_DARK", "WFI_FLAT", "WFI_WFSC"],
+)
+def test_tweakreg_skips_invalid_exposure_types(exposure_type, tmp_path, base_image):
+    """Test that TweakReg updates meta.cal_step with tweakreg = COMPLETE."""
+    img1 = base_image(shift_1=1000, shift_2=1000)
+    img1.meta.exposure.type = exposure_type
+    img2 = base_image(shift_1=1000, shift_2=1000)
+    img2.meta.exposure.type = exposure_type
+    res = trs.TweakRegStep.call([img1, img2])
+
+    assert type(res) == ModelLibrary
+    with res:
+        for i, model in enumerate(res):
+            assert hasattr(model.meta.cal_step, "tweakreg")
+            assert model.meta.cal_step.tweakreg == "SKIPPED"
+            res.shelve(model, i, modify=False)
diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index 6c6b207ba..3b7818dfa 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -155,142 +155,146 @@ def process(self, input):
             self.expand_refcat = True
 
         # build the catalogs for input images
+        imcats = []
         with images:
             for i, image_model in enumerate(images):
                 exposure_type = image_model.meta.exposure.type
                 if exposure_type != "WFI_IMAGE":
                     self.log.info("Skipping TweakReg for spectral exposure.")
                     image_model.meta.cal_step.tweakreg = "SKIPPED"
-                    images.shelve(image_model)
-                    return image_model
-
-                source_detection = getattr(image_model.meta, "source_detection", None)
-                if source_detection is None:
-                    images.shelve(image_model, i, modify=False)
-                    raise AttributeError(
-                        "Attribute 'meta.source_detection' is missing. "
-                        "Please either run SourceDetectionStep or provide a custom source catalog."
+                else:
+                    source_detection = getattr(
+                        image_model.meta, "source_detection", None
+                    )
+                    if source_detection is None:
+                        images.shelve(image_model, i, modify=False)
+                        raise AttributeError(
+                            "Attribute 'meta.source_detection' is missing. "
+                            "Please either run SourceDetectionStep or provide a custom source catalog."
+                        )
+
+                    try:
+                        catalog = self.get_tweakreg_catalog(
+                            source_detection, image_model, i
+                        )
+                    except AttributeError as e:
+                        self.log.error(f"Failed to retrieve tweakreg_catalog: {e}")
+                        images.shelve(image_model, i, modify=False)
+                        raise AttributeError() from e
+
+                    try:
+                        for axis in ["x", "y"]:
+                            # validate catalog columns
+                            if axis not in catalog.colnames:
+                                long_axis = f"{axis}centroid"
+                                if long_axis in catalog.colnames:
+                                    catalog.rename_column(long_axis, axis)
+                                else:
+                                    raise ValueError(
+                                        "'tweakreg' source catalogs must contain a header with "
+                                        "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
+                                    )
+                    except ValueError as e:
+                        self.log.error(f"Failed to validate catalog columns: {e}")
+                        images.shelve(image_model, i, modify=False)
+                        raise ValueError() from e
+
+                    filename = image_model.meta.filename
+                    catalog = tweakreg.filter_catalog_by_bounding_box(
+                        catalog, image_model.meta.wcs.bounding_box
                     )
 
-                try:
-                    catalog = self.get_tweakreg_catalog(
-                        source_detection, image_model, i
+                    if self.save_abs_catalog:
+                        output_name = os.path.join(
+                            self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
+                        )
+                        catalog.write(
+                            output_name, format=self.catalog_format, overwrite=True
+                        )
+
+                    image_model.meta["tweakreg_catalog"] = catalog.as_array()
+                    nsources = len(catalog)
+                    self.log.info(
+                        f"Detected {nsources} sources in {filename}."
+                        if nsources
+                        else f"No sources found in {filename}."
                     )
-                except AttributeError as e:
-                    self.log.error(f"Failed to retrieve tweakreg_catalog: {e}")
-                    images.shelve(image_model, i, modify=False)
-                    raise AttributeError() from e
-
-                try:
-                    for axis in ["x", "y"]:
-                        # validate catalog columns
-                        if axis not in catalog.colnames:
-                            long_axis = f"{axis}centroid"
-                            if long_axis in catalog.colnames:
-                                catalog.rename_column(long_axis, axis)
-                            else:
-                                raise ValueError(
-                                    "'tweakreg' source catalogs must contain a header with "
-                                    "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
-                                )
-                except ValueError as e:
-                    self.log.error(f"Failed to validate catalog columns: {e}")
-                    images.shelve(image_model, i, modify=False)
-                    raise ValueError() from e
-
-                filename = image_model.meta.filename
-                catalog = tweakreg.filter_catalog_by_bounding_box(
-                    catalog, image_model.meta.wcs.bounding_box
-                )
-
-                if self.save_abs_catalog:
-                    output_name = os.path.join(
-                        self.catalog_path, f"fit_{self.abs_refcat.lower()}_ref.ecsv"
+                    # build image catalog
+                    # catalog name
+                    catalog_name = os.path.splitext(image_model.meta.filename)[0].strip(
+                        "_- "
                     )
-                    catalog.write(
-                        output_name, format=self.catalog_format, overwrite=True
+                    # catalog data
+                    catalog_table = Table(image_model.meta.tweakreg_catalog)
+                    catalog_table.meta["name"] = catalog_name
+
+                    imcats.append(
+                        tweakreg.construct_wcs_corrector(
+                            wcs=image_model.meta.wcs,
+                            refang=image_model.meta.wcsinfo,
+                            catalog=catalog_table,
+                            group_id=image_model.meta.group_id,
+                        )
                     )
-
-                image_model.meta["tweakreg_catalog"] = catalog.as_array()
-                nsources = len(catalog)
-                self.log.info(
-                    f"Detected {nsources} sources in {filename}."
-                    if nsources
-                    else f"No sources found in {filename}."
-                )
                 images.shelve(image_model, i)
 
-        # build image catalogs
-        imcats = []
-        with images:
-            for i, m in enumerate(images):
-                # catalog name
-                catalog_name = os.path.splitext(m.meta.filename)[0].strip("_- ")
-                # catalog data
-                catalog_table = Table(m.meta.tweakreg_catalog)
-                catalog_table.meta["name"] = catalog_name
-
-                imcats.append(
-                    tweakreg.construct_wcs_corrector(
-                        wcs=m.meta.wcs,
-                        refang=m.meta.wcsinfo,
-                        catalog=catalog_table,
-                        group_id=m.meta.group_id,
-                    )
-                )
-                images.shelve(m, i, modify=False)
+        # run alignment only if it was possible to build image catalogs
+        if len(imcats):
+            if getattr(images, "group_indices", None) and len(images.group_indices) > 1:
+                self.do_relative_alignment(imcats)
 
-        if getattr(images, "group_indices", None) and len(images.group_indices) > 1:
-            self.do_relative_alignment(imcats)
+            if self.abs_refcat in SINGLE_GROUP_REFCAT:
+                self.do_absolute_alignment(ref_image, imcats)
 
-        if self.abs_refcat in SINGLE_GROUP_REFCAT:
-            self.do_absolute_alignment(ref_image, imcats)
-
-        # finalize step
-        with images:
-            for i, imcat in enumerate(imcats):
-                image_model = images.borrow(i)
-                image_model.meta.cal_step["tweakreg"] = "COMPLETE"
-                # remove source catalog
-                del image_model.meta["tweakreg_catalog"]
-
-                # retrieve fit status and update wcs if fit is successful:
-                if "SUCCESS" in imcat.meta.get("fit_info")["status"]:
-                    # Update/create the WCS .name attribute with information
-                    # on this astrometric fit as the only record that it was
-                    # successful:
-
-                    # NOTE: This .name attrib agreed upon by the JWST Cal
-                    #       Working Group.
-                    #       Current value is merely a place-holder based
-                    #       on HST conventions. This value should also be
-                    #       translated to the FITS WCSNAME keyword
-                    #       IF that is what gets recorded in the archive
-                    #       for end-user searches.
-                    imcat.wcs.name = f"FIT-LVL2-{self.abs_refcat}"
-
-                    # serialize object from tweakwcs
-                    # (typecasting numpy objects to python types so that it doesn't cause an
-                    # issue when saving datamodel to ASDF)
-                    wcs_fit_results = {
-                        k: v.tolist() if isinstance(v, (np.ndarray, np.bool_)) else v
-                        for k, v in imcat.meta["fit_info"].items()
-                    }
-                    # add fit results and new WCS to datamodel
-                    image_model.meta["wcs_fit_results"] = wcs_fit_results
-                    # remove unwanted keys from WCS fit results
-                    for k in [
-                        "eff_minobj",
-                        "matched_ref_idx",
-                        "matched_input_idx",
-                        "fit_RA",
-                        "fit_DEC",
-                        "fitmask",
-                    ]:
-                        del image_model.meta["wcs_fit_results"][k]
-
-                    image_model.meta.wcs = imcat.wcs
-                images.shelve(image_model, i)
+            # finalize step
+            with images:
+                for i, imcat in enumerate(imcats):
+                    image_model = images.borrow(i)
+                    image_model.meta.cal_step["tweakreg"] = "COMPLETE"
+                    # remove source catalog
+                    del image_model.meta["tweakreg_catalog"]
+
+                    # retrieve fit status and update wcs if fit is successful:
+                    if "SUCCESS" in imcat.meta.get("fit_info")["status"]:
+                        # Update/create the WCS .name attribute with information
+                        # on this astrometric fit as the only record that it was
+                        # successful:
+
+                        # NOTE: This .name attrib agreed upon by the JWST Cal
+                        #       Working Group.
+                        #       Current value is merely a place-holder based
+                        #       on HST conventions. This value should also be
+                        #       translated to the FITS WCSNAME keyword
+                        #       IF that is what gets recorded in the archive
+                        #       for end-user searches.
+                        imcat.wcs.name = f"FIT-LVL2-{self.abs_refcat}"
+
+                        # serialize object from tweakwcs
+                        # (typecasting numpy objects to python types so that it doesn't cause an
+                        # issue when saving datamodel to ASDF)
+                        wcs_fit_results = {
+                            k: (
+                                v.tolist()
+                                if isinstance(v, (np.ndarray, np.bool_))
+                                else v
+                            )
+                            for k, v in imcat.meta["fit_info"].items()
+                        }
+                        # add fit results and new WCS to datamodel
+                        image_model.meta["wcs_fit_results"] = wcs_fit_results
+                        # remove unwanted keys from WCS fit results
+                        for k in [
+                            "eff_minobj",
+                            "matched_ref_idx",
+                            "matched_input_idx",
+                            "fit_RA",
+                            "fit_DEC",
+                            "fitmask",
+                        ]:
+                            del image_model.meta["wcs_fit_results"][k]
+
+                        image_model.meta.wcs = imcat.wcs
+                    images.shelve(image_model, i)
 
         return images
 

From f3169208eba1549bcc4c6b4de3536ccf796c023a Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 16 Sep 2024 15:05:34 +0000
Subject: [PATCH 14/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/tweakreg/tests/test_tweakreg.py | 2 +-
 romancal/tweakreg/tweakreg_step.py       | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index 75adbb83b..e01052875 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -11,7 +11,7 @@
 from astropy import coordinates as coord
 from astropy import table
 from astropy import units as u
-from astropy.modeling import models, Model
+from astropy.modeling import models
 from astropy.modeling.models import RotationSequence3D, Scale, Shift
 from astropy.time import Time
 from gwcs import coordinate_frames as cf
diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index 3b7818dfa..19218b9ae 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -4,7 +4,6 @@
 
 import os
 from pathlib import Path
-from typing import List
 
 import numpy as np
 from astropy.table import Table

From 9d481147246447eaa2a0efd2a2fc41cfb1ffa45d Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 17 Sep 2024 17:08:41 -0400
Subject: [PATCH 15/74] Address comments 2.

---
 romancal/tweakreg/tests/test_tweakreg.py |  72 +++++++++++++-
 romancal/tweakreg/tweakreg_step.py       | 115 +++++++++++++----------
 2 files changed, 135 insertions(+), 52 deletions(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index e01052875..d4305ac5b 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -20,6 +20,7 @@
 from roman_datamodels import datamodels as rdm
 from roman_datamodels import maker_utils
 from stcal.tweakreg.astrometric_utils import get_catalog
+from romancal.tweakreg.tweakreg_step import _validate_catalog_columns
 
 from romancal.datamodels import ModelLibrary
 from romancal.tweakreg import tweakreg_step as trs
@@ -638,7 +639,6 @@ def test_tweakreg_updates_group_id(tmp_path, base_image):
 )
 def test_tweakreg_save_valid_abs_refcat(tmp_path, abs_refcat, request):
     """Test that TweakReg saves the catalog used for absolute astrometry."""
-    os.chdir(tmp_path)
 
     img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000)
     catalog_filename = "ref_catalog.ecsv"
@@ -646,7 +646,11 @@ def test_tweakreg_save_valid_abs_refcat(tmp_path, abs_refcat, request):
     add_tweakreg_catalog_attribute(tmp_path, img, catalog_filename=catalog_filename)
 
     trs.TweakRegStep.call(
-        [img], save_abs_catalog=True, abs_refcat=abs_refcat, catalog_path=str(tmp_path)
+        [img],
+        save_abs_catalog=True,
+        abs_refcat=abs_refcat,
+        catalog_path=str(tmp_path),
+        output_dir=str(tmp_path),
     )
 
     assert os.path.exists(tmp_path / abs_refcat_filename)
@@ -658,7 +662,6 @@ def test_tweakreg_save_valid_abs_refcat(tmp_path, abs_refcat, request):
 )
 def test_tweakreg_defaults_to_valid_abs_refcat(tmp_path, abs_refcat, request):
     """Test that TweakReg defaults to DEFAULT_ABS_REFCAT on invalid values."""
-    os.chdir(tmp_path)
 
     img = request.getfixturevalue("base_image")(shift_1=1000, shift_2=1000)
     catalog_filename = "ref_catalog.ecsv"
@@ -666,7 +669,11 @@ def test_tweakreg_defaults_to_valid_abs_refcat(tmp_path, abs_refcat, request):
     add_tweakreg_catalog_attribute(tmp_path, img, catalog_filename=catalog_filename)
 
     trs.TweakRegStep.call(
-        [img], save_abs_catalog=True, abs_refcat=abs_refcat, catalog_path=str(tmp_path)
+        [img],
+        save_abs_catalog=True,
+        abs_refcat=abs_refcat,
+        catalog_path=str(tmp_path),
+        output_dir=str(tmp_path),
     )
 
     assert os.path.exists(tmp_path / abs_refcat_filename)
@@ -1004,3 +1011,60 @@ def test_tweakreg_skips_invalid_exposure_types(exposure_type, tmp_path, base_ima
             assert hasattr(model.meta.cal_step, "tweakreg")
             assert model.meta.cal_step.tweakreg == "SKIPPED"
             res.shelve(model, i, modify=False)
+
+
+@pytest.mark.parametrize(
+    "catalog_data, expected_colnames, raises_exception",
+    [
+        # both 'x' and 'y' columns present
+        ({"x": [1, 2, 3], "y": [4, 5, 6]}, ["x", "y"], False),
+        # 'xcentroid' and 'ycentroid' columns present, should be renamed
+        ({"xcentroid": [1, 2, 3], "ycentroid": [4, 5, 6]}, ["x", "y"], False),
+        # 'x' present, 'ycentroid' present, should rename 'ycentroid' to 'y'
+        ({"x": [1, 2, 3], "ycentroid": [4, 5, 6]}, ["x", "y"], False),
+        # 'xcentroid' present, 'y' present, should rename 'xcentroid' to 'x'
+        ({"xcentroid": [1, 2, 3], "y": [4, 5, 6]}, ["x", "y"], False),
+        # neither 'x' nor 'xcentroid' present
+        ({"y": [4, 5, 6]}, None, True),
+        # neither 'y' nor 'ycentroid' present
+        ({"x": [1, 2, 3]}, None, True),
+        # no relevant columns present
+        (
+            {"a": [1, 2, 3], "b": [4, 5, 6]},
+            None,
+            True,
+        ),
+    ],
+)
+def test_validate_catalog_columns(catalog_data, expected_colnames, raises_exception):
+    """Test that TweakRegStep._validate_catalog_columns() correctly validates the
+    presence of required columns ('x' and 'y') in the provided catalog."""
+    catalog = table.Table(catalog_data)
+    if raises_exception:
+        with pytest.raises(ValueError):
+            _validate_catalog_columns(catalog)
+    else:
+        _validate_catalog_columns(catalog)
+        assert set(catalog.colnames) == set(expected_colnames)
+
+
+def test_tweakreg_handles_mixed_exposure_types(tmp_path, base_image):
+    """Test that TweakReg can handle mixed exposure types
+    (non-WFI_IMAGE data will be marked as SKIPPED only and won't be processed)."""
+    img1 = base_image(shift_1=1000, shift_2=1000)
+    add_tweakreg_catalog_attribute(tmp_path, img1, catalog_filename="img1")
+    img1.meta.exposure.type = "WFI_IMAGE"
+
+    img2 = base_image(shift_1=1000, shift_2=1000)
+    add_tweakreg_catalog_attribute(tmp_path, img2, catalog_filename="img2")
+    img2.meta.exposure.type = "WFI_IMAGE"
+
+    img3 = base_image(shift_1=1000, shift_2=1000)
+    img3.meta.exposure.type = "WFI_GRISM"
+
+    res = trs.TweakRegStep.call([img1, img2, img3])
+
+    assert len(res) == 3
+    assert img1.meta.cal_step.tweakreg == "COMPLETE"
+    assert img2.meta.cal_step.tweakreg == "COMPLETE"
+    assert img3.meta.cal_step.tweakreg == "SKIPPED"
diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index 19218b9ae..a2880c941 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -15,21 +15,8 @@
 from ..stpipe import RomanStep
 
 
-def _oxford_or_str_join(str_list):
-    nelem = len(str_list)
-    if not nelem:
-        return "N/A"
-    str_list = list(map(repr, str_list))
-    if nelem == 1:
-        return str_list
-    elif nelem == 2:
-        return f"{str_list[0]} or {str_list[1]}"
-    else:
-        return ", ".join(map(repr, str_list[:-1])) + ", or " + repr(str_list[-1])
-
-
-SINGLE_GROUP_REFCAT = ["GAIADR3", "GAIADR2", "GAIADR1"]
-_SINGLE_GROUP_REFCAT_STR = _oxford_or_str_join(SINGLE_GROUP_REFCAT)
+SINGLE_GROUP_REFCAT = tweakreg.SINGLE_GROUP_REFCAT
+_SINGLE_GROUP_REFCAT_STR = tweakreg._SINGLE_GROUP_REFCAT_STR
 DEFAULT_ABS_REFCAT = SINGLE_GROUP_REFCAT[0]
 
 __all__ = ["TweakRegStep"]
@@ -99,11 +86,12 @@ def process(self, input):
         if not images:
             raise ValueError("Input must contain at least one image model.")
 
-        self.log.info("")
         self.log.info(
             f"Number of image groups to be aligned: {len(images.group_indices):d}."
         )
         self.log.info("Image groups:")
+        for name in images.group_names:
+            self.log.info(f"  {name}")
         # set the first image as reference
         with images:
             ref_image = images.borrow(0)
@@ -174,31 +162,21 @@ def process(self, input):
 
                     try:
                         catalog = self.get_tweakreg_catalog(
-                            source_detection, image_model, i
+                            source_detection, image_model
                         )
                     except AttributeError as e:
                         self.log.error(f"Failed to retrieve tweakreg_catalog: {e}")
                         images.shelve(image_model, i, modify=False)
-                        raise AttributeError() from e
+                        raise e
 
                     try:
-                        for axis in ["x", "y"]:
-                            # validate catalog columns
-                            if axis not in catalog.colnames:
-                                long_axis = f"{axis}centroid"
-                                if long_axis in catalog.colnames:
-                                    catalog.rename_column(long_axis, axis)
-                                else:
-                                    raise ValueError(
-                                        "'tweakreg' source catalogs must contain a header with "
-                                        "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
-                                    )
+                        # validate catalog columns
+                        _validate_catalog_columns(catalog)
                     except ValueError as e:
                         self.log.error(f"Failed to validate catalog columns: {e}")
                         images.shelve(image_model, i, modify=False)
-                        raise ValueError() from e
+                        raise e
 
-                    filename = image_model.meta.filename
                     catalog = tweakreg.filter_catalog_by_bounding_box(
                         catalog, image_model.meta.wcs.bounding_box
                     )
@@ -214,9 +192,9 @@ def process(self, input):
                     image_model.meta["tweakreg_catalog"] = catalog.as_array()
                     nsources = len(catalog)
                     self.log.info(
-                        f"Detected {nsources} sources in {filename}."
+                        f"Detected {nsources} sources in {image_model.meta.filename}."
                         if nsources
-                        else f"No sources found in {filename}."
+                        else f"No sources found in {image_model.meta.filename}."
                     )
                     # build image catalog
                     # catalog name
@@ -228,27 +206,33 @@ def process(self, input):
                     catalog_table.meta["name"] = catalog_name
 
                     imcats.append(
-                        tweakreg.construct_wcs_corrector(
-                            wcs=image_model.meta.wcs,
-                            refang=image_model.meta.wcsinfo,
-                            catalog=catalog_table,
-                            group_id=image_model.meta.group_id,
-                        )
+                        {
+                            "model_index": i,
+                            "imcat": tweakreg.construct_wcs_corrector(
+                                wcs=image_model.meta.wcs,
+                                refang=image_model.meta.wcsinfo,
+                                catalog=catalog_table,
+                                group_id=image_model.meta.group_id,
+                            ),
+                        }
                     )
                 images.shelve(image_model, i)
 
         # run alignment only if it was possible to build image catalogs
         if len(imcats):
-            if getattr(images, "group_indices", None) and len(images.group_indices) > 1:
-                self.do_relative_alignment(imcats)
+            # extract WCS correctors to use for image alignment
+            correctors = [x["imcat"] for x in imcats]
+            if len(images.group_indices) > 1:
+                self.do_relative_alignment(correctors)
 
             if self.abs_refcat in SINGLE_GROUP_REFCAT:
-                self.do_absolute_alignment(ref_image, imcats)
+                self.do_absolute_alignment(ref_image, correctors)
 
             # finalize step
             with images:
-                for i, imcat in enumerate(imcats):
-                    image_model = images.borrow(i)
+                for item in imcats:
+                    imcat = item["imcat"]
+                    image_model = images.borrow(item["model_index"])
                     image_model.meta.cal_step["tweakreg"] = "COMPLETE"
                     # remove source catalog
                     del image_model.meta["tweakreg_catalog"]
@@ -293,7 +277,7 @@ def process(self, input):
                             del image_model.meta["wcs_fit_results"][k]
 
                         image_model.meta.wcs = imcat.wcs
-                    images.shelve(image_model, i)
+                    images.shelve(image_model, item["model_index"])
 
         return images
 
@@ -327,7 +311,7 @@ def read_catalog(self, catalog_name):
             catalog = Table.read(catalog_name, format=self.catalog_format)
         return catalog
 
-    def get_tweakreg_catalog(self, source_detection, image_model, index):
+    def get_tweakreg_catalog(self, source_detection, image_model):
         """
         Retrieve the tweakreg catalog from source detection.
 
@@ -341,8 +325,6 @@ def get_tweakreg_catalog(self, source_detection, image_model, index):
             The source detection metadata containing catalog information.
         image_model : DataModel
             The image model associated with the source detection.
-        index : int
-            The index of the image model in the collection.
 
         Returns
         -------
@@ -489,3 +471,40 @@ def _parse_catfile(catfile):
                 raise ValueError("'catfile' can contain at most two columns.")
 
     return catdict
+
+
+def _validate_catalog_columns(catalog):
+    """
+    Validate the presence of required columns in the catalog.
+
+    This method checks if the specified axis column exists in the catalog.
+    If the axis is not found, it looks for a corresponding centroid column
+    and renames it if present. If neither is found, it raises an error.
+
+    Parameters
+    ----------
+    catalog : Table
+        The catalog to validate, which should contain source information.
+    axis : str
+        The axis to check for in the catalog (e.g., 'x' or 'y').
+
+    Returns
+    -------
+    None
+
+    Raises
+    ------
+    ValueError
+        If the required columns are missing from the catalog.
+    """
+    for axis in ["x", "y"]:
+        if axis not in catalog.colnames:
+            long_axis = f"{axis}centroid"
+            if long_axis in catalog.colnames:
+                catalog.rename_column(long_axis, axis)
+            else:
+                raise ValueError(
+                    "'tweakreg' source catalogs must contain a header with "
+                    "columns named either 'x' and 'y' or 'xcentroid' and 'ycentroid'."
+                )
+    return catalog

From a97482ab226883f36cb14ec605e3ffe6a5c96e1f Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 17 Sep 2024 21:09:16 +0000
Subject: [PATCH 16/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/tweakreg/tests/test_tweakreg.py | 2 +-
 romancal/tweakreg/tweakreg_step.py       | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index d4305ac5b..025056d28 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -20,10 +20,10 @@
 from roman_datamodels import datamodels as rdm
 from roman_datamodels import maker_utils
 from stcal.tweakreg.astrometric_utils import get_catalog
-from romancal.tweakreg.tweakreg_step import _validate_catalog_columns
 
 from romancal.datamodels import ModelLibrary
 from romancal.tweakreg import tweakreg_step as trs
+from romancal.tweakreg.tweakreg_step import _validate_catalog_columns
 
 
 class MockConnectionError:
diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index a2880c941..c80abd9be 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -14,7 +14,6 @@
 from ..datamodels import ModelLibrary
 from ..stpipe import RomanStep
 
-
 SINGLE_GROUP_REFCAT = tweakreg.SINGLE_GROUP_REFCAT
 _SINGLE_GROUP_REFCAT_STR = tweakreg._SINGLE_GROUP_REFCAT_STR
 DEFAULT_ABS_REFCAT = SINGLE_GROUP_REFCAT[0]

From c3654374ba49fdfeccddf2e106947ce6e4e0fbe9 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Wed, 18 Sep 2024 10:34:20 -0400
Subject: [PATCH 17/74] Style refactoring.

---
 romancal/tweakreg/tests/test_tweakreg.py | 26 ++++++++++++++-----
 romancal/tweakreg/tweakreg_step.py       | 32 ++++++++++--------------
 2 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index 025056d28..ac21e447f 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -685,7 +685,13 @@ def test_tweakreg_raises_error_on_invalid_abs_refcat(tmp_path, base_image):
     add_tweakreg_catalog_attribute(tmp_path, img)
 
     with pytest.raises(Exception) as exec_info:
-        trs.TweakRegStep.call([img], save_abs_catalog=True, abs_refcat="my_ref_cat")
+        trs.TweakRegStep.call(
+            [img],
+            save_abs_catalog=True,
+            abs_refcat="my_ref_cat",
+            catalog_path=str(tmp_path),
+            output_dir=str(tmp_path),
+        )
 
     assert type(exec_info.value) == TypeError
 
@@ -1052,8 +1058,7 @@ def test_tweakreg_handles_mixed_exposure_types(tmp_path, base_image):
     """Test that TweakReg can handle mixed exposure types
     (non-WFI_IMAGE data will be marked as SKIPPED only and won't be processed)."""
     img1 = base_image(shift_1=1000, shift_2=1000)
-    add_tweakreg_catalog_attribute(tmp_path, img1, catalog_filename="img1")
-    img1.meta.exposure.type = "WFI_IMAGE"
+    img1.meta.exposure.type = "WFI_GRISM"
 
     img2 = base_image(shift_1=1000, shift_2=1000)
     add_tweakreg_catalog_attribute(tmp_path, img2, catalog_filename="img2")
@@ -1062,9 +1067,18 @@ def test_tweakreg_handles_mixed_exposure_types(tmp_path, base_image):
     img3 = base_image(shift_1=1000, shift_2=1000)
     img3.meta.exposure.type = "WFI_GRISM"
 
-    res = trs.TweakRegStep.call([img1, img2, img3])
+    img4 = base_image(shift_1=1000, shift_2=1000)
+    add_tweakreg_catalog_attribute(tmp_path, img4, catalog_filename="img4")
+    img4.meta.exposure.type = "WFI_IMAGE"
+
+    img5 = base_image(shift_1=1000, shift_2=1000)
+    img5.meta.exposure.type = "WFI_GRISM"
+
+    res = trs.TweakRegStep.call([img1, img2, img3, img4, img5])
 
-    assert len(res) == 3
-    assert img1.meta.cal_step.tweakreg == "COMPLETE"
+    assert len(res) == 5
+    assert img1.meta.cal_step.tweakreg == "SKIPPED"
     assert img2.meta.cal_step.tweakreg == "COMPLETE"
     assert img3.meta.cal_step.tweakreg == "SKIPPED"
+    assert img4.meta.cal_step.tweakreg == "COMPLETE"
+    assert img5.meta.cal_step.tweakreg == "SKIPPED"
diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index c80abd9be..d85267ecd 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -9,13 +9,12 @@
 from astropy.table import Table
 from roman_datamodels import datamodels as rdm
 from stcal.tweakreg import tweakreg
+from stcal.tweakreg.tweakreg import _SINGLE_GROUP_REFCAT_STR, SINGLE_GROUP_REFCAT
 
 # LOCAL
 from ..datamodels import ModelLibrary
 from ..stpipe import RomanStep
 
-SINGLE_GROUP_REFCAT = tweakreg.SINGLE_GROUP_REFCAT
-_SINGLE_GROUP_REFCAT_STR = tweakreg._SINGLE_GROUP_REFCAT_STR
 DEFAULT_ABS_REFCAT = SINGLE_GROUP_REFCAT[0]
 
 __all__ = ["TweakRegStep"]
@@ -204,34 +203,29 @@ def process(self, input):
                     catalog_table = Table(image_model.meta.tweakreg_catalog)
                     catalog_table.meta["name"] = catalog_name
 
-                    imcats.append(
-                        {
-                            "model_index": i,
-                            "imcat": tweakreg.construct_wcs_corrector(
-                                wcs=image_model.meta.wcs,
-                                refang=image_model.meta.wcsinfo,
-                                catalog=catalog_table,
-                                group_id=image_model.meta.group_id,
-                            ),
-                        }
+                    imcat = tweakreg.construct_wcs_corrector(
+                        wcs=image_model.meta.wcs,
+                        refang=image_model.meta.wcsinfo,
+                        catalog=catalog_table,
+                        group_id=image_model.meta.group_id,
                     )
+                    imcat.meta["model_index"] = i
+                    imcats.append(imcat)
                 images.shelve(image_model, i)
 
         # run alignment only if it was possible to build image catalogs
         if len(imcats):
             # extract WCS correctors to use for image alignment
-            correctors = [x["imcat"] for x in imcats]
             if len(images.group_indices) > 1:
-                self.do_relative_alignment(correctors)
+                self.do_relative_alignment(imcats)
 
             if self.abs_refcat in SINGLE_GROUP_REFCAT:
-                self.do_absolute_alignment(ref_image, correctors)
+                self.do_absolute_alignment(ref_image, imcats)
 
             # finalize step
             with images:
-                for item in imcats:
-                    imcat = item["imcat"]
-                    image_model = images.borrow(item["model_index"])
+                for imcat in imcats:
+                    image_model = images.borrow(imcat.meta["model_index"])
                     image_model.meta.cal_step["tweakreg"] = "COMPLETE"
                     # remove source catalog
                     del image_model.meta["tweakreg_catalog"]
@@ -276,7 +270,7 @@ def process(self, input):
                             del image_model.meta["wcs_fit_results"][k]
 
                         image_model.meta.wcs = imcat.wcs
-                    images.shelve(image_model, item["model_index"])
+                    images.shelve(image_model, imcat.meta["model_index"])
 
         return images
 

From 487645713047e3f709799023993c1b12e7b2353a Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 23 Sep 2024 10:26:20 -0400
Subject: [PATCH 18/74] remove default values from docs

---
 docs/roman/datamodels/models.rst | 35 ++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/docs/roman/datamodels/models.rst b/docs/roman/datamodels/models.rst
index 95f424aee..cd00411d9 100644
--- a/docs/roman/datamodels/models.rst
+++ b/docs/roman/datamodels/models.rst
@@ -33,13 +33,23 @@ Reading a data model
 If you have an existing data file it is straightforward to access the file
 using python.
 
+.. testsetup::
+    >>> from roman_datamodels import datamodels as rdm
+    >>> from roman_datamodels.maker_utils import mk_datamodel
+    >>> fn = 'r0019106003005004023_03203_0034_WFI01_cal.asdf'
+    >>> image_model = mk_datamodel(rdm.ImageModel)
+
+    # set some metadata for the below tests
+    >>> image_model.meta.filename = fn
+
+
 .. code-block:: python
 
     >>> from roman_datamodels import datamodels as rdm
     >>> fn = 'r0019106003005004023_03203_0034_WFI01_cal.asdf'
-    >>> data_file = rdm.open(fn)  # doctest: +SKIP
-    >>> type(data_file) # doctest: +SKIP
-    roman_datamodels.datamodels.ImageModel
+    >>> image_model = rdm.open(fn)  # doctest: +SKIP
+    >>> type(image_model)
+    <class 'roman_datamodels.datamodels._datamodels.ImageModel'>
 
 Where the output of the type command tells you that you have imported an
 ImageModel from roman_datamodels,
@@ -54,8 +64,8 @@ To create a new `ImageModel`, you can just
     >>> from roman_datamodels import datamodels as rdm
     >>> from roman_datamodels.maker_utils import mk_datamodel
 
-    >>> image_model = mk_datamodel(rdm.ImageModel)
-    >>> type(image_model)
+    >>> new_model = mk_datamodel(rdm.ImageModel)
+    >>> type(new_model)
     <class 'roman_datamodels.datamodels._datamodels.ImageModel'>
 
 .. warning::
@@ -134,7 +144,7 @@ You can examine the contents of your model from within python using
     >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.items()), "\n") # doctest: +ELLIPSIS
     meta.calibration_software_version   9.9.0
     meta.sdf_software_version   7.7.7
-           meta.filename        dummy value
+           meta.filename        r0019106003005004023_03203_0034_WFI01_cal.asdf
           meta.file_date        2020-01-01T00:00:00.000
          meta.model_type        ImageModel
              meta.origin        STSCI
@@ -148,15 +158,10 @@ or you can print specifics
 
 .. code-block:: python
 
-    >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.meta.wcsinfo.items()), "\n")
-                  v2_ref        -999999
-                  v3_ref        -999999
-                 vparity        -999999
-                v3yangle        -999999
-                  ra_ref        -999999
-                 dec_ref        -999999
-                roll_ref        -999999
-                s_region        dummy value
+    >>> print("\n".join("{: >20}\t{}".format(k, v) for k, v in image_model.meta.instrument.items()))
+                    name        WFI
+                detector        WFI01
+         optical_element        F158
 
 .. note::
 

From 1b6729abef968e11b2687b8129d44866e0bf8404 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 23 Sep 2024 10:37:21 -0400
Subject: [PATCH 19/74] add changelog

---
 changes/1419.docs.rst | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/1419.docs.rst

diff --git a/changes/1419.docs.rst b/changes/1419.docs.rst
new file mode 100644
index 000000000..7e7a16cf6
--- /dev/null
+++ b/changes/1419.docs.rst
@@ -0,0 +1 @@
+Update docs to not include default fake values.

From be1390366f338a28d3d0e80699fe46377038374d Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 23 Sep 2024 10:56:01 -0400
Subject: [PATCH 20/74] fix warning

---
 docs/roman/datamodels/models.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/roman/datamodels/models.rst b/docs/roman/datamodels/models.rst
index cd00411d9..9c889031b 100644
--- a/docs/roman/datamodels/models.rst
+++ b/docs/roman/datamodels/models.rst
@@ -70,7 +70,7 @@ To create a new `ImageModel`, you can just
 
 .. warning::
 
-    The values in the file generated by create_wfi_image are intended to be
+    The values in the file generated by mk_datamodel are intended to be
     clearly incorrect and should be replaced if the file is intended to be used
     for anything besides a demonstration.
 

From 89e7148c0bfd4caaabf396d855ae15140f3f14e2 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 24 Sep 2024 12:07:54 -0400
Subject: [PATCH 21/74] [CI] rename changelog check job to be more explicit on
 its purpose (#1422)

---
 .github/workflows/changelog.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
index c07950898..ba3140df6 100644
--- a/.github/workflows/changelog.yml
+++ b/.github/workflows/changelog.yml
@@ -28,11 +28,12 @@ jobs:
       - run: pip install towncrier
       - run: towncrier check
       - run: towncrier build --draft | grep -P '#${{ github.event.number }}'
-  changelog_uses_towncrier:
+  prevent_manually_editing_changlog:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'allow-manual-changelog-edit') }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - run: git diff HEAD ${{ github.event.pull_request.base.sha }} --no-patch --exit-code CHANGES.rst
+      - name: prevent direct changes to `CHANGES.rst` (write a towncrier fragment in `changes/` instead; you can override this with the `allow-manual-changelog-edit` label)
+        run: git diff HEAD ${{ github.event.pull_request.base.sha }} --no-patch --exit-code CHANGES.rst

From 7ea6e0aec29878b3de67ea1ac3b620dfcc7bcea8 Mon Sep 17 00:00:00 2001
From: Zach Burnett <zburnett@stsci.edu>
Date: Tue, 24 Sep 2024 12:09:21 -0400
Subject: [PATCH 22/74] update pull request checklist (#1336)

---
 .github/pull_request_template.md | 73 +++++++++++++++++---------------
 1 file changed, 40 insertions(+), 33 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index eff267329..cd4983131 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -8,38 +8,45 @@ Closes #
 <!-- describe the changes comprising this PR here -->
 This PR addresses ...
 
-**Checklist**
-- [ ] for a public change, added a towncrier news fragment in `changes/` <details><summary>`echo "changed something" > changes/<PR#>.<changetype>.rst`</summary>
+<!-- if you can't perform these due to permissions, please ask a maintainer to do them -->
+## Tasks
+- [ ] **request a review from someone specific**, to avoid making the maintainers review every PR
+- [ ] add a build milestone, i.e. `24Q4_B15` (use the [latest build](https://github.com/spacetelescope/romancal/milestones) if not sure)
+- [ ] Does this PR change user-facing code / API? (if not, label with `no-changelog-entry-needed`)
+  - [ ] write news fragment(s) in `changes/`: `echo "changed something" > changes/<PR#>.<changetype>.rst` (see below for change types)
+  - [ ] update or add relevant tests
+  - [ ] update relevant docstrings and / or `docs/` page
+  - [ ] [start a regression test](https://github.com/spacetelescope/RegressionTests/actions/workflows/romancal.yml) and include a link to the running job ([click here for instructions](https://github.com/spacetelescope/RegressionTests/blob/main/docs/running_regression_tests.md))
+    - [ ] Do truth files need to be updated ("okified")?
+      - [ ] **after the reviewer has approved these changes**, run `okify_regtests` to update the truth files
+- [ ] if a JIRA ticket exists, [make sure it is resolved properly](https://github.com/spacetelescope/romancal/wiki/How-to-resolve-JIRA-issues)
 
-    - ``changes/<PR#>.general.rst``: infrastructure or miscellaneous change
-    - ``changes/<PR#>.docs.rst``
-    - ``changes/<PR#>.stpipe.rst``
-    - ``changes/<PR#>.associations.rst``
-    - ``changes/<PR#>.scripts.rst``
-    - ``changes/<PR#>.mosaic_pipeline.rst``
-    - ``changes/<PR#>.patch_match.rst``
+<details><summary>news fragment change types...</summary>
 
-    ## steps
-    - ``changes/<PR#>.dq_init.rst``
-    - ``changes/<PR#>.saturation.rst``
-    - ``changes/<PR#>.refpix.rst``
-    - ``changes/<PR#>.linearity.rst``
-    - ``changes/<PR#>.dark_current.rst``
-    - ``changes/<PR#>.jump_detection.rst``
-    - ``changes/<PR#>.ramp_fitting.rst``
-    - ``changes/<PR#>.assign_wcs.rst``
-    - ``changes/<PR#>.flatfield.rst``
-    - ``changes/<PR#>.photom.rst``
-    - ``changes/<PR#>.flux.rst``
-    - ``changes/<PR#>.source_detection.rst``
-    - ``changes/<PR#>.tweakreg.rst``
-    - ``changes/<PR#>.skymatch.rst``
-    - ``changes/<PR#>.outlier_detection.rst``
-    - ``changes/<PR#>.resample.rst``
-    - ``changes/<PR#>.source_catalog.rst``
-  </details>
-- [ ] updated relevant tests
-- [ ] updated relevant documentation
-- [ ] updated relevant milestone(s)
-- [ ] added relevant label(s)
-- [ ] ran regression tests, post a link to the Jenkins job below. [How to run regression tests on a PR](https://github.com/spacetelescope/romancal/wiki/Running-Regression-Tests-Against-PR-Branches)
+  - ``changes/<PR#>.general.rst``: infrastructure or miscellaneous change
+  - ``changes/<PR#>.docs.rst``
+  - ``changes/<PR#>.stpipe.rst``
+  - ``changes/<PR#>.associations.rst``
+  - ``changes/<PR#>.scripts.rst``
+  - ``changes/<PR#>.mosaic_pipeline.rst``
+  - ``changes/<PR#>.patch_match.rst``
+
+  ## steps
+  - ``changes/<PR#>.dq_init.rst``
+  - ``changes/<PR#>.saturation.rst``
+  - ``changes/<PR#>.refpix.rst``
+  - ``changes/<PR#>.linearity.rst``
+  - ``changes/<PR#>.dark_current.rst``
+  - ``changes/<PR#>.jump_detection.rst``
+  - ``changes/<PR#>.ramp_fitting.rst``
+  - ``changes/<PR#>.assign_wcs.rst``
+  - ``changes/<PR#>.flatfield.rst``
+  - ``changes/<PR#>.photom.rst``
+  - ``changes/<PR#>.flux.rst``
+  - ``changes/<PR#>.source_detection.rst``
+  - ``changes/<PR#>.tweakreg.rst``
+  - ``changes/<PR#>.skymatch.rst``
+  - ``changes/<PR#>.outlier_detection.rst``
+  - ``changes/<PR#>.resample.rst``
+  - ``changes/<PR#>.source_catalog.rst``
+</details>

From 2f96fba36dc1c20cf8348605a8a24a50b3271810 Mon Sep 17 00:00:00 2001
From: mairan <mteodoro@stsci.edu>
Date: Thu, 26 Sep 2024 10:30:31 -0400
Subject: [PATCH 23/74] RCAL-895: allow updating source catalog with tweaked
 WCS when running ELP (#1373)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Zach Burnett <zachary.r.burnett@gmail.com>
Co-authored-by: Brett <brettgraham@gmail.com>
---
 changes/1373.general.rst                 |   1 +
 romancal/pipeline/exposure_pipeline.py   |   2 +
 romancal/tweakreg/tests/test_tweakreg.py | 231 ++++++++++++++++++++++-
 romancal/tweakreg/tweakreg_step.py       |  45 +++++
 4 files changed, 275 insertions(+), 4 deletions(-)
 create mode 100644 changes/1373.general.rst

diff --git a/changes/1373.general.rst b/changes/1373.general.rst
new file mode 100644
index 000000000..359959479
--- /dev/null
+++ b/changes/1373.general.rst
@@ -0,0 +1 @@
+Update source catalog file with the tweaked coordinates.
diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py
index 7e61969d0..6fb029620 100644
--- a/romancal/pipeline/exposure_pipeline.py
+++ b/romancal/pipeline/exposure_pipeline.py
@@ -71,6 +71,8 @@ def process(self, input):
 
         # make sure source_catalog returns the updated datamodel
         self.source_catalog.return_updated_model = True
+        # make sure we update source catalog coordinates afer running TweakRegStep
+        self.tweakreg.update_source_catalog_coordinates = True
 
         log.info("Starting Roman exposure calibration pipeline ...")
         if isinstance(input, str):
diff --git a/romancal/tweakreg/tests/test_tweakreg.py b/romancal/tweakreg/tests/test_tweakreg.py
index ac21e447f..9929e3393 100644
--- a/romancal/tweakreg/tests/test_tweakreg.py
+++ b/romancal/tweakreg/tests/test_tweakreg.py
@@ -1,6 +1,7 @@
 import copy
 import json
 import os
+import shutil
 from io import StringIO
 from pathlib import Path
 from typing import Tuple
@@ -9,14 +10,15 @@
 import pytest
 import requests
 from astropy import coordinates as coord
-from astropy import table
 from astropy import units as u
 from astropy.modeling import models
 from astropy.modeling.models import RotationSequence3D, Scale, Shift
+from astropy.table import Table
 from astropy.time import Time
 from gwcs import coordinate_frames as cf
 from gwcs import wcs
 from gwcs.geometry import CartesianToSpherical, SphericalToCartesian
+from numpy.random import default_rng
 from roman_datamodels import datamodels as rdm
 from roman_datamodels import maker_utils
 from stcal.tweakreg.astrometric_utils import get_catalog
@@ -78,7 +80,7 @@ def create_custom_catalogs(tmp_path, base_image, catalog_format="ascii.ecsv"):
         # write line to catfile
         catfile_content.write(f"{x.get('cat_datamodel')} {x.get('cat_filename')}\n")
         # write out the catalog data
-        t = table.Table(x.get("cat_data"), names=("x", "y"))
+        t = Table(x.get("cat_data"), names=("x", "y"))
         t.write(tmp_path / x.get("cat_filename"), format=catalog_format)
     with open(catfile, mode="w") as f:
         print(catfile_content.getvalue(), file=f)
@@ -377,7 +379,7 @@ def create_base_image_source_catalog(
     """
     src_detector_coords = catalog_data
     output = os.path.join(tmp_path, output_filename)
-    t = table.Table(src_detector_coords, names=("x", "y"))
+    t = Table(src_detector_coords, names=("x", "y"))
     if save_catalogs:
         t.write((tmp_path / output), format=catalog_format)
     # mimic the same output format from SourceDetectionStep
@@ -999,6 +1001,227 @@ def test_parse_catfile_raises_error_on_invalid_content(tmp_path, catfile_line_co
     assert type(exec_info.value) == ValueError
 
 
+def test_update_source_catalog_coordinates(tmp_path, base_image):
+    """Test that TweakReg updates the catalog coordinates with the tweaked WCS."""
+
+    os.chdir(tmp_path)
+
+    img = base_image(shift_1=1000, shift_2=1000)
+    add_tweakreg_catalog_attribute(tmp_path, img, catalog_filename="img_1")
+
+    tweakreg = trs.TweakRegStep()
+
+    # create SourceCatalogModel
+    source_catalog_model = setup_source_catalog_model(img)
+
+    # save SourceCatalogModel
+    tweakreg.save_model(
+        source_catalog_model,
+        output_file="img_1.asdf",
+        suffix="cat",
+        force=True,
+    )
+
+    # update tweakreg catalog name
+    img.meta.source_detection.tweakreg_catalog_name = "img_1_cat.asdf"
+
+    # run TweakRegStep
+    res = trs.TweakRegStep.call([img])
+
+    # tweak the current WCS using TweakRegStep and save the updated cat file
+    with res:
+        dm = res.borrow(0)
+        assert dm.meta.source_detection.tweakreg_catalog_name == "img_1_cat.asdf"
+        tweakreg.update_catalog_coordinates(
+            dm.meta.source_detection.tweakreg_catalog_name, dm.meta.wcs
+        )
+        res.shelve(dm, 0)
+
+    # read in saved catalog coords
+    cat = rdm.open("img_1_cat.asdf")
+    cat_ra_centroid = cat.source_catalog["ra_centroid"]
+    cat_dec_centroid = cat.source_catalog["dec_centroid"]
+    cat_ra_psf = cat.source_catalog["ra_psf"]
+    cat_dec_psf = cat.source_catalog["dec_psf"]
+
+    # calculate world coords using tweaked WCS
+    expected_centroid = img.meta.wcs(
+        cat.source_catalog["xcentroid"], cat.source_catalog["ycentroid"]
+    )
+    expected_psf = img.meta.wcs(
+        cat.source_catalog["x_psf"], cat.source_catalog["y_psf"]
+    )
+
+    # compare coordinates (make sure tweaked WCS was applied to cat file coords)
+    np.testing.assert_array_equal(cat_ra_centroid, expected_centroid[0])
+    np.testing.assert_array_equal(cat_dec_centroid, expected_centroid[1])
+    np.testing.assert_array_equal(cat_ra_psf, expected_psf[0])
+    np.testing.assert_array_equal(cat_dec_psf, expected_psf[1])
+
+
+def test_source_catalog_coordinates_have_changed(tmp_path, base_image):
+    """Test that the original catalog file content is different from the updated file."""
+
+    os.chdir(tmp_path)
+
+    img = base_image(shift_1=1000, shift_2=1000)
+    add_tweakreg_catalog_attribute(tmp_path, img, catalog_filename="img_1")
+
+    tweakreg = trs.TweakRegStep()
+
+    # create SourceCatalogModel
+    source_catalog_model = setup_source_catalog_model(img)
+
+    # save SourceCatalogModel
+    tweakreg.save_model(
+        source_catalog_model,
+        output_file="img_1.asdf",
+        suffix="cat",
+        force=True,
+    )
+    # save original data
+    shutil.copy("img_1_cat.asdf", "img_1_cat_original.asdf")
+
+    # update tweakreg catalog name
+    img.meta.source_detection.tweakreg_catalog_name = "img_1_cat.asdf"
+
+    # run TweakRegStep
+    res = trs.TweakRegStep.call([img])
+
+    # tweak the current WCS using TweakRegStep and save the updated cat file
+    with res:
+        dm = res.borrow(0)
+        assert dm.meta.source_detection.tweakreg_catalog_name == "img_1_cat.asdf"
+        tweakreg.update_catalog_coordinates(
+            dm.meta.source_detection.tweakreg_catalog_name, dm.meta.wcs
+        )
+        res.shelve(dm, 0)
+
+    cat_original = rdm.open("img_1_cat_original.asdf")
+    cat_updated = rdm.open("img_1_cat.asdf")
+
+    # set max absolute and relative tolerance to ~ 1/2 a pixel
+    atol = u.Quantity(0.11 / 2, "arcsec").to("deg").value
+    rtol = 5e-8
+
+    # testing that nothing moved by more than 1/2 a pixel
+    assert np.allclose(
+        cat_original.source_catalog["ra_centroid"],
+        cat_updated.source_catalog["ra_centroid"],
+        atol=atol,
+        rtol=rtol,
+    )
+    assert np.allclose(
+        cat_original.source_catalog["dec_centroid"],
+        cat_updated.source_catalog["dec_centroid"],
+        atol=atol,
+        rtol=rtol,
+    )
+    assert np.allclose(
+        cat_original.source_catalog["ra_psf"],
+        cat_updated.source_catalog["ra_psf"],
+        atol=atol,
+        rtol=rtol,
+    )
+    assert np.allclose(
+        cat_original.source_catalog["dec_psf"],
+        cat_updated.source_catalog["dec_psf"],
+        atol=atol,
+        rtol=rtol,
+    )
+    # testing that things did move by more than ~ 1/100 of a pixel
+    assert not np.allclose(
+        cat_original.source_catalog["ra_centroid"],
+        cat_updated.source_catalog["ra_centroid"],
+        atol=atol / 100,
+        rtol=rtol / 100,
+    )
+    assert not np.allclose(
+        cat_original.source_catalog["dec_centroid"],
+        cat_updated.source_catalog["dec_centroid"],
+        atol=atol / 100,
+        rtol=rtol / 100,
+    )
+    assert not np.allclose(
+        cat_original.source_catalog["ra_psf"],
+        cat_updated.source_catalog["ra_psf"],
+        atol=atol / 100,
+        rtol=rtol / 100,
+    )
+    assert not np.allclose(
+        cat_original.source_catalog["dec_psf"],
+        cat_updated.source_catalog["dec_psf"],
+        atol=atol / 100,
+        rtol=rtol / 100,
+    )
+
+
+def setup_source_catalog_model(img):
+    """
+    Set up the source catalog model.
+
+    Notes
+    -----
+    This function reads the source catalog from a file, renames columns to match
+    expected names, adds mock PSF coordinates, applies random shifts to the centroid
+    and PSF coordinates, and calculates the world coordinates for the centroids.
+    """
+    cat_model = rdm.SourceCatalogModel
+    source_catalog_model = maker_utils.mk_datamodel(cat_model)
+    # this will be the output filename
+    source_catalog_model.meta.filename = "img_1.asdf"
+
+    # read in the mock table
+    source_catalog = Table.read("img_1", format="ascii.ecsv")
+    # rename columns to match expected column names
+    source_catalog.rename_columns(["x", "y"], ["xcentroid", "ycentroid"])
+    # add mock PSF coordinates
+    source_catalog["x_psf"] = source_catalog["xcentroid"]
+    source_catalog["y_psf"] = source_catalog["ycentroid"]
+
+    # generate a set of random shifts to be added to the original coordinates
+    seed = 13
+    rng = default_rng(seed)
+    shift_x = rng.uniform(-0.5, 0.5, size=len(source_catalog))
+    shift_y = rng.uniform(-0.5, 0.5, size=len(source_catalog))
+    # add random fraction of a pixel shifts to the centroid coordinates
+    source_catalog["xcentroid"] += shift_x
+    source_catalog["ycentroid"] += shift_y
+
+    # generate another set of random shifts to be added to the original coordinates
+    seed = 5
+    rng = default_rng(seed)
+    shift_x = rng.uniform(-0.5, 0.5, size=len(source_catalog))
+    shift_y = rng.uniform(-0.5, 0.5, size=len(source_catalog))
+    # add random fraction of a pixel shifts to the centroid coordinates
+    source_catalog["x_psf"] += shift_x
+    source_catalog["y_psf"] += shift_y
+
+    # calculate centroid world coordinates
+    centroid = img.meta.wcs(
+        source_catalog["xcentroid"],
+        source_catalog["ycentroid"],
+    )
+    # calculate PSF world coordinates
+    psf = img.meta.wcs(
+        source_catalog["x_psf"],
+        source_catalog["y_psf"],
+    )
+    # add world coordinates to catalog
+    source_catalog["ra_centroid"], source_catalog["dec_centroid"] = centroid
+    source_catalog["ra_psf"], source_catalog["dec_psf"] = psf
+    # add units
+    source_catalog["ra_centroid"].unit = u.deg
+    source_catalog["dec_centroid"].unit = u.deg
+    source_catalog["ra_psf"].unit = u.deg
+    source_catalog["dec_psf"].unit = u.deg
+
+    # add source catalog to SourceCatalogModel
+    source_catalog_model.source_catalog = source_catalog
+
+    return source_catalog_model
+
+
 @pytest.mark.parametrize(
     "exposure_type",
     ["WFI_GRISM", "WFI_PRISM", "WFI_DARK", "WFI_FLAT", "WFI_WFSC"],
@@ -1045,7 +1268,7 @@ def test_tweakreg_skips_invalid_exposure_types(exposure_type, tmp_path, base_ima
 def test_validate_catalog_columns(catalog_data, expected_colnames, raises_exception):
     """Test that TweakRegStep._validate_catalog_columns() correctly validates the
     presence of required columns ('x' and 'y') in the provided catalog."""
-    catalog = table.Table(catalog_data)
+    catalog = Table(catalog_data)
     if raises_exception:
         with pytest.raises(ValueError):
             _validate_catalog_columns(catalog)
diff --git a/romancal/tweakreg/tweakreg_step.py b/romancal/tweakreg/tweakreg_step.py
index d85267ecd..9dbddd710 100644
--- a/romancal/tweakreg/tweakreg_step.py
+++ b/romancal/tweakreg/tweakreg_step.py
@@ -57,6 +57,7 @@ class TweakRegStep(RomanStep):
         abs_nclip = integer(min=0, default=3) # Number of clipping iterations in fit when performing absolute astrometry
         abs_sigma = float(min=0.0, default=3.0) # Clipping limit in sigma units when performing absolute astrometry
         output_use_model = boolean(default=True)  # When saving use `DataModel.meta.filename`
+        update_source_catalog_coordinates = boolean(default=False) # Update source catalog file with tweaked coordinates?
     """  # noqa: E501
 
     reference_file_types = []
@@ -274,6 +275,50 @@ def process(self, input):
 
         return images
 
+    def update_catalog_coordinates(self, tweakreg_catalog_name, tweaked_wcs):
+        """
+        Update the source catalog coordinates using the tweaked WCS.
+
+        Parameters
+        ----------
+        tweakreg_catalog_name : str
+            The name of the TweakReg catalog file produced by `SourceCatalog`.
+        tweaked_wcs : `gwcs.wcs.WCS`
+            The tweaked World Coordinate System (WCS) object.
+
+        Returns
+        -------
+        None
+        """
+        # read in cat file
+        with rdm.open(tweakreg_catalog_name) as source_catalog_model:
+            # get catalog
+            catalog = source_catalog_model.source_catalog
+
+            # define mapping between pixel and world coordinates
+            colname_mapping = {
+                ("xcentroid", "ycentroid"): ("ra_centroid", "dec_centroid"),
+                ("x_psf", "y_psf"): ("ra_psf", "dec_psf"),
+            }
+
+            for k, v in colname_mapping.items():
+                # get column names
+                x_colname, y_colname = k
+                ra_colname, dec_colname = v
+
+                # calculate new coordinates using tweaked WCS and update catalog coordinates
+                catalog[ra_colname], catalog[dec_colname] = tweaked_wcs(
+                    catalog[x_colname], catalog[y_colname]
+                )
+
+            # save updated catalog (overwrite cat file)
+            self.save_model(
+                source_catalog_model,
+                output_file=source_catalog_model.meta.filename,
+                suffix="cat",
+                force=True,
+            )
+
     def read_catalog(self, catalog_name):
         """
         Reads a source catalog from a specified file.

From 91e9794ae8ee303fac05c1edc0dfca7e032965af Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Tue, 1 Oct 2024 08:37:24 -0400
Subject: [PATCH 24/74] fix artifactory_repos for pytest 8

---
 romancal/regtest/conftest.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/romancal/regtest/conftest.py b/romancal/regtest/conftest.py
index 9f9586bc8..618fd35ca 100644
--- a/romancal/regtest/conftest.py
+++ b/romancal/regtest/conftest.py
@@ -17,12 +17,17 @@
 @pytest.fixture(scope="session")
 def artifactory_repos(pytestconfig):
     """Provides Artifactory inputs_root and results_root"""
-    try:
-        inputs_root = pytestconfig.getini("inputs_root")[0]
-        results_root = pytestconfig.getini("results_root")[0]
-    except IndexError:
+    inputs_root = pytestconfig.getini("inputs_root")
+    if not inputs_root:
         inputs_root = "roman-pipeline"
+    else:
+        inputs_root = inputs_root[0]
+
+    results_root = pytestconfig.getini("results_root")
+    if not results_root:
         results_root = "roman-pipeline-results"
+    else:
+        results_root = results_root[0]
     return inputs_root, results_root
 
 

From 6fb921c9e5a6c0ed33db96478cf54957d52711f3 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Thu, 22 Aug 2024 15:11:36 -0400
Subject: [PATCH 25/74] remove MultilineLogger

---
 romancal/associations/association.py    |  3 ++-
 romancal/associations/generate.py       |  2 +-
 romancal/associations/lib/log_config.py | 23 -----------------------
 romancal/flux/flux_step.py              |  2 +-
 4 files changed, 4 insertions(+), 26 deletions(-)

diff --git a/romancal/associations/association.py b/romancal/associations/association.py
index b9bb7425d..bc270cab6 100644
--- a/romancal/associations/association.py
+++ b/romancal/associations/association.py
@@ -197,7 +197,8 @@ def validate(cls, asn):
         try:
             jsonschema.validate(asn_data, asn_schema)
         except (AttributeError, jsonschema.ValidationError) as err:
-            logger.debug("Validation failed:\n%s", err)
+            logger.debug("Validation failed:")
+            logger.debug("%s", err)
             raise AssociationNotValidError("Validation failed") from err
         return True
 
diff --git a/romancal/associations/generate.py b/romancal/associations/generate.py
index ed35dff30..f8f012c80 100644
--- a/romancal/associations/generate.py
+++ b/romancal/associations/generate.py
@@ -94,7 +94,7 @@ def generate(pool, rules, version_id=None, finalize=True):
         logger.debug("New process lists: %d", total_reprocess)
         logger.debug("Updated process queue: %s", process_queue)
         logger.debug("# associations: %d", len(associations))
-        logger.debug("Seconds to process: %.2f\n", timer() - time_start)
+        logger.debug("Seconds to process: %.2f", timer() - time_start)
 
     # Finalize found associations
     logger.debug("# associations before finalization: %d", len(associations))
diff --git a/romancal/associations/lib/log_config.py b/romancal/associations/lib/log_config.py
index 81b67a426..36b742b47 100644
--- a/romancal/associations/lib/log_config.py
+++ b/romancal/associations/lib/log_config.py
@@ -3,7 +3,6 @@
 import logging
 import sys
 from collections import defaultdict
-from functools import partialmethod
 from logging.config import dictConfig
 
 __all__ = ["log_config"]
@@ -130,28 +129,6 @@ def format(self, record):
 }
 
 
-class MultilineLogger(logging.getLoggerClass()):
-    """Split multilines so that each line is logged separately"""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def log(self, level, msg, *args, **kwargs):
-        if self.isEnabledFor(level):
-            for line in msg.split("\n"):
-                self._log(level, line, args, **kwargs)
-
-    debug = partialmethod(log, logging.DEBUG)
-    info = partialmethod(log, logging.INFO)
-    warning = partialmethod(log, logging.WARNING)
-    error = partialmethod(log, logging.ERROR)
-    critical = partialmethod(log, logging.CRITICAL)
-    fatal = critical
-
-
-logging.setLoggerClass(MultilineLogger)
-
-
 def log_config(name=None, user_name=None, logger_config=None, config=None, merge=True):
     """Setup logging with defaults
 
diff --git a/romancal/flux/flux_step.py b/romancal/flux/flux_step.py
index ca7fc5cce..2fdf0647a 100644
--- a/romancal/flux/flux_step.py
+++ b/romancal/flux/flux_step.py
@@ -109,7 +109,7 @@ def apply_flux_correction(model):
             f"Input data is already in flux units of MJy/sr."
             "\nFlux correction already applied."
         )
-        log.info(message)
+        log.info("Flux correction already applied.")
         return
 
     # Apply the correction.

From cc62ed2586f738861082a821d6affded2d351896 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Thu, 8 Aug 2024 09:03:46 -0400
Subject: [PATCH 26/74] remove unused variables

---
 romancal/outlier_detection/outlier_detection_step.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/romancal/outlier_detection/outlier_detection_step.py b/romancal/outlier_detection/outlier_detection_step.py
index 332791e4b..7e1fd1cdf 100644
--- a/romancal/outlier_detection/outlier_detection_step.py
+++ b/romancal/outlier_detection/outlier_detection_step.py
@@ -34,10 +34,7 @@ class OutlierDetectionStep(RomanStep):
         pixfrac = float(default=1.0) # Fraction by which input pixels are shrunk before being drizzled onto the output image grid
         kernel = string(default='square') # Shape of the kernel used for flux distribution onto output images
         fillval = string(default='INDEF') # Value assigned to output pixels that have zero weight or no flux during drizzling
-        nlow = integer(default=0) # The number of low values in each pixel stack to ignore when computing the median value
-        nhigh = integer(default=0) # The number of high values in each pixel stack to ignore when computing the median value
         maskpt = float(default=0.7) # Percentage of weight image values below which they are flagged as bad pixels
-        grow = integer(default=1) # The distance beyond the rejection limit for additional pixels to be rejected in an image
         snr = string(default='5.0 4.0') # The signal-to-noise values to use for bad pixel identification
         scale = string(default='1.2 0.7') # The scaling factor applied to derivative used to identify bad pixels
         backg = float(default=0.0) # User-specified background value to subtract during final identification step
@@ -45,7 +42,6 @@ class OutlierDetectionStep(RomanStep):
         save_intermediate_results = boolean(default=False) # Specifies whether or not to write out intermediate products to disk
         resample_data = boolean(default=True) # Specifies whether or not to resample the input images when performing outlier detection
         good_bits = string(default="~DO_NOT_USE+NON_SCIENCE")  # DQ bit value to be considered 'good'
-        allowed_memory = float(default=None)  # Fraction of memory to use for the combined image
         in_memory = boolean(default=False) # Specifies whether or not to keep all intermediate products and datamodels in memory
     """  # noqa: E501
 
@@ -109,10 +105,7 @@ def process(self, input_models):
             "pixfrac": self.pixfrac,
             "kernel": self.kernel,
             "fillval": self.fillval,
-            "nlow": self.nlow,
-            "nhigh": self.nhigh,
             "maskpt": self.maskpt,
-            "grow": self.grow,
             "snr": self.snr,
             "scale": self.scale,
             "backg": self.backg,
@@ -120,7 +113,6 @@ def process(self, input_models):
             "save_intermediate_results": self.save_intermediate_results,
             "resample_data": self.resample_data,
             "good_bits": self.good_bits,
-            "allowed_memory": self.allowed_memory,
             "in_memory": self.in_memory,
             "make_output_path": self.make_output_path,
             "resample_suffix": "i2d",

From a0d224a9a668ceea3fa092590471f0fbd45a978d Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Thu, 8 Aug 2024 09:20:34 -0400
Subject: [PATCH 27/74] raise on invalid input

---
 .../outlier_detection_step.py                 | 44 +++++++------------
 .../tests/test_outlier_detection.py           | 18 +++-----
 2 files changed, 21 insertions(+), 41 deletions(-)

diff --git a/romancal/outlier_detection/outlier_detection_step.py b/romancal/outlier_detection/outlier_detection_step.py
index 7e1fd1cdf..05edfe307 100644
--- a/romancal/outlier_detection/outlier_detection_step.py
+++ b/romancal/outlier_detection/outlier_detection_step.py
@@ -48,19 +48,10 @@ class OutlierDetectionStep(RomanStep):
     def process(self, input_models):
         """Perform outlier detection processing on input data."""
 
-        self.skip = False
-
         if isinstance(input_models, ModelLibrary):
             library = input_models
         else:
-            try:
-                library = ModelLibrary(input_models)
-            except Exception:
-                self.log.warning(
-                    "Skipping outlier_detection - input cannot be parsed into a ModelLibrary."
-                )
-                self.skip = True
-                return input_models
+            library = ModelLibrary(input_models)
 
         # check number of input models
         if len(library) < 2:
@@ -70,29 +61,24 @@ def process(self, input_models):
             self.log.warning(
                 "Skipping outlier_detection - at least two imaging observations are needed."
             )
-            self.skip = True
 
-        # check that all inputs are WFI_IMAGE
-        if not self.skip:
-            with library:
-                for i, model in enumerate(library):
-                    if model.meta.exposure.type != "WFI_IMAGE":
-                        self.skip = True
-                    library.shelve(model, i, modify=False)
-                if self.skip:
-                    self.log.warning(
-                        "Skipping outlier_detection - all WFI_IMAGE exposures are required."
-                    )
+            def set_skip(model, index):
+                model.meta.cal_step["outlier_detection"] = "SKIPPED"
+
+            list(library.map_function(set_skip))
 
-        # if skipping for any reason above...
-        if self.skip:
-            # set meta.cal_step.outlier_detection to SKIPPED
-            with library:
-                for i, model in enumerate(library):
-                    model.meta.cal_step["outlier_detection"] = "SKIPPED"
-                    library.shelve(model, i)
             return library
 
+        # check that all inputs are WFI_IMAGE
+        def get_exptype(model, index):
+            return model.meta.exposure.type
+
+        exptypes = list(library.map_function(get_exptype, modify=False))
+        if any(exptype != "WFI_IMAGE" for exptype in exptypes):
+            raise ValueError(
+                f"outlier_detection only supports WFI_IMAGE exposure types: {set(exptypes)}"
+            )
+
         # Setup output path naming if associations are involved.
         asn_id = library.asn.get("asn_id", None)
         if asn_id is not None:
diff --git a/romancal/outlier_detection/tests/test_outlier_detection.py b/romancal/outlier_detection/tests/test_outlier_detection.py
index d27ed9ed9..78d296195 100644
--- a/romancal/outlier_detection/tests/test_outlier_detection.py
+++ b/romancal/outlier_detection/tests/test_outlier_detection.py
@@ -12,16 +12,14 @@
 @pytest.mark.parametrize(
     "input_models",
     [
-        list(),
         "",
     ],
 )
-def test_outlier_raises_error_on_invalid_input_models(input_models, caplog):
+def test_outlier_raises_error_on_invalid_input_models(input_models):
     """Test that OutlierDetection logs out a WARNING if input is invalid."""
 
-    OutlierDetectionStep.call(input_models)
-
-    assert "WARNING" in [x.levelname for x in caplog.records]
+    with pytest.raises(IsADirectoryError):
+        OutlierDetectionStep.call(input_models)
 
 
 def test_outlier_skips_step_on_invalid_number_of_elements_in_input(base_image):
@@ -37,7 +35,7 @@ def test_outlier_skips_step_on_invalid_number_of_elements_in_input(base_image):
             res.shelve(m, i, modify=False)
 
 
-def test_outlier_skips_step_on_exposure_type_different_from_wfi_image(base_image):
+def test_outlier_raises_exception_on_exposure_type_different_from_wfi_image(base_image):
     """
     Test if the outlier detection step is skipped when the exposure type is different from WFI image.
     """
@@ -46,12 +44,8 @@ def test_outlier_skips_step_on_exposure_type_different_from_wfi_image(base_image
     img_2 = base_image()
     img_2.meta.exposure.type = "WFI_PRISM"
 
-    res = OutlierDetectionStep.call(ModelLibrary([img_1, img_2]))
-
-    with res:
-        for i, m in enumerate(res):
-            assert m.meta.cal_step.outlier_detection == "SKIPPED"
-            res.shelve(m, i, modify=False)
+    with pytest.raises(ValueError, match="only supports WFI_IMAGE"):
+        OutlierDetectionStep.call(ModelLibrary([img_1, img_2]))
 
 
 def test_outlier_valid_input_asn(tmp_path, base_image, create_mock_asn_file):

From a016b3738c1fd73f984f5cf428a763c388bdb212 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Thu, 8 Aug 2024 16:04:02 -0400
Subject: [PATCH 28/74] update outlier detection docs for spec update

---
 docs/roman/outlier_detection/arguments.rst    | 28 +++----------------
 .../outlier_detection/outlier_detection.rst   |  5 ----
 2 files changed, 4 insertions(+), 29 deletions(-)

diff --git a/docs/roman/outlier_detection/arguments.rst b/docs/roman/outlier_detection/arguments.rst
index 1ecbe63ba..b6bd76e28 100644
--- a/docs/roman/outlier_detection/arguments.rst
+++ b/docs/roman/outlier_detection/arguments.rst
@@ -5,7 +5,7 @@ Step Arguments
 The ``outlier_detection`` step has the following optional arguments that control the
 behavior of the processing:
 
-``--weight_type`` (string, default='exptime')
+``--weight_type`` (string, default='ivm')
   The type of data weighting to use during resampling the images for creating the
   median image used for detecting outliers; options are `'ivm'`, `'exptime'`,
   and `None` (see :ref:`weight_type_options_details_section` for details).
@@ -55,23 +55,11 @@ behavior of the processing:
     Any floating-point value, given as a string, is valid.
     A value of 'INDEF' will use the last zero weight flux.
 
-``--nlow`` (integer, default=0)
-  The number of low values in each pixel stack to ignore when computing the median
-  value.
-
-``--nhigh`` (integer, default=0)
-  The number of high values in each pixel stack to ignore when computing the median
-  value.
-
 ``--maskpt`` (float, default=0.7)
   Percentage of weight image values below which they are flagged as bad and rejected
   from the median image. Valid values range from 0.0 to 1.0.
 
-``--grow`` (integer, default=1)
-  The distance, in pixels, beyond the limit set by the rejection algorithm being
-  used, for additional pixels to be rejected in an image.
-
-``--snr`` (string, default='4.0 3.0')
+``--snr`` (string, default='5.0 4.0')
   The signal-to-noise values to use for bad pixel identification. Since cosmic rays
   often extend across several pixels the user must specify two cut-off values for
   determining whether a pixel should be masked: the first for detecting the primary
@@ -79,7 +67,7 @@ behavior of the processing:
   pixels adjacent to those found in the first pass.  Valid values are a pair of
   floating-point values in a single string.
 
-``--scale`` (string, default='0.5 0.4')
+``--scale`` (string, default='1.2 0.7')
   The scaling factor applied to derivative used to identify bad pixels. Since cosmic
   rays often extend across several pixels the user must specify two cut-off values for
   determining whether a pixel should be masked: the first for detecting the primary
@@ -104,21 +92,13 @@ behavior of the processing:
   Specifies whether or not to resample the input images when performing outlier
   detection.
 
-``--good_bits`` (string, default=0)
+``--good_bits`` (string, default='~DO_NOT_USE+NON_SCIENCE')
   The DQ bit values from the input image DQ arrays that should be considered 'good'
   when creating masks of bad pixels during outlier detection when resampling the data.
   See `Roman's Data Quality Flags
   <https://github.com/spacetelescope/romancal/blob/main/romancal/lib/dqflags.py>`_
   for details.
 
-``--allowed_memory`` (float, default=None)
-  Specifies the fractional amount of free memory to allow when creating the resampled
-  image. If ``None``, the environment variable ``DMODEL_ALLOWED_MEMORY`` is used. If
-  not defined, no check is made. If the resampled image would be larger than specified,
-  an ``OutputTooLargeError`` exception will be generated. For example, if set to
-  ``0.5``, only resampled images that use less than half the available memory can be
-  created.
-
 ``--in_memory`` (boolean, default=False)
   Specifies whether or not to keep all intermediate products and datamodels in
   memory at the same time during the processing of this step.  If set to `False`,
diff --git a/docs/roman/outlier_detection/outlier_detection.rst b/docs/roman/outlier_detection/outlier_detection.rst
index c11aa482c..22bfbd4a2 100644
--- a/docs/roman/outlier_detection/outlier_detection.rst
+++ b/docs/roman/outlier_detection/outlier_detection.rst
@@ -55,14 +55,9 @@ Specifically, this routine performs the following operations:
 
    * The median image is created by combining all grouped mosaic images or
      non-resampled input data pixel-by-pixel.
-   * The ``nlow`` and ``nhigh`` parameters specify how many low and high values
-     to ignore when computing the median for any given pixel.
    * The ``maskpt`` parameter sets the percentage of the weight image values to
      use, and any pixel with a weight below this value gets flagged as "bad" and
      ignored when resampled.
-   * The ``grow`` parameter sets the width, in pixels, beyond the limit set by
-     the rejection algorithm being used, for additional pixels to be rejected in
-     an image.
    * The median image is written out to disk as `_<asn_id>_median` by default.
 
 #. By default, the median image is blotted back (inverse of resampling) to

From ac9d89e0592fd8761a90fd95dccc9dea516925ef Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Thu, 8 Aug 2024 16:09:11 -0400
Subject: [PATCH 29/74] reference spec in arguments docs

---
 docs/roman/outlier_detection/arguments.rst | 36 ++++++++++++----------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/docs/roman/outlier_detection/arguments.rst b/docs/roman/outlier_detection/arguments.rst
index b6bd76e28..b2f040d74 100644
--- a/docs/roman/outlier_detection/arguments.rst
+++ b/docs/roman/outlier_detection/arguments.rst
@@ -1,16 +1,19 @@
 .. _outlier_detection_step_args:
 
+For more details about step arguments (including datatypes, possible values
+and defaults) see :py:obj:`romancal.outlier_detection.OutlierDetectionStep.spec`.
+
 Step Arguments
 ==============
 The ``outlier_detection`` step has the following optional arguments that control the
 behavior of the processing:
 
-``--weight_type`` (string, default='ivm')
+``--weight_type``
   The type of data weighting to use during resampling the images for creating the
   median image used for detecting outliers; options are `'ivm'`, `'exptime'`,
   and `None` (see :ref:`weight_type_options_details_section` for details).
 
-``--pixfrac`` (float, default=1.0)
+``--pixfrac``
   Fraction by which input pixels are “shrunk” before being drizzled onto the output
   image grid, given as a real number between 0 and 1. This specifies the size of the
   footprint, or “dropsize”, of a pixel in units of the input pixel size. If `pixfrac`
@@ -20,7 +23,7 @@ behavior of the processing:
   output drizzled image is fully populated with pixels from the input image.
   Valid values range from 0.0 to 1.0.
 
-``--kernel`` (string, default='square')
+``--kernel``
   This parameter specifies the form of the kernel function used to distribute
   flux onto the separate output images, for the initial separate drizzling
   operation only. The value options for this parameter include:
@@ -43,7 +46,7 @@ behavior of the processing:
          should never be used for ``pixfrac != 1.0``, and is not recommended
          for ``scale!=1.0``.
 
-``--fillval`` (string, default='INDEF')
+``--fillval``
     The value for this parameter is to be assigned to the output pixels that
     have zero weight or which do not receive flux from any input pixels during
     drizzling. This parameter corresponds to the ``fillval`` parameter of the
@@ -55,51 +58,52 @@ behavior of the processing:
     Any floating-point value, given as a string, is valid.
     A value of 'INDEF' will use the last zero weight flux.
 
-``--maskpt`` (float, default=0.7)
+``--maskpt``
   Percentage of weight image values below which they are flagged as bad and rejected
   from the median image. Valid values range from 0.0 to 1.0.
 
-``--snr`` (string, default='5.0 4.0')
+``--snr``
   The signal-to-noise values to use for bad pixel identification. Since cosmic rays
   often extend across several pixels the user must specify two cut-off values for
   determining whether a pixel should be masked: the first for detecting the primary
   cosmic ray, and the second (typically lower threshold) for masking lower-level bad
   pixels adjacent to those found in the first pass.  Valid values are a pair of
-  floating-point values in a single string.
+  floating-point values in a single string (for example "5.0 4.0").
 
-``--scale`` (string, default='1.2 0.7')
+``--scale``
   The scaling factor applied to derivative used to identify bad pixels. Since cosmic
   rays often extend across several pixels the user must specify two cut-off values for
   determining whether a pixel should be masked: the first for detecting the primary
   cosmic ray, and the second (typically lower threshold) for masking lower-level bad
   pixels adjacent to those found in the first pass.  Valid values are a pair of
-  floating-point values in a single string.
+  floating-point values in a single string (for example "1.2 0.7").
 
-``--backg`` (float, default=0.0)
+``--backg``
   User-specified background value (scalar) to subtract during final identification
   step of outliers in `driz_cr` computation.
 
-``--kernel_size`` (string, default='7 7')
+``--kernel_size``
   Size of kernel to be used during resampling of the data
-  (i.e. when `resample_data=True`).
+  (i.e. when `resample_data=True`). Valid values are a pair of ints in a single string
+  (for example "7 7").
 
-``--save_intermediate_results`` (boolean, default=False)
+``--save_intermediate_results``
   Specifies whether or not to write out intermediate products such as median image or
   resampled individual input exposures to disk. Typically, only used to track down
   problems with final results when too many or too few pixels are flagged as outliers.
 
-``--resample_data`` (boolean, default=True)
+``--resample_data``
   Specifies whether or not to resample the input images when performing outlier
   detection.
 
-``--good_bits`` (string, default='~DO_NOT_USE+NON_SCIENCE')
+``--good_bits``
   The DQ bit values from the input image DQ arrays that should be considered 'good'
   when creating masks of bad pixels during outlier detection when resampling the data.
   See `Roman's Data Quality Flags
   <https://github.com/spacetelescope/romancal/blob/main/romancal/lib/dqflags.py>`_
   for details.
 
-``--in_memory`` (boolean, default=False)
+``--in_memory``
   Specifies whether or not to keep all intermediate products and datamodels in
   memory at the same time during the processing of this step.  If set to `False`,
   all input and output data will be written to disk at the start of the step

From ddb72f3874488eb58bb6a17d849f05bd4f8730bc Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 30 Sep 2024 13:18:13 -0400
Subject: [PATCH 30/74] add fragments

---
 changes/1357.outlier_detection.0.rst | 1 +
 changes/1357.outlier_detection.1.rst | 1 +
 changes/1357.outlier_detection.2.rst | 1 +
 3 files changed, 3 insertions(+)
 create mode 100644 changes/1357.outlier_detection.0.rst
 create mode 100644 changes/1357.outlier_detection.1.rst
 create mode 100644 changes/1357.outlier_detection.2.rst

diff --git a/changes/1357.outlier_detection.0.rst b/changes/1357.outlier_detection.0.rst
new file mode 100644
index 000000000..3c3667401
--- /dev/null
+++ b/changes/1357.outlier_detection.0.rst
@@ -0,0 +1 @@
+Remove unused arguments to outlier detection.
diff --git a/changes/1357.outlier_detection.1.rst b/changes/1357.outlier_detection.1.rst
new file mode 100644
index 000000000..798d48c35
--- /dev/null
+++ b/changes/1357.outlier_detection.1.rst
@@ -0,0 +1 @@
+Update input handling to raise an exception on an invalid input instead of issuing a warning and skipping the step.
diff --git a/changes/1357.outlier_detection.2.rst b/changes/1357.outlier_detection.2.rst
new file mode 100644
index 000000000..dc0be573d
--- /dev/null
+++ b/changes/1357.outlier_detection.2.rst
@@ -0,0 +1 @@
+Use stcal common code in outlier detection.

From 128b657005c0b4a939f9064c5496adafc170cdf1 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 30 Sep 2024 16:00:06 -0400
Subject: [PATCH 31/74] use outlier detection from stcal

---
 romancal/outlier_detection/_fileio.py         |  44 ++
 .../outlier_detection/outlier_detection.py    | 401 ------------------
 .../outlier_detection_step.py                 |  75 +---
 .../tests/test_outlier_detection.py           |  57 +--
 romancal/outlier_detection/utils.py           | 317 ++++++++++++++
 romancal/resample/resample.py                 |  84 ++++
 6 files changed, 471 insertions(+), 507 deletions(-)
 create mode 100644 romancal/outlier_detection/_fileio.py
 delete mode 100644 romancal/outlier_detection/outlier_detection.py
 create mode 100644 romancal/outlier_detection/utils.py

diff --git a/romancal/outlier_detection/_fileio.py b/romancal/outlier_detection/_fileio.py
new file mode 100644
index 000000000..3cf13d933
--- /dev/null
+++ b/romancal/outlier_detection/_fileio.py
@@ -0,0 +1,44 @@
+import logging
+
+from astropy.units import Quantity
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.DEBUG)
+
+
+def save_median(example_model, median_data, median_wcs, make_output_path):
+    _save_intermediate_output(
+        _make_median_model(example_model, median_data, median_wcs),
+        "median",
+        make_output_path,
+    )
+
+
+def save_drizzled(drizzled_model, make_output_path):
+    _save_intermediate_output(drizzled_model, "outlier_i2d", make_output_path)
+
+
+def _make_median_model(example_model, data, wcs):
+    model = example_model.copy()
+    model.data = Quantity(data, unit=model.data.unit)
+    model.meta.wcs = wcs
+    return model
+
+
+def _save_intermediate_output(model, suffix, make_output_path):
+    """
+    Ensure all intermediate outputs from OutlierDetectionStep have consistent file naming conventions
+
+    Notes
+    -----
+    self.make_output_path() is updated globally for the step in the main pipeline
+    to include the asn_id in the output path, so no need to handle it here.
+    """
+
+    # outlier_?2d is not a known suffix, and make_output_path cannot handle an
+    # underscore in an unknown suffix, so do a manual string replacement
+    input_path = model.meta.filename.replace("_outlier_", "_")
+
+    output_path = make_output_path(input_path, suffix=suffix)
+    model.save(output_path)
+    log.info(f"Saved {suffix} model in {output_path}")
diff --git a/romancal/outlier_detection/outlier_detection.py b/romancal/outlier_detection/outlier_detection.py
deleted file mode 100644
index 3a1b9d175..000000000
--- a/romancal/outlier_detection/outlier_detection.py
+++ /dev/null
@@ -1,401 +0,0 @@
-"""Primary code for performing outlier detection on Roman observations."""
-
-import copy
-import logging
-from functools import partial
-
-import numpy as np
-from astropy.stats import sigma_clip
-from astropy.units import Quantity
-from drizzle.cdrizzle import tblot
-from roman_datamodels.dqflags import pixel
-from scipy import ndimage
-
-from romancal.resample import resample
-from romancal.resample.resample_utils import build_driz_weight, calc_gwcs_pixmap
-
-from ..stpipe import RomanStep
-
-log = logging.getLogger(__name__)
-log.setLevel(logging.DEBUG)
-
-__all__ = ["OutlierDetection", "flag_cr", "abs_deriv"]
-
-
-class OutlierDetection:
-    """Main class for performing outlier detection.
-
-    This is the controlling routine for the outlier detection process.
-    It loads and sets the various input data and parameters needed by
-    the various functions and then controls the operation of this process
-    through all the steps used for the detection.
-
-    Notes
-    -----
-    This routine performs the following operations::
-
-      1. Extracts parameter settings from input model and merges
-         them with any user-provided values
-      2. Resamples all input images into grouped observation mosaics.
-      3. Creates a median image from all grouped observation mosaics.
-      4. Blot median image to match each original input image.
-      5. Perform statistical comparison between blotted image and original
-         image to identify outliers.
-      6. Updates input data model DQ arrays with mask of detected outliers.
-
-    """
-
-    default_suffix = "i2d"
-
-    def __init__(self, input_models, **pars):
-        """
-        Initialize the class with input ModelLibrary.
-
-        Parameters
-        ----------
-        input_models : ~romancal.datamodels.ModelLibrary
-            A `~romancal.datamodels.ModelLibrary` object containing the data
-            to be processed.
-
-        pars : dict, optional
-            Optional user-specified parameters to modify how outlier_detection
-            will operate.  Valid parameters include:
-            - resample_suffix
-
-        """
-        self.input_models = input_models
-
-        self.outlierpars = dict(pars)
-        self.resample_suffix = f"_outlier_{self.default_suffix if pars.get('resample_suffix') is None else pars.get('resample_suffix')}.asdf"
-        log.debug(f"Defined output product suffix as: {self.resample_suffix}")
-
-        # Define how file names are created
-        self.make_output_path = pars.get(
-            "make_output_path", partial(RomanStep._make_output_path, None)
-        )
-
-    def do_detection(self):
-        """Flag outlier pixels in DQ of input images."""  # self._convert_inputs()
-        pars = self.outlierpars
-
-        if pars["resample_data"]:
-            # Start by creating resampled/mosaic images for
-            # each group of exposures
-            resamp = resample.ResampleData(
-                self.input_models, single=True, blendheaders=False, **pars
-            )
-            drizzled_models = resamp.do_drizzle()
-
-        else:
-            # for non-dithered data, the resampled image is just the original image
-            drizzled_models = self.input_models
-            with drizzled_models:
-                for i, model in enumerate(drizzled_models):
-                    model["weight"] = build_driz_weight(
-                        model,
-                        weight_type="ivm",
-                        good_bits=pars["good_bits"],
-                    )
-                    drizzled_models.shelve(model, i)
-
-        # Perform median combination on set of drizzled mosaics
-        median_data = self.create_median(drizzled_models)
-
-        # Initialize intermediate products used in the outlier detection
-        with drizzled_models:
-            example_model = drizzled_models.borrow(0)
-            median_wcs = copy.deepcopy(example_model.meta.wcs)
-            if pars["save_intermediate_results"]:
-                median_model = example_model.copy()
-                median_model.data = median_data
-                median_model.meta.filename = "drizzled_median.asdf"
-                median_model_output_path = self.make_output_path(
-                    basepath=median_model.meta.filename,
-                    suffix="median",
-                )
-                median_model.save(median_model_output_path)
-                log.info(f"Saved model in {median_model_output_path}")
-            drizzled_models.shelve(example_model, 0, modify=False)
-
-        # Perform outlier detection using statistical comparisons between
-        # each original input image and its blotted version of the median image
-        self.detect_outliers(median_data, median_wcs, pars["resample_data"])
-
-        # clean-up (just to be explicit about being finished with
-        # these results)
-        del median_data, median_wcs
-
-    def create_median(self, resampled_models):
-        """Create a median image from the singly resampled images.
-
-        NOTES
-        -----
-        This version is simplified from astrodrizzle's version in the
-        following ways:
-        - type of combination: fixed to 'median'
-        - 'minmed' not implemented as an option
-        """
-        maskpt = self.outlierpars.get("maskpt", 0.7)
-
-        log.info("Computing median")
-
-        data = []
-
-        # Compute weight means without keeping DataModel for eacn input open
-        # keep track of resulting computation for each input resampled datamodel
-        weight_thresholds = []
-        # For each model, compute the bad-pixel threshold from the weight arrays
-        with resampled_models:
-            for i, model in enumerate(resampled_models):
-                weight = model.weight
-                # necessary in order to assure that mask gets applied correctly
-                if hasattr(weight, "_mask"):
-                    del weight._mask
-                mask_zero_weight = np.equal(weight, 0.0)
-                mask_nans = np.isnan(weight)
-                # Combine the masks
-                weight_masked = np.ma.array(
-                    weight, mask=np.logical_or(mask_zero_weight, mask_nans)
-                )
-                # Sigma-clip the unmasked data
-                weight_masked = sigma_clip(weight_masked, sigma=3, maxiters=5)
-                mean_weight = np.mean(weight_masked)
-                # Mask pixels where weight falls below maskpt percent
-                weight_threshold = mean_weight * maskpt
-                weight_thresholds.append(weight_threshold)
-                this_data = model.data.copy()
-                this_data[model.weight < weight_threshold] = np.nan
-                data.append(this_data)
-
-                resampled_models.shelve(model, i, modify=False)
-
-        median_image = np.nanmedian(data, axis=0)
-        return median_image
-
-    def detect_outliers(self, median_data, median_wcs, resampled):
-        """Flag DQ array for cosmic rays in input images.
-
-        The science frame in each ImageModel in self.input_models is compared to
-        the a blotted median image (generated with median_data and median_wcs).
-        The result is an updated DQ array in each ImageModel in input_models.
-
-        Parameters
-        ----------
-        median_data : numpy.ndarray
-            Median array that will be used as the "reference" for detecting
-            outliers.
-
-        median_wcs : gwcs.WCS
-            WCS for the median data
-
-        resampled : bool
-            True if the median data was generated from resampling the input
-            images.
-
-        Returns
-        -------
-        None
-            The dq array in each input model is modified in place
-
-        """
-        interp = self.outlierpars.get("interp", "linear")
-        sinscl = self.outlierpars.get("sinscl", 1.0)
-        log.info("Flagging outliers")
-        with self.input_models:
-            for i, image in enumerate(self.input_models):
-                # make blot_data Quantity (same unit as image.data)
-                if resampled:
-                    # blot back onto image
-                    blot_data = gwcs_blot(
-                        median_data, median_wcs, image, interp=interp, sinscl=sinscl
-                    )
-                else:
-                    # use median
-                    blot_data = median_data.copy()
-                flag_cr(image, blot_data, **self.outlierpars)
-                self.input_models.shelve(image, i)
-
-
-def flag_cr(
-    sci_image,
-    blot_data,
-    snr="5.0 4.0",
-    scale="1.2 0.7",
-    backg=0,
-    resample_data=True,
-    **kwargs,
-):
-    """Masks outliers in science image by updating DQ in-place
-
-    Mask blemishes in dithered data by comparing a science image
-    with a model image and the derivative of the model image.
-
-    Parameters
-    ----------
-    sci_image : ~romancal.DataModel.ImageModel
-        the science data
-
-    blot_data : Quantity
-        the blotted median image of the dithered science frames
-
-    snr : str
-        Signal-to-noise ratio
-
-    scale : str
-        scaling factor applied to the derivative
-
-    backg : float
-        Background value (scalar) to subtract
-
-    resample_data : bool
-        Boolean to indicate whether blot_image is created from resampled,
-        dithered data or not
-    """
-    snr1, snr2 = (float(val) for val in snr.split())
-    scale1, scale2 = (float(val) for val in scale.split())
-
-    # Get background level of science data if it has not been subtracted, so it
-    # can be added into the level of the blotted data, which has been
-    # background-subtracted
-    if (
-        hasattr(sci_image.meta, "background")
-        and sci_image.meta.background.subtracted is False
-        and sci_image.meta.background.level is not None
-    ):
-        subtracted_background = sci_image.meta.background.level
-        log.debug(f"Adding background level {subtracted_background} to blotted image")
-    else:
-        # No subtracted background.  Allow user-set value, which defaults to 0
-        subtracted_background = backg
-
-    sci_data = sci_image.data
-    blot_deriv = abs_deriv(blot_data)
-    err_data = np.nan_to_num(sci_image.err)
-
-    # create the outlier mask
-    if resample_data:  # dithered outlier detection
-        blot_data += subtracted_background
-        diff_noise = np.abs(sci_data - blot_data)
-
-        # Create a boolean mask based on a scaled version of
-        # the derivative image (dealing with interpolating issues?)
-        # and the standard n*sigma above the noise
-        threshold1 = scale1 * blot_deriv + snr1 * err_data
-        mask1 = np.greater(diff_noise, threshold1)
-
-        # Smooth the boolean mask with a 3x3 boxcar kernel
-        kernel = np.ones((3, 3), dtype=int)
-        mask1_smoothed = ndimage.convolve(mask1, kernel, mode="nearest")
-
-        # Create a 2nd boolean mask based on the 2nd set of
-        # scale and threshold values
-        threshold2 = scale2 * blot_deriv + snr2 * err_data
-        mask2 = np.greater(diff_noise, threshold2)
-
-        # Final boolean mask
-        cr_mask = mask1_smoothed & mask2
-
-    else:  # stack outlier detection
-        diff_noise = np.abs(sci_data - blot_data)
-
-        # straightforward detection of outliers for non-dithered data since
-        # err_data includes all noise sources (photon, read, and flat for baseline)
-        cr_mask = np.greater(diff_noise, snr1 * err_data)
-
-    # Count existing DO_NOT_USE pixels
-    count_existing = np.count_nonzero(sci_image.dq & pixel.DO_NOT_USE)
-
-    # Update the DQ array values in the input image but preserve datatype.
-    sci_image.dq = np.bitwise_or(
-        sci_image.dq, cr_mask * (pixel.DO_NOT_USE | pixel.OUTLIER)
-    ).astype(np.uint32)
-
-    # Report number (and percent) of new DO_NOT_USE pixels found
-    count_outlier = np.count_nonzero(sci_image.dq & pixel.DO_NOT_USE)
-    count_added = count_outlier - count_existing
-    percent_cr = count_added / (sci_image.shape[0] * sci_image.shape[1]) * 100
-    log.info(f"New pixels flagged as outliers: {count_added} ({percent_cr:.2f}%)")
-
-
-def abs_deriv(array):
-    """Take the absolute derivative of a numpy array."""
-    tmp = np.zeros(array.shape, dtype=np.float64)
-    out = np.zeros(array.shape, dtype=np.float64)
-
-    tmp[1:, :] = array[:-1, :]
-    tmp, out = _absolute_subtract(array, tmp, out)
-    tmp[:-1, :] = array[1:, :]
-    tmp, out = _absolute_subtract(array, tmp, out)
-
-    tmp[:, 1:] = array[:, :-1]
-    tmp, out = _absolute_subtract(array, tmp, out)
-    tmp[:, :-1] = array[:, 1:]
-    tmp, out = _absolute_subtract(array, tmp, out)
-
-    return out
-
-
-def _absolute_subtract(array, tmp, out):
-    tmp = np.abs(array - tmp)
-    out = np.maximum(tmp, out)
-    tmp = tmp * 0.0
-    return tmp, out
-
-
-def gwcs_blot(median_data, median_wcs, blot_img, interp="poly5", sinscl=1.0):
-    """
-    Resample the median_data to recreate an input image based on
-    the blot_img's WCS.
-
-    Parameters
-    ----------
-    median_data : numpy.ndarray
-        Median data used as the source data for blotting.
-
-    median_wcs : gwcs.WCS
-        WCS for median_data.
-
-    blot_img : datamodel
-        Datamodel containing header and WCS to define the 'blotted' image
-
-    interp : str, optional
-        The type of interpolation used in the resampling. The
-        possible values are "nearest" (nearest neighbor interpolation),
-        "linear" (bilinear interpolation), "poly3" (cubic polynomial
-        interpolation), "poly5" (quintic polynomial interpolation),
-        "sinc" (sinc interpolation), "lan3" (3rd order Lanczos
-        interpolation), and "lan5" (5th order Lanczos interpolation).
-
-    sinscl : float, optional
-        The scaling factor for sinc interpolation.
-    """
-    blot_wcs = blot_img.meta.wcs
-
-    # Compute the mapping between the input and output pixel coordinates
-    pixmap = calc_gwcs_pixmap(blot_wcs, median_wcs, blot_img.data.shape)
-    log.debug(f"Pixmap shape: {pixmap[:, :, 0].shape}")
-    log.debug(f"Sci shape: {blot_img.data.shape}")
-
-    pix_ratio = 1
-    log.info(f"Blotting {blot_img.data.shape} <-- {median_data.shape}")
-
-    outsci = np.zeros(blot_img.shape, dtype=np.float32)
-
-    # Currently tblot cannot handle nans in the pixmap, so we need to give some
-    # other value.  -1 is not optimal and may have side effects.  But this is
-    # what we've been doing up until now, so more investigation is needed
-    # before a change is made.  Preferably, fix tblot in drizzle.
-    pixmap[np.isnan(pixmap)] = -1
-    tblot(
-        median_data,
-        pixmap,
-        outsci,
-        scale=pix_ratio,
-        kscale=1.0,
-        interp=interp,
-        exptime=1.0,
-        misval=0.0,
-        sinscl=sinscl,
-    )
-
-    return outsci
diff --git a/romancal/outlier_detection/outlier_detection_step.py b/romancal/outlier_detection/outlier_detection_step.py
index 05edfe307..1029aad87 100644
--- a/romancal/outlier_detection/outlier_detection_step.py
+++ b/romancal/outlier_detection/outlier_detection_step.py
@@ -1,10 +1,9 @@
 """Public common step definition for OutlierDetection processing."""
 
 from functools import partial
-from pathlib import Path
 
 from romancal.datamodels import ModelLibrary
-from romancal.outlier_detection import outlier_detection
+from romancal.outlier_detection.utils import detect_outliers
 
 from ..stpipe import RomanStep
 
@@ -38,7 +37,6 @@ class OutlierDetectionStep(RomanStep):
         snr = string(default='5.0 4.0') # The signal-to-noise values to use for bad pixel identification
         scale = string(default='1.2 0.7') # The scaling factor applied to derivative used to identify bad pixels
         backg = float(default=0.0) # User-specified background value to subtract during final identification step
-        kernel_size = string(default='7 7') # Size of kernel to be used during resampling of the data
         save_intermediate_results = boolean(default=False) # Specifies whether or not to write out intermediate products to disk
         resample_data = boolean(default=True) # Specifies whether or not to resample the input images when performing outlier detection
         good_bits = string(default="~DO_NOT_USE+NON_SCIENCE")  # DQ bit value to be considered 'good'
@@ -85,57 +83,26 @@ def get_exptype(model, index):
             _make_output_path = self.search_attr("_make_output_path", parent_first=True)
             self._make_output_path = partial(_make_output_path, asn_id=asn_id)
 
-        detection_step = outlier_detection.OutlierDetection
-        pars = {
-            "weight_type": self.weight_type,
-            "pixfrac": self.pixfrac,
-            "kernel": self.kernel,
-            "fillval": self.fillval,
-            "maskpt": self.maskpt,
-            "snr": self.snr,
-            "scale": self.scale,
-            "backg": self.backg,
-            "kernel_size": self.kernel_size,
-            "save_intermediate_results": self.save_intermediate_results,
-            "resample_data": self.resample_data,
-            "good_bits": self.good_bits,
-            "in_memory": self.in_memory,
-            "make_output_path": self.make_output_path,
-            "resample_suffix": "i2d",
-        }
-
-        self.log.debug(f"Using {detection_step.__name__} class for outlier_detection")
+        snr1, snr2 = (float(v) for v in self.snr.split())
+        scale1, scale2 = (float(v) for v in self.scale.split())
 
         # Set up outlier detection, then do detection
-        step = detection_step(library, **pars)
-        step.do_detection()
-
-        state = "COMPLETE"
-
-        if not self.save_intermediate_results:
-            self.log.debug(
-                "The following files will be deleted since \
-                save_intermediate_results=False:"
-            )
-        with library:
-            for i, model in enumerate(library):
-                model.meta.cal_step["outlier_detection"] = state
-                if not self.save_intermediate_results:
-                    #  remove intermediate files found in
-                    #  make_output_path() and the local dir
-                    intermediate_files_paths = [
-                        Path(self.make_output_path()).parent,
-                        Path().cwd(),
-                    ]
-                    intermediate_files_suffixes = (
-                        "*blot.asdf",
-                        "*median.asdf",
-                        f'*outlier_{pars.get("resample_suffix")}.asdf',
-                    )
-                    for current_path in intermediate_files_paths:
-                        for suffix in intermediate_files_suffixes:
-                            for filename in current_path.glob(suffix):
-                                filename.unlink()
-                                self.log.debug(f"    {filename}")
-                library.shelve(model, i)
+        detect_outliers(
+            library,
+            self.weight_type,
+            self.pixfrac,
+            self.kernel,
+            self.fillval,
+            self.maskpt,
+            snr1,
+            snr2,
+            scale1,
+            scale2,
+            self.backg,
+            self.save_intermediate_results,
+            self.resample_data,
+            self.good_bits,
+            self.in_memory,
+            self.make_output_path,
+        )
         return library
diff --git a/romancal/outlier_detection/tests/test_outlier_detection.py b/romancal/outlier_detection/tests/test_outlier_detection.py
index 78d296195..94f137b78 100644
--- a/romancal/outlier_detection/tests/test_outlier_detection.py
+++ b/romancal/outlier_detection/tests/test_outlier_detection.py
@@ -6,7 +6,7 @@
 from astropy.units import Quantity
 
 from romancal.datamodels import ModelLibrary
-from romancal.outlier_detection import OutlierDetectionStep, outlier_detection
+from romancal.outlier_detection import OutlierDetectionStep
 
 
 @pytest.mark.parametrize(
@@ -98,54 +98,9 @@ def test_outlier_valid_input_modelcontainer(tmp_path, base_image):
             res.shelve(m, i, modify=False)
 
 
-@pytest.mark.parametrize(
-    "pars",
-    [
-        {
-            "weight_type": "exptime",
-            "pixfrac": 1.0,
-            "kernel": "square",
-            "fillval": "INDEF",
-            "nlow": 0,
-            "nhigh": 0,
-            "maskpt": 0.7,
-            "grow": 1,
-            "snr": "4.0 3.0",
-            "scale": "0.5 0.4",
-            "backg": 0.0,
-            "kernel_size": "7 7",
-            "save_intermediate_results": False,
-            "resample_data": True,
-            "good_bits": 0,
-            "allowed_memory": None,
-            "in_memory": True,
-            "make_output_path": None,
-            "resample_suffix": "i2d",
-        },
-        {
-            "weight_type": "exptime",
-            "save_intermediate_results": True,
-            "make_output_path": None,
-            "resample_suffix": "some_other_suffix",
-        },
-    ],
+@pytest.mark.skip(
+    reason="This creates a step then calls an internal class that no longer exists"
 )
-def test_outlier_init_default_parameters(pars, base_image):
-    """
-    Test parameter setting on initialization for OutlierDetection.
-    """
-    img_1 = base_image()
-    img_1.meta.filename = "img_1.asdf"
-    input_models = ModelLibrary([img_1])
-
-    step = outlier_detection.OutlierDetection(input_models, **pars)
-
-    assert step.input_models == input_models
-    assert step.outlierpars == pars
-    assert step.make_output_path == pars["make_output_path"]
-    assert step.resample_suffix == f"_outlier_{pars['resample_suffix']}.asdf"
-
-
 def test_outlier_do_detection_write_files_to_custom_location(tmp_path, base_image):
     """
     Test that OutlierDetection can create files on disk in a custom location.
@@ -191,7 +146,7 @@ def test_outlier_do_detection_write_files_to_custom_location(tmp_path, base_imag
         median_path,
     ]
 
-    step = outlier_detection.OutlierDetection(input_models, **pars)
+    step = outlier_detection.OutlierDetection(input_models, **pars)  # noqa: F821
     step.do_detection()
 
     assert all(x.exists() for x in outlier_files_path)
@@ -278,9 +233,7 @@ def test_identical_images(tmp_path, base_image, caplog):
     result = outlier_step(input_models)
 
     # assert that log shows no new outliers detected
-    assert "New pixels flagged as outliers: 0 (0.00%)" in {
-        x.message for x in caplog.records
-    }
+    assert "0 pixels marked as outliers" in {x.message for x in caplog.records}
     # assert that DQ array has nothing flagged as outliers
     with result:
         for i, model in enumerate(result):
diff --git a/romancal/outlier_detection/utils.py b/romancal/outlier_detection/utils.py
new file mode 100644
index 000000000..8e993a925
--- /dev/null
+++ b/romancal/outlier_detection/utils.py
@@ -0,0 +1,317 @@
+import copy
+import logging
+from functools import partial
+
+import numpy as np
+from roman_datamodels.dqflags import pixel
+from stcal.outlier_detection.median import MedianComputer
+from stcal.outlier_detection.utils import (
+    compute_weight_threshold,
+    flag_crs,
+    flag_resampled_crs,
+    gwcs_blot,
+)
+
+from romancal.resample.resample import ResampleData
+from romancal.resample.resample_utils import build_driz_weight
+
+from . import _fileio
+
+__all__ = ["detect_outliers"]
+
+
+log = logging.getLogger(__name__)
+log.setLevel(logging.DEBUG)
+
+
+def _median_with_resampling(
+    input_models,
+    resamp,
+    maskpt,
+    save_intermediate_results,
+    make_output_path,
+    buffer_size=None,
+):
+    """
+    Compute median of resampled data from models in a library.
+
+    Parameters
+    ----------
+    input_models : ModelLibrary
+        The input datamodels.
+
+    resamp : resample.resample.ResampleData object
+        The controlling object for the resampling process.
+
+    maskpt : float
+        The weight threshold for masking out low weight pixels.
+
+    save_intermediate_results : bool
+        if True, save the drizzled models and median model to fits.
+
+    make_output_path : function
+        The functools.partial instance to pass to save_median. Must be
+        specified if save_intermediate_results is True. Default None.
+
+    buffer_size : int
+        The size of chunk in bytes that will be read into memory when computing the median.
+        This parameter has no effect if the input library has its on_disk attribute
+        set to False.
+    """
+    if not resamp.single:
+        raise ValueError(
+            "median_with_resampling should only be used for resample_many_to_many"
+        )
+
+    in_memory = not input_models._on_disk
+    indices_by_group = list(input_models.group_indices.values())
+    ngroups = len(indices_by_group)
+    example_model = None
+    median_wcs = resamp.output_wcs
+
+    with input_models:
+        for i, indices in enumerate(indices_by_group):
+
+            drizzled_model = resamp.resample_group(input_models, indices)
+
+            if save_intermediate_results:
+                # write the drizzled model to file
+                _fileio.save_drizzled(drizzled_model, make_output_path)
+
+            if i == 0:
+                input_shape = (ngroups,) + drizzled_model.data.shape
+                dtype = drizzled_model.data.dtype
+                computer = MedianComputer(input_shape, in_memory, buffer_size, dtype)
+                example_model = drizzled_model
+
+            weight_threshold = compute_weight_threshold(drizzled_model.weight, maskpt)
+            drizzled_model.data[drizzled_model.weight < weight_threshold] = np.nan
+            computer.append(drizzled_model.data, i)
+            del drizzled_model
+
+    # Perform median combination on set of drizzled mosaics
+    median_data = computer.evaluate()
+
+    if save_intermediate_results:
+        # drizzled model already contains asn_id
+        _fileio.save_median(
+            example_model,
+            median_data,
+            median_wcs,
+            partial(make_output_path, asn_id=None),
+        )
+
+    return median_data, median_wcs
+
+
+def _median_without_resampling(
+    input_models,
+    maskpt,
+    weight_type,
+    good_bits,
+    save_intermediate_results,
+    make_output_path,
+    buffer_size=None,
+):
+    """
+    Compute median of data from models in a library.
+
+    Parameters
+    ----------
+    input_models : ModelLibrary
+        The input datamodels.
+
+    maskpt : float
+        The weight threshold for masking out low weight pixels.
+
+    weight_type : str
+        The type of weighting to use when combining images. Options are:
+        'ivm' (inverse variance) or 'exptime' (exposure time).
+
+    good_bits : int
+        The bit values that are considered good when determining the
+        data quality of the input.
+
+    save_intermediate_results : bool
+        if True, save the models and median model to fits.
+
+    make_output_path : function
+        The functools.partial instance to pass to save_median. Must be
+        specified if save_intermediate_results is True. Default None.
+
+    buffer_size : int
+        The size of chunk in bytes that will be read into memory when computing the median.
+        This parameter has no effect if the input library has its on_disk attribute
+        set to False.
+    """
+    in_memory = not input_models._on_disk
+    ngroups = len(input_models)
+    example_model = None
+
+    with input_models:
+        for i in range(len(input_models)):
+
+            model = input_models.borrow(i)
+            # TODO does this need to be assigned to the model?
+            wht = build_driz_weight(
+                model,
+                # FIXME this was hard-coded to "ivm"
+                weight_type="ivm",
+                good_bits=good_bits,
+            )
+
+            if save_intermediate_results:
+                # write the model to file
+                _fileio.save_drizzled(model, make_output_path)
+
+            if i == 0:
+                input_shape = (ngroups,) + model.data.shape
+                dtype = model.data.dtype
+                computer = MedianComputer(input_shape, in_memory, buffer_size, dtype)
+                example_model = model
+                median_wcs = copy.deepcopy(model.meta.wcs)
+
+            weight_threshold = compute_weight_threshold(wht, maskpt)
+
+            data_copy = model.data.copy()
+            data_copy[wht < weight_threshold] = np.nan
+            computer.append(data_copy, i)
+
+            input_models.shelve(model, i, modify=True)
+            del data_copy, model
+
+    # Perform median combination on set of drizzled mosaics
+    median_data = computer.evaluate()
+
+    if save_intermediate_results:
+        _fileio.save_median(example_model, median_data, median_wcs, make_output_path)
+
+    return median_data, median_wcs
+
+
+def _flag_resampled_model_crs(
+    image,
+    median_data,
+    median_wcs,
+    snr1,
+    snr2,
+    scale1,
+    scale2,
+    backg,
+    save_intermediate_results,
+    make_output_path,
+):
+    blot = gwcs_blot(median_data, median_wcs, image.data.shape, image.meta.wcs, 1.0)
+
+    # Get background level of science data if it has not been subtracted, so it
+    # can be added into the level of the blotted data, which has been
+    # background-subtracted
+    if (
+        hasattr(image.meta, "background")
+        and image.meta.background.subtracted is False
+        and image.meta.background.level is not None
+    ):
+        backg = image.meta.background.level.value
+        log.debug(
+            f"Adding background level {image.meta.background.level} to blotted image"
+        )
+
+    cr_mask = flag_resampled_crs(
+        image.data.value, image.err.value, blot, snr1, snr2, scale1, scale2, backg
+    )
+
+    # update the dq flags in-place
+    image.dq |= cr_mask * np.uint32(pixel.DO_NOT_USE | pixel.OUTLIER)
+    log.info(f"{np.count_nonzero(cr_mask)} pixels marked as outliers")
+
+
+def _flag_model_crs(image, median_data, snr):
+    cr_mask = flag_crs(image.data.value, image.err.value, median_data, snr)
+
+    # Update dq array in-place
+    image.dq |= cr_mask * np.uint32(pixel.DO_NOT_USE | pixel.OUTLIER)
+
+    log.info(f"{np.count_nonzero(cr_mask)} pixels marked as outliers")
+
+
+def detect_outliers(
+    library,
+    weight_type,
+    pixfrac,
+    kernel,
+    fillval,
+    maskpt,
+    snr1,
+    snr2,
+    scale1,
+    scale2,
+    backg,
+    save_intermediate_results,
+    resample_data,
+    good_bits,
+    in_memory,
+    make_output_path,
+):
+    # TODO why was make_output_path overwritten for median?
+
+    # setup ResampleData
+    # call
+    if resample_data:
+        resamp = ResampleData(
+            library,
+            single=True,
+            blendheaders=False,
+            # FIXME prior code provided weight_type when only wht_type is understood
+            # both default to 'ivm' but tests that set this to something else did
+            # not change the resampling weight type. For now, disabling it to match
+            # the previous behavior.
+            # wht_type=weight_type
+            pixfrac=pixfrac,
+            kernel=kernel,
+            fillval=fillval,
+            good_bits=good_bits,
+            in_memory=in_memory,
+        )
+        median_data, median_wcs = _median_with_resampling(
+            library,
+            resamp,
+            maskpt,
+            save_intermediate_results=save_intermediate_results,
+            make_output_path=make_output_path,
+        )
+    else:
+        median_data, median_wcs = _median_without_resampling(
+            library,
+            maskpt,
+            weight_type,
+            good_bits,
+            save_intermediate_results=save_intermediate_results,
+            make_output_path=make_output_path,
+        )
+
+    # Perform outlier detection using statistical comparisons between
+    # each original input image and its blotted version of the median image
+    with library:
+        for image in library:
+            if resample_data:
+                _flag_resampled_model_crs(
+                    image,
+                    median_data,
+                    median_wcs,
+                    snr1,
+                    snr2,
+                    scale1,
+                    scale2,
+                    backg,
+                    save_intermediate_results,
+                    make_output_path,
+                )
+            else:
+                _flag_model_crs(image, median_data, snr1)
+
+            # mark step as complete
+            image.meta.cal_step["outlier_detection"] = "COMPLETE"
+
+            library.shelve(image, modify=True)
+
+    return library
diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py
index 6c0da2173..023dc31ec 100644
--- a/romancal/resample/resample.py
+++ b/romancal/resample/resample.py
@@ -191,6 +191,81 @@ def do_drizzle(self):
         else:
             return self.resample_many_to_one()
 
+    def resample_group(self, input_models, indices):
+        """Apply resample_many_to_many for one group
+
+        Parameters
+        ----------
+        input_models : ModelLibrary
+
+        indices : list
+        """
+        output_model = self.blank_output.copy()
+        output_model.meta["resample"] = maker_utils.mk_resample()
+
+        copy_asn_info_from_library(input_models, output_model)
+
+        with input_models:
+            example_image = input_models.borrow(indices[0])
+
+            # Determine output file type from input exposure filenames
+            # Use this for defining the output filename
+            indx = example_image.meta.filename.rfind(".")
+            output_type = example_image.meta.filename[indx:]
+            output_root = "_".join(
+                example_image.meta.filename.replace(output_type, "").split("_")[:-1]
+            )
+            output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}"
+            input_models.shelve(example_image, indices[0], modify=False)
+            del example_image
+
+            # Initialize the output with the wcs
+            driz = gwcs_drizzle.GWCSDrizzle(
+                output_model,
+                pixfrac=self.pixfrac,
+                kernel=self.kernel,
+                fillval=self.fillval,
+            )
+
+            log.info(f"{len(indices)} exposures to drizzle together")
+            for index in indices:
+                img = input_models.borrow(index)
+                # TODO: should weight_type=None here?
+                inwht = resample_utils.build_driz_weight(
+                    img, weight_type=self.weight_type, good_bits=self.good_bits
+                )
+
+                # apply sky subtraction
+                if (
+                    hasattr(img.meta, "background")
+                    and img.meta.background.subtracted is False
+                    and img.meta.background.level is not None
+                ):
+                    data = img.data - img.meta.background.level
+                else:
+                    data = img.data
+
+                xmin, xmax, ymin, ymax = resample_utils.resample_range(
+                    data.shape, img.meta.wcs.bounding_box
+                )
+
+                driz.add_image(
+                    data,
+                    img.meta.wcs,
+                    inwht=inwht,
+                    xmin=xmin,
+                    xmax=xmax,
+                    ymin=ymin,
+                    ymax=ymax,
+                )
+                del data
+                self.input_models.shelve(img, index, modify=False)
+                del img
+
+        # cast context array to uint32
+        output_model.context = output_model.context.astype("uint32")
+        return output_model
+
     def resample_many_to_many(self):
         """Resample many inputs to many outputs where outputs have a common frame.
 
@@ -200,6 +275,7 @@ def resample_many_to_many(self):
 
         Used for outlier detection
         """
+        # FIXME update to use resample_group
         output_list = []
         for group_id, indices in self.input_models.group_indices.items():
             output_model = self.blank_output
@@ -941,3 +1017,11 @@ def populate_mosaic_individual(
     input_metas = [datamodel.meta for datamodel in input_models]
     for input_meta in input_metas:
         output_model.append_individual_image_meta(input_meta)
+
+
+def copy_asn_info_from_library(input_models, output_model):
+    # copy over asn information
+    if (asn_pool := input_models.asn.get("asn_pool", None)) is not None:
+        output_model.meta.asn.pool_name = asn_pool
+    if (asn_table_name := input_models.asn.get("table_name", None)) is not None:
+        output_model.meta.asn.table_name = asn_table_name

From 4f6a6e90912844abb3214f5078560b5302f8877d Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 30 Sep 2024 16:03:51 -0400
Subject: [PATCH 32/74] remove unused kernel_size from docs

---
 docs/roman/outlier_detection/arguments.rst | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/docs/roman/outlier_detection/arguments.rst b/docs/roman/outlier_detection/arguments.rst
index b2f040d74..d54a17fea 100644
--- a/docs/roman/outlier_detection/arguments.rst
+++ b/docs/roman/outlier_detection/arguments.rst
@@ -82,11 +82,6 @@ behavior of the processing:
   User-specified background value (scalar) to subtract during final identification
   step of outliers in `driz_cr` computation.
 
-``--kernel_size``
-  Size of kernel to be used during resampling of the data
-  (i.e. when `resample_data=True`). Valid values are a pair of ints in a single string
-  (for example "7 7").
-
 ``--save_intermediate_results``
   Specifies whether or not to write out intermediate products such as median image or
   resampled individual input exposures to disk. Typically, only used to track down

From a6a180ad340c823b2355eca75717404ff6c213a4 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 30 Sep 2024 16:22:06 -0400
Subject: [PATCH 33/74] update docs for new memory model

---
 .../outlier_detection/outlier_detection.rst   | 28 ++++++-------------
 .../outlier_detection_step.rst                |  5 ++--
 2 files changed, 10 insertions(+), 23 deletions(-)

diff --git a/docs/roman/outlier_detection/outlier_detection.rst b/docs/roman/outlier_detection/outlier_detection.rst
index 22bfbd4a2..f47309cd6 100644
--- a/docs/roman/outlier_detection/outlier_detection.rst
+++ b/docs/roman/outlier_detection/outlier_detection.rst
@@ -58,12 +58,10 @@ Specifically, this routine performs the following operations:
    * The ``maskpt`` parameter sets the percentage of the weight image values to
      use, and any pixel with a weight below this value gets flagged as "bad" and
      ignored when resampled.
-   * The median image is written out to disk as `_<asn_id>_median` by default.
 
 #. By default, the median image is blotted back (inverse of resampling) to
    match each original input image.
 
-   * Blotted images are written out to disk as `_<asn_id>_blot` by default.
    * **If resampling is turned off**, the median image is compared directly to
      each input image.
 
@@ -131,26 +129,16 @@ memory usage at the expense of file I/O.  The control over this memory model hap
 with the use of the ``in_memory`` parameter.  The full impact of this parameter
 during processing includes:
 
-#. The ``save_open`` parameter gets set to `False`
+#. The ``on_disk`` parameter gets set to `True`
    when opening the input :py:class:`~romancal.datamodels.library.ModelLibrary`
-   object. This forces all input models in the input
-   :py:class:`~romancal.datamodels.library.ModelLibrary` to get written out to disk.
-   It then uses the filename of the input model during subsequent processing.
+   object. This causes modified models to be written to temporary files.
 
-#. The ``in_memory`` parameter gets passed to the :py:class:`~romancal.resample.ResampleStep`
-   to set whether or not to keep the resampled images in memory or not.  By default,
-   the outlier detection processing sets this parameter to `False` so that each resampled
-   image gets written out to disk.
-
-#. Computing the median image works section-by-section by only keeping 1Mb of each input
-   in memory at a time.  As a result, only the final output product array for the final
-   median image along with a stack of 1Mb image sections are kept in memory.
-
-#. The final resampling step also avoids keeping all inputs in memory by only reading
-   each input into memory 1 at a time as it gets resampled onto the final output product.
+#. Computing the median image uses temporary files. Each resampled group
+   is split into sections (1 per "row") and each section is appended to a different
+   temporary file. After resampling all groups, each temporary file is read and a
+   median computed for all sections in that file (yielding a median for that
+   section across all resampled groups). Finally these median sections are
+   combined in a final median image.
 
 These changes result in a minimum amount of memory usage during processing at the obvious
 expense of reading and writing the products from disk.
-
-
-.. automodapi:: romancal.outlier_detection.outlier_detection
diff --git a/docs/roman/outlier_detection/outlier_detection_step.rst b/docs/roman/outlier_detection/outlier_detection_step.rst
index 0be906290..353bca28d 100644
--- a/docs/roman/outlier_detection/outlier_detection_step.rst
+++ b/docs/roman/outlier_detection/outlier_detection_step.rst
@@ -4,9 +4,8 @@ OutlierDetectionStep
 --------------------
 
 This module provides the sole interface to all methods of performing outlier detection
-on Roman observations. The outlier detection algorithm used for WFI data is implemented
-in :py:class:`~romancal.outlier_detection.outlier_detection.OutlierDetection`
-and described in :ref:`outlier-detection-imaging`.
+on Roman observations. The outlier detection algorithm used for WFI data is
+described in :ref:`outlier-detection-imaging`.
 
 .. note::
     Whether the data are being provided in an `association file`_ or as a list of ASDF filenames,

From 3d31901ba1cdb26f57e33474bac3028ff1e52186 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 30 Sep 2024 17:34:43 -0400
Subject: [PATCH 34/74] use resample_group in resample_many_to_many

---
 romancal/resample/resample.py | 116 ++++++++--------------------------
 1 file changed, 28 insertions(+), 88 deletions(-)

diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py
index 023dc31ec..2f6f48f59 100644
--- a/romancal/resample/resample.py
+++ b/romancal/resample/resample.py
@@ -1,4 +1,6 @@
+import json
 import logging
+import os
 from typing import List
 
 import numpy as np
@@ -8,6 +10,8 @@
 from roman_datamodels import datamodels, maker_utils, stnode
 from stcal.alignment.util import compute_scale
 
+from romancal.associations.asn_from_list import asn_from_list
+
 from ..assign_wcs import utils
 from ..datamodels import ModelLibrary
 from . import gwcs_drizzle, resample_utils
@@ -272,96 +276,32 @@ def resample_many_to_many(self):
         Coadd only different detectors of the same exposure (e.g. map SCA 1 and
         10 onto the same output image), as they image different areas of the
         sky.
-
-        Used for outlier detection
         """
-        # FIXME update to use resample_group
-        output_list = []
+        output_models = []
         for group_id, indices in self.input_models.group_indices.items():
-            output_model = self.blank_output
-            output_model.meta["resample"] = maker_utils.mk_resample()
-
-            # copy over asn information
-            if (asn_pool := self.input_models.asn.get("asn_pool", None)) is not None:
-                output_model.meta.asn.pool_name = asn_pool
-            if (
-                asn_table_name := self.input_models.asn.get("table_name", None)
-            ) is not None:
-                output_model.meta.asn.table_name = asn_table_name
-
-            with self.input_models:
-                example_image = self.input_models.borrow(indices[0])
-
-                # Determine output file type from input exposure filenames
-                # Use this for defining the output filename
-                indx = example_image.meta.filename.rfind(".")
-                output_type = example_image.meta.filename[indx:]
-                output_root = "_".join(
-                    example_image.meta.filename.replace(output_type, "").split("_")[:-1]
-                )
-                output_model.meta.filename = f"{output_root}_outlier_i2d{output_type}"
-
-                self.input_models.shelve(example_image, indices[0], modify=False)
-
-                # Initialize the output with the wcs
-                driz = gwcs_drizzle.GWCSDrizzle(
-                    output_model,
-                    pixfrac=self.pixfrac,
-                    kernel=self.kernel,
-                    fillval=self.fillval,
-                )
-
-                log.info(f"{len(indices)} exposures to drizzle together")
-                for index in indices:
-                    img = self.input_models.borrow(index)
-                    # TODO: should weight_type=None here?
-                    inwht = resample_utils.build_driz_weight(
-                        img, weight_type=self.weight_type, good_bits=self.good_bits
-                    )
-
-                    # apply sky subtraction
-                    if (
-                        hasattr(img.meta, "background")
-                        and img.meta.background.subtracted is False
-                        and img.meta.background.level is not None
-                    ):
-                        data = img.data - img.meta.background.level
-                    else:
-                        data = img.data
-
-                    xmin, xmax, ymin, ymax = resample_utils.resample_range(
-                        data.shape, img.meta.wcs.bounding_box
-                    )
-
-                    driz.add_image(
-                        data,
-                        img.meta.wcs,
-                        inwht=inwht,
-                        xmin=xmin,
-                        xmax=xmax,
-                        ymin=ymin,
-                        ymax=ymax,
-                    )
-                    del data
-                    self.input_models.shelve(img, index)
-
-                # cast context array to uint32
-                output_model.context = output_model.context.astype("uint32")
-
-                # copy over asn information
-                if not self.in_memory:
-                    # Write out model to disk, then return filename
-                    output_name = output_model.meta.filename
-                    output_model.save(output_name)
-                    log.info(f"Exposure {output_name} saved to file")
-                    output_list.append(output_name)
-                else:
-                    output_list.append(output_model.copy())
-
-                output_model.data *= 0.0
-                output_model.weight *= 0.0
-
-        return ModelLibrary(output_list)
+            output_model = self.resample_group(self.input_models, indices)
+
+            if not self.in_memory:
+                # Write out model to disk, then return filename
+                output_name = output_model.meta.filename
+                if self.output_dir is not None:
+                    output_name = os.path.join(self.output_dir, output_name)
+                output_model.save(output_name)
+                log.info(f"Saved model in {output_name}")
+                output_models.append(output_name)
+            else:
+                output_models.append(output_model)
+
+        if not self.in_memory:
+            # build ModelLibrary as an association from the output files
+            # this saves memory if there are multiple groups
+            asn = asn_from_list(output_models, product_name="outlier_i2d")
+            asn_dict = json.loads(
+                asn.dump()[1]
+            )  # serializes the asn and converts to dict
+            return ModelLibrary(asn_dict, on_disk=True)
+        # otherwise just build it as a list of in-memory models
+        return ModelLibrary(output_models, on_disk=False)
 
     def resample_many_to_one(self):
         """Resample and coadd many inputs to a single output.

From 1f159fafd82239a9090f6f7510c1ea5d01349aa1 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 30 Sep 2024 17:53:11 -0400
Subject: [PATCH 35/74] unskip and fix test

---
 romancal/outlier_detection/_fileio.py         |  1 +
 .../tests/test_outlier_detection.py           | 41 ++++---------------
 2 files changed, 10 insertions(+), 32 deletions(-)

diff --git a/romancal/outlier_detection/_fileio.py b/romancal/outlier_detection/_fileio.py
index 3cf13d933..d1dde8489 100644
--- a/romancal/outlier_detection/_fileio.py
+++ b/romancal/outlier_detection/_fileio.py
@@ -21,6 +21,7 @@ def save_drizzled(drizzled_model, make_output_path):
 def _make_median_model(example_model, data, wcs):
     model = example_model.copy()
     model.data = Quantity(data, unit=model.data.unit)
+    model.meta.filename = "drizzled_median.asdf"
     model.meta.wcs = wcs
     return model
 
diff --git a/romancal/outlier_detection/tests/test_outlier_detection.py b/romancal/outlier_detection/tests/test_outlier_detection.py
index 94f137b78..d20aa7e77 100644
--- a/romancal/outlier_detection/tests/test_outlier_detection.py
+++ b/romancal/outlier_detection/tests/test_outlier_detection.py
@@ -98,46 +98,26 @@ def test_outlier_valid_input_modelcontainer(tmp_path, base_image):
             res.shelve(m, i, modify=False)
 
 
-@pytest.mark.skip(
-    reason="This creates a step then calls an internal class that no longer exists"
-)
 def test_outlier_do_detection_write_files_to_custom_location(tmp_path, base_image):
     """
     Test that OutlierDetection can create files on disk in a custom location.
     """
     img_1 = base_image()
-    img_1.meta.filename = "img_1.asdf"
+    img_1.meta.filename = "img1_cal.asdf"
+    img_1.meta.background.level = 0 * u.DN / u.s
     img_2 = base_image()
-    img_2.meta.filename = "img_2.asdf"
+    img_2.meta.filename = "img2_cal.asdf"
+    img_2.meta.background.level = 0 * u.DN / u.s
     input_models = ModelLibrary([img_1, img_2])
 
-    outlier_step = OutlierDetectionStep()
+    outlier_step = OutlierDetectionStep(
+        in_memory=False,
+        save_intermediate_results=True,
+    )
     # set output dir for all files created by the step
     outlier_step.output_dir = tmp_path.as_posix()
-    # make sure files are written out to disk
-    outlier_step.in_memory = False
 
-    pars = {
-        "weight_type": "exptime",
-        "pixfrac": 1.0,
-        "kernel": "square",
-        "fillval": "INDEF",
-        "nlow": 0,
-        "nhigh": 0,
-        "maskpt": 0.7,
-        "grow": 1,
-        "snr": "4.0 3.0",
-        "scale": "0.5 0.4",
-        "backg": 0.0,
-        "kernel_size": "7 7",
-        "save_intermediate_results": True,
-        "resample_data": False,
-        "good_bits": 0,
-        "allowed_memory": None,
-        "in_memory": outlier_step.in_memory,
-        "make_output_path": outlier_step.make_output_path,
-        "resample_suffix": "i2d",
-    }
+    outlier_step(input_models)
 
     # meta.filename for the median image created by OutlierDetection.do_detection()
     median_path = tmp_path / "drizzled_median.asdf"
@@ -146,9 +126,6 @@ def test_outlier_do_detection_write_files_to_custom_location(tmp_path, base_imag
         median_path,
     ]
 
-    step = outlier_detection.OutlierDetection(input_models, **pars)  # noqa: F821
-    step.do_detection()
-
     assert all(x.exists() for x in outlier_files_path)
 
 

From dc7dd0953e870631883356d8f11656f34b98e304 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Mon, 30 Sep 2024 17:54:46 -0400
Subject: [PATCH 36/74] remove outdated comments

---
 romancal/outlier_detection/utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/romancal/outlier_detection/utils.py b/romancal/outlier_detection/utils.py
index 8e993a925..d1ccec1e2 100644
--- a/romancal/outlier_detection/utils.py
+++ b/romancal/outlier_detection/utils.py
@@ -152,7 +152,6 @@ def _median_without_resampling(
         for i in range(len(input_models)):
 
             model = input_models.borrow(i)
-            # TODO does this need to be assigned to the model?
             wht = build_driz_weight(
                 model,
                 # FIXME this was hard-coded to "ivm"
@@ -252,8 +251,6 @@ def detect_outliers(
     in_memory,
     make_output_path,
 ):
-    # TODO why was make_output_path overwritten for median?
-
     # setup ResampleData
     # call
     if resample_data:

From 35c87732b8ee52ad0aae0c6f146e3884938781d1 Mon Sep 17 00:00:00 2001
From: Brett Graham <brettgraham@gmail.com>
Date: Tue, 1 Oct 2024 11:36:50 -0400
Subject: [PATCH 37/74] Update docs from review.

Co-authored-by: Ned Molter <emolter@users.noreply.github.com>
---
 docs/roman/outlier_detection/arguments.rst         | 6 +++---
 docs/roman/outlier_detection/outlier_detection.rst | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/roman/outlier_detection/arguments.rst b/docs/roman/outlier_detection/arguments.rst
index d54a17fea..3491eb74e 100644
--- a/docs/roman/outlier_detection/arguments.rst
+++ b/docs/roman/outlier_detection/arguments.rst
@@ -83,12 +83,12 @@ behavior of the processing:
   step of outliers in `driz_cr` computation.
 
 ``--save_intermediate_results``
-  Specifies whether or not to write out intermediate products such as median image or
+  Boolean specifying whether or not to write out intermediate products such as median image or
   resampled individual input exposures to disk. Typically, only used to track down
   problems with final results when too many or too few pixels are flagged as outliers.
 
 ``--resample_data``
-  Specifies whether or not to resample the input images when performing outlier
+  Boolean specifying whether or not to resample the input images when performing outlier
   detection.
 
 ``--good_bits``
@@ -99,7 +99,7 @@ behavior of the processing:
   for details.
 
 ``--in_memory``
-  Specifies whether or not to keep all intermediate products and datamodels in
+  Boolean specifying whether or not to keep all intermediate products and datamodels in
   memory at the same time during the processing of this step.  If set to `False`,
   all input and output data will be written to disk at the start of the step
   (as much as `roman_datamodels` will allow, anyway), then read in to memory only when
diff --git a/docs/roman/outlier_detection/outlier_detection.rst b/docs/roman/outlier_detection/outlier_detection.rst
index f47309cd6..7fee83327 100644
--- a/docs/roman/outlier_detection/outlier_detection.rst
+++ b/docs/roman/outlier_detection/outlier_detection.rst
@@ -136,9 +136,9 @@ during processing includes:
 #. Computing the median image uses temporary files. Each resampled group
    is split into sections (1 per "row") and each section is appended to a different
    temporary file. After resampling all groups, each temporary file is read and a
-   median computed for all sections in that file (yielding a median for that
-   section across all resampled groups). Finally these median sections are
-   combined in a final median image.
+   median is computed for all sections in that file (yielding a median for that
+   section across all resampled groups). Finally, these median sections are
+   combined into a final median image.
 
 These changes result in a minimum amount of memory usage during processing at the obvious
 expense of reading and writing the products from disk.

From a6bbac5aced56a22e78e189d3957ba63c7c334fe Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Tue, 1 Oct 2024 11:53:00 -0400
Subject: [PATCH 38/74] clarify in_memory

---
 docs/roman/outlier_detection/arguments.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/roman/outlier_detection/arguments.rst b/docs/roman/outlier_detection/arguments.rst
index 3491eb74e..db90f33d4 100644
--- a/docs/roman/outlier_detection/arguments.rst
+++ b/docs/roman/outlier_detection/arguments.rst
@@ -101,10 +101,10 @@ behavior of the processing:
 ``--in_memory``
   Boolean specifying whether or not to keep all intermediate products and datamodels in
   memory at the same time during the processing of this step.  If set to `False`,
-  all input and output data will be written to disk at the start of the step
-  (as much as `roman_datamodels` will allow, anyway), then read in to memory only when
-  accessed.  This results in a much lower memory profile at the expense of file I/O,
-  which can allow large mosaics to process in more limited amounts of memory.
+  any `ModelLibrary` opened by this step will use ``on_disk=True` and use temporary
+  files to store model modifications. Additionally any resampled images will
+  be kept in memory (as long as needed). This can result in much lower memory
+  usage (at the expense of file I/O) to process large associations.
 
 .. _weight_type_options_details_section:
 

From d290ed4020bd8c3a81545dc17cef3a1ecf8a7420 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Tue, 1 Oct 2024 11:55:53 -0400
Subject: [PATCH 39/74] fix test docstrings

---
 romancal/outlier_detection/tests/test_outlier_detection.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/romancal/outlier_detection/tests/test_outlier_detection.py b/romancal/outlier_detection/tests/test_outlier_detection.py
index d20aa7e77..c859a1d52 100644
--- a/romancal/outlier_detection/tests/test_outlier_detection.py
+++ b/romancal/outlier_detection/tests/test_outlier_detection.py
@@ -16,7 +16,7 @@
     ],
 )
 def test_outlier_raises_error_on_invalid_input_models(input_models):
-    """Test that OutlierDetection logs out a WARNING if input is invalid."""
+    """Test that OutlierDetection raises an Exception if input is invalid."""
 
     with pytest.raises(IsADirectoryError):
         OutlierDetectionStep.call(input_models)
@@ -37,7 +37,7 @@ def test_outlier_skips_step_on_invalid_number_of_elements_in_input(base_image):
 
 def test_outlier_raises_exception_on_exposure_type_different_from_wfi_image(base_image):
     """
-    Test if the outlier detection step is skipped when the exposure type is different from WFI image.
+    Test if the outlier detection step raises an Exception for non-image inputs.
     """
     img_1 = base_image()
     img_1.meta.exposure.type = "WFI_PRISM"

From 6c960ab04078b08750a6c323a1e7f7572037becc Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Tue, 1 Oct 2024 12:00:33 -0400
Subject: [PATCH 40/74] fix make_output_path docstring entry, add description
 of default buffer_size

---
 romancal/outlier_detection/utils.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/romancal/outlier_detection/utils.py b/romancal/outlier_detection/utils.py
index d1ccec1e2..56d3bbcce 100644
--- a/romancal/outlier_detection/utils.py
+++ b/romancal/outlier_detection/utils.py
@@ -50,13 +50,13 @@ def _median_with_resampling(
         if True, save the drizzled models and median model to fits.
 
     make_output_path : function
-        The functools.partial instance to pass to save_median. Must be
-        specified if save_intermediate_results is True. Default None.
+        The functools.partial instance to pass to save_median.
 
     buffer_size : int
         The size of chunk in bytes that will be read into memory when computing the median.
         This parameter has no effect if the input library has its on_disk attribute
-        set to False.
+        set to False. If None or 0 the buffer size will be set to the size of one
+        resampled image.
     """
     if not resamp.single:
         raise ValueError(
@@ -136,13 +136,14 @@ def _median_without_resampling(
         if True, save the models and median model to fits.
 
     make_output_path : function
-        The functools.partial instance to pass to save_median. Must be
-        specified if save_intermediate_results is True. Default None.
+        The functools.partial instance to pass to save_median.
 
     buffer_size : int
         The size of chunk in bytes that will be read into memory when computing the median.
         This parameter has no effect if the input library has its on_disk attribute
-        set to False.
+        set to False. If None or 0 the buffer size will be set to the size of one
+        input image.
+
     """
     in_memory = not input_models._on_disk
     ngroups = len(input_models)

From e81ff486917924422c459ae7f3be98e3c712b596 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Tue, 1 Oct 2024 12:08:02 -0400
Subject: [PATCH 41/74] expand resample_group docstring

---
 romancal/resample/resample.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/romancal/resample/resample.py b/romancal/resample/resample.py
index 2f6f48f59..d2a9b0b2e 100644
--- a/romancal/resample/resample.py
+++ b/romancal/resample/resample.py
@@ -196,13 +196,18 @@ def do_drizzle(self):
             return self.resample_many_to_one()
 
     def resample_group(self, input_models, indices):
-        """Apply resample_many_to_many for one group
+        """
+        Resample models at the provided indices.
+
+        This is used by outlier detection and will not blend metadata
+        and will not resample variance, error or exposure time.
 
         Parameters
         ----------
         input_models : ModelLibrary
 
         indices : list
+            List of model indices to include in this resampling
         """
         output_model = self.blank_output.copy()
         output_model.meta["resample"] = maker_utils.mk_resample()

From d99682f2b4b8a550a70fe8303cdee41380446386 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Tue, 1 Oct 2024 12:56:18 -0400
Subject: [PATCH 42/74] docs typo

---
 docs/roman/outlier_detection/arguments.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/roman/outlier_detection/arguments.rst b/docs/roman/outlier_detection/arguments.rst
index db90f33d4..0f8e6446f 100644
--- a/docs/roman/outlier_detection/arguments.rst
+++ b/docs/roman/outlier_detection/arguments.rst
@@ -101,7 +101,7 @@ behavior of the processing:
 ``--in_memory``
   Boolean specifying whether or not to keep all intermediate products and datamodels in
   memory at the same time during the processing of this step.  If set to `False`,
-  any `ModelLibrary` opened by this step will use ``on_disk=True` and use temporary
+  any `ModelLibrary` opened by this step will use ``on_disk=True`` and use temporary
   files to store model modifications. Additionally any resampled images will
   be kept in memory (as long as needed). This can result in much lower memory
   usage (at the expense of file I/O) to process large associations.

From 9c2a33ff555a529bbaf3698fba49346b60774da8 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Thu, 3 Oct 2024 14:13:00 -0400
Subject: [PATCH 43/74] add failing unit test

---
 .../tests/test_outlier_detection.py           | 25 ++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/romancal/outlier_detection/tests/test_outlier_detection.py b/romancal/outlier_detection/tests/test_outlier_detection.py
index c859a1d52..0069fde88 100644
--- a/romancal/outlier_detection/tests/test_outlier_detection.py
+++ b/romancal/outlier_detection/tests/test_outlier_detection.py
@@ -129,7 +129,8 @@ def test_outlier_do_detection_write_files_to_custom_location(tmp_path, base_imag
     assert all(x.exists() for x in outlier_files_path)
 
 
-def test_find_outliers(tmp_path, base_image):
+@pytest.mark.parametrize("on_disk", (True, False))
+def test_find_outliers(tmp_path, base_image, on_disk):
     """
     Test that OutlierDetection can find outliers.
     """
@@ -158,13 +159,31 @@ def test_find_outliers(tmp_path, base_image):
     imgs[0].data[img_0_input_coords[0], img_0_input_coords[1]] = cr_value
     imgs[1].data[img_1_input_coords[0], img_1_input_coords[1]] = cr_value
 
-    input_models = ModelLibrary(imgs)
+    if on_disk:
+        # write out models and asn to disk
+        for img in imgs:
+            img.save(img.meta.filename)
+        asn_dict = {
+            "asn_id": "a3001",
+            "asn_pool": "none",
+            "products": [
+                {
+                    "name": "test_asn",
+                    "members": [
+                        {"expname": m.meta.filename, "exptype": "science"} for m in imgs
+                    ],
+                }
+            ],
+        }
+        input_models = ModelLibrary(asn_dict, on_disk=True)
+    else:
+        input_models = ModelLibrary(imgs)
 
     outlier_step = OutlierDetectionStep()
     # set output dir for all files created by the step
     outlier_step.output_dir = tmp_path.as_posix()
     # make sure files are written out to disk
-    outlier_step.in_memory = False
+    outlier_step.in_memory = not on_disk
 
     result = outlier_step(input_models)
 

From ae00da243135c24b46252b6f41af14130015e7f3 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Thu, 3 Oct 2024 14:14:42 -0400
Subject: [PATCH 44/74] work around NotImplemented Quantity.tofile

---
 romancal/outlier_detection/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/romancal/outlier_detection/utils.py b/romancal/outlier_detection/utils.py
index 56d3bbcce..6e529d052 100644
--- a/romancal/outlier_detection/utils.py
+++ b/romancal/outlier_detection/utils.py
@@ -86,7 +86,7 @@ def _median_with_resampling(
 
             weight_threshold = compute_weight_threshold(drizzled_model.weight, maskpt)
             drizzled_model.data[drizzled_model.weight < weight_threshold] = np.nan
-            computer.append(drizzled_model.data, i)
+            computer.append(drizzled_model.data.value, i)
             del drizzled_model
 
     # Perform median combination on set of drizzled mosaics
@@ -173,7 +173,7 @@ def _median_without_resampling(
 
             weight_threshold = compute_weight_threshold(wht, maskpt)
 
-            data_copy = model.data.copy()
+            data_copy = model.data.value.copy()
             data_copy[wht < weight_threshold] = np.nan
             computer.append(data_copy, i)
 

From 50b439aee5c212117d64e51bff6331d00d5ab281 Mon Sep 17 00:00:00 2001
From: Brett <brettgraham@gmail.com>
Date: Thu, 3 Oct 2024 14:24:37 -0400
Subject: [PATCH 45/74] add changelog fragment

---
 changes/1436.outlier_detection.rst | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/1436.outlier_detection.rst

diff --git a/changes/1436.outlier_detection.rst b/changes/1436.outlier_detection.rst
new file mode 100644
index 000000000..045081ea0
--- /dev/null
+++ b/changes/1436.outlier_detection.rst
@@ -0,0 +1 @@
+Fix bug where on_disk=True could fail due to Quantities not implementing tofile.

From 4abdef1910006bda46f3ee39f19fbdab3c28c345 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 7 Oct 2024 16:21:58 +0000
Subject: [PATCH 46/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/flux/flux_step.py            |  2 +-
 romancal/regtest/test_mos_pipeline.py | 11 +++++------
 romancal/source_catalog/detection.py  |  1 -
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/romancal/flux/flux_step.py b/romancal/flux/flux_step.py
index 2fdf0647a..6fe3a770f 100644
--- a/romancal/flux/flux_step.py
+++ b/romancal/flux/flux_step.py
@@ -106,7 +106,7 @@ def apply_flux_correction(model):
 
     if model.meta.cal_step["flux"] == "COMPLETE":
         message = (
-            f"Input data is already in flux units of MJy/sr."
+            "Input data is already in flux units of MJy/sr."
             "\nFlux correction already applied."
         )
         log.info("Flux correction already applied.")
diff --git a/romancal/regtest/test_mos_pipeline.py b/romancal/regtest/test_mos_pipeline.py
index 62e0149e2..10a805b0a 100644
--- a/romancal/regtest/test_mos_pipeline.py
+++ b/romancal/regtest/test_mos_pipeline.py
@@ -1,20 +1,19 @@
 """ Roman tests for the High Level Pipeline """
 
+import json
 import os
+from pathlib import Path
 
+import asdf
 import pytest
 import roman_datamodels as rdm
+from astropy.units import Quantity
 
 from romancal.associations.asn_from_list import asn_from_list
 from romancal.pipeline.mosaic_pipeline import MosaicPipeline
 
-from .regtestdata import compare_asdf
-from pathlib import Path
-from astropy.units import Quantity
-import asdf
-
 from ..associations.association_io import json as asn_json
-import json
+from .regtestdata import compare_asdf
 
 
 def passfail(bool_expr):
diff --git a/romancal/source_catalog/detection.py b/romancal/source_catalog/detection.py
index febc984cc..c86982bee 100644
--- a/romancal/source_catalog/detection.py
+++ b/romancal/source_catalog/detection.py
@@ -7,7 +7,6 @@
 import warnings
 
 from astropy.convolution import convolve
-from astropy.units import Quantity
 from astropy.utils.exceptions import AstropyUserWarning
 from photutils.segmentation import SourceFinder, make_2dgaussian_kernel
 from photutils.utils.exceptions import NoDetectionsWarning

From 83eae2820d00ddc94c8e02018e061ba0d6ee6a98 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Mon, 7 Oct 2024 12:28:12 -0400
Subject: [PATCH 47/74] Check-style fixes.

---
 romancal/flux/flux_step.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/romancal/flux/flux_step.py b/romancal/flux/flux_step.py
index 6fe3a770f..c1f333515 100644
--- a/romancal/flux/flux_step.py
+++ b/romancal/flux/flux_step.py
@@ -2,7 +2,6 @@
 
 import logging
 
-import astropy.units as u
 from roman_datamodels import datamodels
 
 from ..datamodels import ModelLibrary
@@ -14,10 +13,6 @@
 __all__ = ["FluxStep"]
 
 
-# Define expected Level 2 units
-LV2_UNITS = u.DN / u.s
-
-
 class FluxStep(RomanStep):
     """Apply flux scaling to count-rate data
 
@@ -109,7 +104,7 @@ def apply_flux_correction(model):
             "Input data is already in flux units of MJy/sr."
             "\nFlux correction already applied."
         )
-        log.info("Flux correction already applied.")
+        log.info(message)
         return
 
     # Apply the correction.

From e704298820f3ccc45971bbc7283b1764f684b9e6 Mon Sep 17 00:00:00 2001
From: Eddie Schlafly <eschlafly@stsci.edu>
Date: Mon, 7 Oct 2024 16:28:59 -0400
Subject: [PATCH 48/74] Begin removing units from ELP.

---
 romancal/dark_current/dark_current_step.py    |  2 +-
 romancal/dark_current/tests/test_dark.py      |  8 +-
 romancal/dq_init/tests/test_dq_init.py        | 18 ++--
 romancal/flatfield/flat_field.py              |  7 +-
 romancal/flatfield/tests/test_flatfield.py    | 20 ++---
 romancal/linearity/linearity_step.py          |  6 +-
 romancal/linearity/tests/test_linearity.py    |  4 +-
 romancal/ramp_fitting/ramp_fit_step.py        | 16 ++--
 .../ramp_fitting/tests/test_ramp_fit_cas22.py |  4 +-
 romancal/refpix/data.py                       | 33 ++-----
 romancal/refpix/tests/conftest.py             |  8 +-
 romancal/refpix/tests/test_data.py            | 22 ++---
 romancal/refpix/tests/test_refpix.py          |  6 +-
 romancal/refpix/tests/test_step.py            | 12 +--
 romancal/saturation/saturation.py             |  2 +-
 romancal/saturation/tests/test_saturation.py  | 88 +++++++++----------
 16 files changed, 91 insertions(+), 165 deletions(-)

diff --git a/romancal/dark_current/dark_current_step.py b/romancal/dark_current/dark_current_step.py
index a8a72658e..ecad8b93b 100644
--- a/romancal/dark_current/dark_current_step.py
+++ b/romancal/dark_current/dark_current_step.py
@@ -54,7 +54,7 @@ def process(self, input):
             # Do the dark correction
             out_model = input_model
             nresultants = len(input_model.meta.exposure["read_pattern"])
-            out_model.data -= dark_model.data[:nresultants]
+            out_model.data -= dark_model.data[:nresultants].value
             out_model.pixeldq |= dark_model.dq
             out_model.meta.cal_step.dark = "COMPLETE"
 
diff --git a/romancal/dark_current/tests/test_dark.py b/romancal/dark_current/tests/test_dark.py
index 5c2ce3d21..7f03a045a 100644
--- a/romancal/dark_current/tests/test_dark.py
+++ b/romancal/dark_current/tests/test_dark.py
@@ -60,7 +60,7 @@ def test_dark_step_subtraction(instrument, exptype):
 
     # populate data array of science cube
     for i in range(0, 20):
-        ramp_model.data[0, 0, i] = i * ramp_model.data.unit
+        ramp_model.data[0, 0, i] = i
         darkref_model.data[0, 0, i] = i * 0.1 * darkref_model.data.unit
     orig_model = ramp_model.copy()
 
@@ -68,12 +68,12 @@ def test_dark_step_subtraction(instrument, exptype):
     result = DarkCurrentStep.call(ramp_model, override_dark=darkref_model)
 
     # check that the dark file is subtracted frame by frame from the science data
-    diff = orig_model.data.value - darkref_model.data.value
+    diff = orig_model.data - darkref_model.data.value
 
     # test that the output data file is equal to the difference found when subtracting
     # reffile from sci file
     np.testing.assert_array_equal(
-        result.data.value, diff, err_msg="dark file should be subtracted from sci file "
+        result.data, diff, err_msg="dark file should be subtracted from sci file "
     )
 
 
@@ -148,7 +148,7 @@ def create_ramp_and_dark(shape, instrument, exptype):
     ramp.meta.instrument.detector = "WFI01"
     ramp.meta.instrument.optical_element = "F158"
     ramp.meta.exposure.type = exptype
-    ramp.data = u.Quantity(np.ones(shape, dtype=np.float32), u.DN, dtype=np.float32)
+    ramp.data = np.ones(shape, dtype=np.float32)
     ramp_model = RampModel(ramp)
 
     # Create dark model
diff --git a/romancal/dq_init/tests/test_dq_init.py b/romancal/dq_init/tests/test_dq_init.py
index 87ac2476e..2b24f8220 100644
--- a/romancal/dq_init/tests/test_dq_init.py
+++ b/romancal/dq_init/tests/test_dq_init.py
@@ -199,9 +199,7 @@ def test_dqinit_step_interface(instrument, exptype):
     wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
     wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
-    wfi_sci_raw.data = u.Quantity(
-        np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16
-    )
+    wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw[extra_key] = extra_value
     wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw)
 
@@ -251,9 +249,7 @@ def test_dqinit_refpix(instrument, exptype):
     wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
     wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
-    wfi_sci_raw.data = u.Quantity(
-        np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16
-    )
+    wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw)
 
     # Create mask model
@@ -272,7 +268,7 @@ def test_dqinit_refpix(instrument, exptype):
 
     # check if reference pixels are correct
     assert result.data.shape == (2, 20, 20)  # no pixels should be trimmed
-    assert result.amp33.value.shape == (2, 4096, 128)
+    assert result.amp33.shape == (2, 4096, 128)
     assert result.border_ref_pix_right.shape == (2, 20, 4)
     assert result.border_ref_pix_left.shape == (2, 20, 4)
     assert result.border_ref_pix_top.shape == (2, 4, 20)
@@ -304,9 +300,7 @@ def test_dqinit_resultantdq(instrument, exptype):
     wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
     wfi_sci_raw.resultantdq[1, 12, 12] = pixel["DROPOUT"]
-    wfi_sci_raw.data = u.Quantity(
-        np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16
-    )
+    wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw)
 
     # Create mask model
@@ -354,9 +348,7 @@ def test_dqinit_getbestref(instrument, exptype):
     wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
     wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
-    wfi_sci_raw.data = u.Quantity(
-        np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16
-    )
+    wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw)
 
     # Perform Data Quality application step
diff --git a/romancal/flatfield/flat_field.py b/romancal/flatfield/flat_field.py
index dd489a926..b74475274 100644
--- a/romancal/flatfield/flat_field.py
+++ b/romancal/flatfield/flat_field.py
@@ -5,7 +5,6 @@
 import logging
 
 import numpy as np
-from astropy import units as u
 from roman_datamodels.dqflags import pixel
 
 log = logging.getLogger(__name__)
@@ -105,9 +104,7 @@ def apply_flat_field(science, flat):
     flat_data[np.where(flat_bad)] = 1.0
     # Now let's apply the correction to science data and error arrays.  Rely
     # on array broadcasting to handle the cubes
-    science.data = u.Quantity(
-        (science.data.value / flat_data), u.DN / u.s, dtype=science.data.dtype
-    )
+    science.data = (science.data / flat_data).astype(science.data.dtype)
 
     # Update the variances using BASELINE algorithm.  For guider data, it has
     # not gone through ramp fitting so there is no Poisson noise or readnoise
@@ -121,8 +118,6 @@ def apply_flat_field(science, flat):
         science.var_flat = science.data**2 / flat_data_squared * flat_err**2
 
     science.err = np.sqrt(science.var_poisson + science.var_rnoise + science.var_flat)
-    science.err = science.err.to(science.data.unit)
-    # Workaround for https://github.com/astropy/astropy/issues/16055
 
     # Combine the science and flat DQ arrays
     science.dq = np.bitwise_or(science.dq, flat_dq)
diff --git a/romancal/flatfield/tests/test_flatfield.py b/romancal/flatfield/tests/test_flatfield.py
index 65d4adab8..964aff33a 100644
--- a/romancal/flatfield/tests/test_flatfield.py
+++ b/romancal/flatfield/tests/test_flatfield.py
@@ -26,22 +26,12 @@ def test_flatfield_step_interface(instrument, exptype):
     wfi_image.meta.instrument.detector = "WFI01"
     wfi_image.meta.instrument.optical_element = "F158"
     wfi_image.meta.exposure.type = exptype
-    wfi_image.data = u.Quantity(
-        np.ones(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32
-    )
+    wfi_image.data =  np.ones(shape, dtype=np.float32)
     wfi_image.dq = np.zeros(shape, dtype=np.uint32)
-    wfi_image.err = u.Quantity(
-        np.zeros(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32
-    )
-    wfi_image.var_poisson = u.Quantity(
-        np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32
-    )
-    wfi_image.var_rnoise = u.Quantity(
-        np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32
-    )
-    wfi_image.var_flat = u.Quantity(
-        np.zeros(shape, dtype=np.float32), u.DN**2 / u.s**2, dtype=np.float32
-    )
+    wfi_image.err = np.zeros(shape, dtype=np.float32)
+    wfi_image.var_poisson = np.zeros(shape, dtype=np.float32)
+    wfi_image.var_rnoise = np.zeros(shape, dtype=np.float32)
+    wfi_image.var_flat = np.zeros(shape, dtype=np.float32)
 
     wfi_image_model = ImageModel(wfi_image)
     flatref = stnode.FlatRef()
diff --git a/romancal/linearity/linearity_step.py b/romancal/linearity/linearity_step.py
index 5320ad8e3..e78dc980f 100644
--- a/romancal/linearity/linearity_step.py
+++ b/romancal/linearity/linearity_step.py
@@ -56,12 +56,10 @@ def process(self, input):
             # The third return value is the procesed zero frame which
             # Roman does not use.
             new_data, new_pdq, _ = linearity_correction(
-                input_model.data.value, gdq, pdq, lin_coeffs, lin_dq, pixel
+                input_model.data, gdq, pdq, lin_coeffs, lin_dq, pixel
             )
 
-            input_model.data = u.Quantity(
-                new_data[0, :, :, :], u.DN, dtype=new_data.dtype
-            )
+            input_model.data = new_data[0, :, :, :]
             input_model.pixeldq = new_pdq
 
         # Update the step status
diff --git a/romancal/linearity/tests/test_linearity.py b/romancal/linearity/tests/test_linearity.py
index b137069fd..ab11fb2d2 100644
--- a/romancal/linearity/tests/test_linearity.py
+++ b/romancal/linearity/tests/test_linearity.py
@@ -35,9 +35,7 @@ def test_linearity_coeff(instrument, exptype):
     wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
     wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = exptype
-    wfi_sci_raw.data = u.Quantity(
-        np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16
-    )
+    wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw)
 
     result = DQInitStep.call(wfi_sci_raw_model)
diff --git a/romancal/ramp_fitting/ramp_fit_step.py b/romancal/ramp_fitting/ramp_fit_step.py
index 0447cfbcc..790bc5098 100644
--- a/romancal/ramp_fitting/ramp_fit_step.py
+++ b/romancal/ramp_fitting/ramp_fit_step.py
@@ -101,7 +101,7 @@ def ols_cas22(self, input_model, readnoise_model, gain_model, dark_model):
         if self.threshold_constant is not None:
             kwargs["threshold_constant"] = self.threshold_constant
 
-        resultants = input_model.data.value
+        resultants = input_model.data
         dq = input_model.groupdq
         read_noise = readnoise_model.data.value
         gain = gain_model.data.value
@@ -183,10 +183,6 @@ def create_image_model(input_model, image_info):
     """
     data, dq, var_poisson, var_rnoise, err = image_info
 
-    data = u.Quantity(data, u.DN / u.s, dtype=data.dtype)
-    var_poisson = u.Quantity(var_poisson, u.DN**2 / u.s**2, dtype=var_poisson.dtype)
-    var_rnoise = u.Quantity(var_rnoise, u.DN**2 / u.s**2, dtype=var_rnoise.dtype)
-    err = u.Quantity(err, u.DN / u.s, dtype=err.dtype)
     if dq is None:
         dq = np.zeros(data.shape, dtype="u4")
 
@@ -198,13 +194,11 @@ def create_image_model(input_model, image_info):
     meta["photometry"] = maker_utils.mk_photometry()
     inst = {
         "meta": meta,
-        "data": u.Quantity(data, u.DN / u.s, dtype=data.dtype),
+        "data": data,
         "dq": dq,
-        "var_poisson": u.Quantity(
-            var_poisson, u.DN**2 / u.s**2, dtype=var_poisson.dtype
-        ),
-        "var_rnoise": u.Quantity(var_rnoise, u.DN**2 / u.s**2, dtype=var_rnoise.dtype),
-        "err": u.Quantity(err, u.DN / u.s, dtype=err.dtype),
+        "var_poisson": var_poisson,
+        "var_rnoise": var_rnoise,
+        "err": err,
         "amp33": input_model.amp33.copy(),
         "border_ref_pix_left": input_model.border_ref_pix_left.copy(),
         "border_ref_pix_right": input_model.border_ref_pix_right.copy(),
diff --git a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py
index 042ca3d85..ee0b677b2 100644
--- a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py
+++ b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py
@@ -226,10 +226,10 @@ def model_from_resultants(resultants, read_pattern=None):
     gdq = np.zeros(shape=shape, dtype=np.uint8)
 
     dm_ramp = maker_utils.mk_ramp(shape=shape)
-    dm_ramp.data = u.Quantity(full_wfi, u.DN, dtype=np.float32)
+    dm_ramp.data = full_wfi
     dm_ramp.pixeldq = pixdq
     dm_ramp.groupdq = gdq
-    dm_ramp.err = u.Quantity(err, u.DN, dtype=np.float32)
+    dm_ramp.err = err
 
     dm_ramp.meta.exposure.frame_time = ROMAN_READ_TIME
     dm_ramp.meta.exposure.ngroups = shape[0]
diff --git a/romancal/refpix/data.py b/romancal/refpix/data.py
index 383305e48..e735a0c35 100644
--- a/romancal/refpix/data.py
+++ b/romancal/refpix/data.py
@@ -82,24 +82,15 @@ class StandardView(BaseView):
         - right = detector[-Const.REF:]
     """
 
-    @staticmethod
-    def extract_value(data):
-        if isinstance(data, u.Quantity):
-            if data.unit != u.DN:
-                raise ValueError(f"Input data must be in units of DN, not {data.unit}")
-            data = data.value
-
-        return data
-
     @classmethod
     def from_datamodel(cls, datamodel: RampModel) -> StandardView:
         """
         Read the datamodel into the standard view.
         """
-        detector = cls.extract_value(datamodel.data)
+        detector = datamodel.data
 
         # Extract amp33
-        amp33 = cls.extract_value(datamodel.amp33)
+        amp33 = datamodel.amp33
         # amp33 is normally a uint16, but this computation requires it to match
         # the data type of the detector pixels.
         amp33 = amp33.astype(detector.dtype)
@@ -112,23 +103,11 @@ def update(self, datamodel: RampModel) -> RampModel:
             - Returns the updated datamodel for a functional approach.
         """
 
-        datamodel.data = u.Quantity(
-            self.detector, unit=datamodel.data.unit, dtype=datamodel.data.dtype
-        )
+        datamodel.data = self.detector
         # ABS to avoid casting negative numbers to uint16
-        datamodel.amp33 = u.Quantity(
-            np.abs(self.amp33), unit=datamodel.amp33.unit, dtype=datamodel.amp33.dtype
-        )
-        datamodel.border_ref_pix_left = u.Quantity(
-            self.left,
-            unit=datamodel.border_ref_pix_left.unit,
-            dtype=datamodel.border_ref_pix_left.dtype,
-        )
-        datamodel.border_ref_pix_right = u.Quantity(
-            self.right,
-            unit=datamodel.border_ref_pix_right.unit,
-            dtype=datamodel.border_ref_pix_right.dtype,
-        )
+        datamodel.amp33 = np.abs(self.amp33).astype(datamodel.amp33.dtype)
+        datamodel.border_ref_pix_left = self.left
+        datamodel.border_ref_pix_right = self.right
 
         return datamodel
 
diff --git a/romancal/refpix/tests/conftest.py b/romancal/refpix/tests/conftest.py
index 3afdd2aab..9856094f1 100644
--- a/romancal/refpix/tests/conftest.py
+++ b/romancal/refpix/tests/conftest.py
@@ -48,12 +48,8 @@ def datamodel(data):
     assert detector.shape == (Dims.N_FRAMES, Dims.N_ROWS, Const.N_COLUMNS)
     assert amp33.shape == (Dims.N_FRAMES, Dims.N_ROWS, Const.CHAN_WIDTH)
 
-    datamodel.data = u.Quantity(
-        detector, unit=datamodel.data.unit, dtype=datamodel.data.dtype
-    )
-    datamodel.amp33 = u.Quantity(
-        amp33, unit=datamodel.amp33.unit, dtype=datamodel.amp33.dtype
-    )
+    datamodel.data = detector
+    datamodel.amp33 = amp33.astype(datamodel.amp33.dtype)
 
     datamodel.border_ref_pix_left = datamodel.data[:, :, : Const.REF]
     datamodel.border_ref_pix_right = datamodel.data[:, :, -Const.REF :]
diff --git a/romancal/refpix/tests/test_data.py b/romancal/refpix/tests/test_data.py
index 92555a08d..75fa2b854 100644
--- a/romancal/refpix/tests/test_data.py
+++ b/romancal/refpix/tests/test_data.py
@@ -57,13 +57,13 @@ def test_construct_from_datamodel(self, datamodel):
         assert standard.data.base is None
 
         # Check the relationship between the standard view and the datamodel
-        assert (standard.detector == datamodel.data.value).all()
-        assert (standard.left == datamodel.border_ref_pix_left.value).all()
-        assert (standard.right == datamodel.border_ref_pix_right.value).all()
+        assert (standard.detector == datamodel.data).all()
+        assert (standard.left == datamodel.border_ref_pix_left).all()
+        assert (standard.right == datamodel.border_ref_pix_right).all()
 
         # The amp33's dtype changes because it needs to be promoted to match that
         # of the rest of the data
-        assert (standard.amp33 == datamodel.amp33.value.astype(np.float32)).all()
+        assert (standard.amp33 == datamodel.amp33.astype(np.float32)).all()
 
     def test_update(self, datamodel):
         standard = StandardView.from_datamodel(datamodel)
@@ -97,20 +97,14 @@ def test_update(self, datamodel):
         assert new.border_ref_pix_left.dtype == old_left.dtype
         assert new.border_ref_pix_right.dtype == old_right.dtype
 
-        # Check the unit has been preserved
-        assert new.data.unit == old_detector.unit
-        assert new.amp33.unit == old_amp33.unit
-        assert new.border_ref_pix_left.unit == old_left.unit
-        assert new.border_ref_pix_right.unit == old_right.unit
-
         # Check the data has been updated correctly
-        assert (new.data.value == standard.detector).all()
-        assert (new.border_ref_pix_left.value == standard.left).all()
-        assert (new.border_ref_pix_right.value == standard.right).all()
+        assert (new.data == standard.detector).all()
+        assert (new.border_ref_pix_left == standard.left).all()
+        assert (new.border_ref_pix_right == standard.right).all()
 
         # The amp33's dtype changes because it needs to be shifted to match the
         # original data's dtype
-        assert (new.amp33.value == standard.amp33.astype(old_amp33.dtype)).all()
+        assert (new.amp33 == standard.amp33.astype(old_amp33.dtype)).all()
 
     def test_create_standard_view(self, data):
         """
diff --git a/romancal/refpix/tests/test_refpix.py b/romancal/refpix/tests/test_refpix.py
index e51181359..c36bb46f0 100644
--- a/romancal/refpix/tests/test_refpix.py
+++ b/romancal/refpix/tests/test_refpix.py
@@ -14,9 +14,5 @@ def test_run_steps_regression(datamodel, ref_pix_ref):
 
     result = run_steps(datamodel, ref_pix_ref, True, True, True, True)
 
-    assert (result.data.value == regression_out).all()
+    assert (result.data == regression_out).all()
     # regression_out does not return amp33 data
-
-    # Check the units
-    assert result.data.unit == u.DN
-    assert result.amp33.unit == u.DN
diff --git a/romancal/refpix/tests/test_step.py b/romancal/refpix/tests/test_step.py
index 2bd95e49b..3445c140a 100644
--- a/romancal/refpix/tests/test_step.py
+++ b/romancal/refpix/tests/test_step.py
@@ -43,21 +43,17 @@ def test_refpix_step(
     assert result is not regression
 
     # Check the data
-    assert (result.data.value == regression.data.value).all()
-    assert result.data.unit == regression.data.unit
+    assert (result.data == regression.data).all()
     # Check the amp33
-    assert (result.amp33.value == regression.amp33.value).all()
-    assert result.amp33.unit == regression.amp33.unit
+    assert (result.amp33 == regression.amp33).all()
     # Check left ref pix
     assert (
-        result.border_ref_pix_left.value == regression.border_ref_pix_left.value
+        result.border_ref_pix_left == regression.border_ref_pix_left
     ).all()
-    assert result.border_ref_pix_left.unit == regression.border_ref_pix_left.unit
     # Check right ref pix
     assert (
-        result.border_ref_pix_right.value == regression.border_ref_pix_right.value
+        result.border_ref_pix_right == regression.border_ref_pix_right
     ).all()
-    assert result.border_ref_pix_right.unit == regression.border_ref_pix_right.unit
     #
     # Run the step with reffile = N/A
     result = RefPixStep.call(
diff --git a/romancal/saturation/saturation.py b/romancal/saturation/saturation.py
index 87314e578..81b6a0378 100644
--- a/romancal/saturation/saturation.py
+++ b/romancal/saturation/saturation.py
@@ -34,7 +34,7 @@ def flag_saturation(input_model, ref_model):
         The input model is modified in place and returned as the output model.
     """
 
-    data = input_model.data[np.newaxis, :].value
+    data = input_model.data[np.newaxis, :]
 
     # Modify input_model in place.
     gdq = input_model.groupdq[np.newaxis, :]
diff --git a/romancal/saturation/tests/test_saturation.py b/romancal/saturation/tests/test_saturation.py
index 76c0eee3d..4d6c945fa 100644
--- a/romancal/saturation/tests/test_saturation.py
+++ b/romancal/saturation/tests/test_saturation.py
@@ -27,11 +27,11 @@ def test_basic_saturation_flagging(setup_wfi_datamodels):
     ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add ramp values up to the saturation limit
-    ramp.data[0, 5, 5] = 0 * ramp.data.unit
-    ramp.data[1, 5, 5] = 20000 * ramp.data.unit
-    ramp.data[2, 5, 5] = 40000 * ramp.data.unit
-    ramp.data[3, 5, 5] = 60000 * ramp.data.unit  # Signal reaches saturation limit
-    ramp.data[4, 5, 5] = 62000 * ramp.data.unit
+    ramp.data[0, 5, 5] = 0
+    ramp.data[1, 5, 5] = 20000
+    ramp.data[2, 5, 5] = 40000
+    ramp.data[3, 5, 5] = 60000  # Signal reaches saturation limit
+    ramp.data[4, 5, 5] = 62000
 
     # Set saturation value in the saturation model
     satmap.data[5, 5] = satvalue * satmap.data.unit
@@ -40,7 +40,7 @@ def test_basic_saturation_flagging(setup_wfi_datamodels):
     output = flag_saturation(ramp, satmap)
 
     # Make sure that groups with signal > saturation limit get flagged
-    satindex = np.argmax(output.data.value[:, 5, 5] == satvalue)
+    satindex = np.argmax(output.data[:, 5, 5] == satvalue)
     assert np.all(output.groupdq[satindex:, 5, 5] == group.SATURATED)
 
 
@@ -56,11 +56,11 @@ def test_read_pattern_saturation_flagging(setup_wfi_datamodels):
     ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add ramp values up to the saturation limit
-    ramp.data[0, 5, 5] = 0 * ramp.data.unit
-    ramp.data[1, 5, 5] = 20000 * ramp.data.unit
-    ramp.data[2, 5, 5] = 40000 * ramp.data.unit
-    ramp.data[3, 5, 5] = 60000 * ramp.data.unit  # Signal reaches saturation limit
-    ramp.data[4, 5, 5] = 62000 * ramp.data.unit
+    ramp.data[0, 5, 5] = 0
+    ramp.data[1, 5, 5] = 20000
+    ramp.data[2, 5, 5] = 40000
+    ramp.data[3, 5, 5] = 60000  # Signal reaches saturation limit
+    ramp.data[4, 5, 5] = 62000
 
     # set read_pattern to have many reads in the third resultant, so that
     # its mean exposure time is much smaller than its last read time
@@ -100,11 +100,11 @@ def test_ad_floor_flagging(setup_wfi_datamodels):
     ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add ramp values up to the saturation limit
-    ramp.data[0, 5, 5] = 0 * ramp.data.unit  # Signal at bottom rail - low saturation
-    ramp.data[1, 5, 5] = 0 * ramp.data.unit  # Signal at bottom rail - low saturation
-    ramp.data[2, 5, 5] = 20 * ramp.data.unit
-    ramp.data[3, 5, 5] = 40 * ramp.data.unit
-    ramp.data[4, 5, 5] = 60 * ramp.data.unit
+    ramp.data[0, 5, 5] = 0  # Signal at bottom rail - low saturation
+    ramp.data[1, 5, 5] = 0  # Signal at bottom rail - low saturation
+    ramp.data[2, 5, 5] = 20
+    ramp.data[3, 5, 5] = 40
+    ramp.data[4, 5, 5] = 60
 
     # frames that should be flagged as saturation (low)
     satindxs = [0, 1]
@@ -134,11 +134,11 @@ def test_ad_floor_and_saturation_flagging(setup_wfi_datamodels):
     ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add ramp values up to the saturation limit
-    ramp.data[0, 5, 5] = 0 * ramp.data.unit  # Signal at bottom rail - low saturation
-    ramp.data[1, 5, 5] = 0 * ramp.data.unit  # Signal at bottom rail - low saturation
-    ramp.data[2, 5, 5] = 20 * ramp.data.unit
-    ramp.data[3, 5, 5] = 40 * ramp.data.unit
-    ramp.data[4, 5, 5] = 61000 * ramp.data.unit  # Signal above the saturation threshold
+    ramp.data[0, 5, 5] = 0  # Signal at bottom rail - low saturation
+    ramp.data[1, 5, 5] = 0  # Signal at bottom rail - low saturation
+    ramp.data[2, 5, 5] = 20
+    ramp.data[3, 5, 5] = 40
+    ramp.data[4, 5, 5] = 61000  # Signal above the saturation threshold
 
     # frames that should be flagged as ad_floor
     floorindxs = [0, 1]
@@ -171,11 +171,11 @@ def test_signal_fluctuation_flagging(setup_wfi_datamodels):
     ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add ramp values up to the saturation limit
-    ramp.data[0, 5, 5] = 10 * ramp.data.unit
-    ramp.data[1, 5, 5] = 20000 * ramp.data.unit
-    ramp.data[2, 5, 5] = 40000 * ramp.data.unit
-    ramp.data[3, 5, 5] = 60000 * ramp.data.unit  # Signal reaches saturation limit
-    ramp.data[4, 5, 5] = 40000 * ramp.data.unit  # Signal drops below saturation limit
+    ramp.data[0, 5, 5] = 10
+    ramp.data[1, 5, 5] = 20000
+    ramp.data[2, 5, 5] = 40000
+    ramp.data[3, 5, 5] = 60000  # Signal reaches saturation limit
+    ramp.data[4, 5, 5] = 40000  # Signal drops below saturation limit
 
     # Set saturation value in the saturation model
     satmap.data[5, 5] = satvalue * satmap.data.unit
@@ -184,7 +184,7 @@ def test_signal_fluctuation_flagging(setup_wfi_datamodels):
     output = flag_saturation(ramp, satmap)
 
     # Make sure that all groups after first saturated group are flagged
-    satindex = np.argmax(output.data.value[:, 5, 5] == satvalue)
+    satindex = np.argmax(output.data[:, 5, 5] == satvalue)
     assert np.all(output.groupdq[satindex:, 5, 5] == group.SATURATED)
 
 
@@ -200,11 +200,11 @@ def test_all_groups_saturated(setup_wfi_datamodels):
     ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add ramp values at or above saturation limit
-    ramp.data[0, 5, 5] = 60000 * ramp.data.unit
-    ramp.data[1, 5, 5] = 62000 * ramp.data.unit
-    ramp.data[2, 5, 5] = 62000 * ramp.data.unit
-    ramp.data[3, 5, 5] = 60000 * ramp.data.unit
-    ramp.data[4, 5, 5] = 62000 * ramp.data.unit
+    ramp.data[0, 5, 5] = 60000
+    ramp.data[1, 5, 5] = 62000
+    ramp.data[2, 5, 5] = 62000
+    ramp.data[3, 5, 5] = 60000
+    ramp.data[4, 5, 5] = 62000
 
     # Set saturation value in the saturation model
     satmap.data[5, 5] = satvalue * satmap.data.unit
@@ -252,11 +252,11 @@ def test_no_sat_check(setup_wfi_datamodels):
     ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add ramp values up to the saturation limit
-    ramp.data[0, 5, 5] = 10 * ramp.data.unit
-    ramp.data[1, 5, 5] = 20000 * ramp.data.unit
-    ramp.data[2, 5, 5] = 40000 * ramp.data.unit
-    ramp.data[3, 5, 5] = 60000 * ramp.data.unit
-    ramp.data[4, 5, 5] = 62000 * ramp.data.unit  # Signal reaches saturation limit
+    ramp.data[0, 5, 5] = 10
+    ramp.data[1, 5, 5] = 20000
+    ramp.data[2, 5, 5] = 40000
+    ramp.data[3, 5, 5] = 60000
+    ramp.data[4, 5, 5] = 62000  # Signal reaches saturation limit
 
     # Set saturation value in the saturation model & DQ value for NO_SAT_CHECK
     satmap.data[5, 5] = satvalue * satmap.data.unit
@@ -291,11 +291,11 @@ def test_nans_in_mask(setup_wfi_datamodels):
     ramp, satmap = setup_wfi_datamodels(ngroups, nrows, ncols)
 
     # Add ramp values up to the saturation limit
-    ramp.data[0, 5, 5] = 10 * ramp.data.unit
-    ramp.data[1, 5, 5] = 20000 * ramp.data.unit
-    ramp.data[2, 5, 5] = 40000 * ramp.data.unit
-    ramp.data[3, 5, 5] = 60000 * ramp.data.unit
-    ramp.data[4, 5, 5] = 62000 * ramp.data.unit
+    ramp.data[0, 5, 5] = 10
+    ramp.data[1, 5, 5] = 20000
+    ramp.data[2, 5, 5] = 40000
+    ramp.data[3, 5, 5] = 60000
+    ramp.data[4, 5, 5] = 62000
 
     # Set saturation value for pixel to NaN
     satmap.data[5, 5] = np.nan * satmap.data.unit
@@ -324,9 +324,7 @@ def test_saturation_getbestref(setup_wfi_datamodels):
     wfi_sci_raw.meta["guidestar"]["gw_window_xstart"] = 1012
     wfi_sci_raw.meta["guidestar"]["gw_window_xsize"] = 16
     wfi_sci_raw.meta.exposure.type = "WFI_IMAGE"
-    wfi_sci_raw.data = u.Quantity(
-        np.ones(shape, dtype=np.uint16), u.DN, dtype=np.uint16
-    )
+    wfi_sci_raw.data = np.ones(shape, dtype=np.uint16)
     wfi_sci_raw_model = ScienceRawModel(wfi_sci_raw, dq=True)
 
     # Run the pipeline

From f11892818666fd3347f2b9601e270aaa9490d72e Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 8 Oct 2024 20:32:36 +0000
Subject: [PATCH 49/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/dark_current/tests/test_dark.py     | 1 -
 romancal/dq_init/tests/test_dq_init.py       | 1 -
 romancal/flatfield/tests/test_flatfield.py   | 3 +--
 romancal/linearity/linearity_step.py         | 1 -
 romancal/linearity/tests/test_linearity.py   | 1 -
 romancal/refpix/data.py                      | 1 -
 romancal/refpix/tests/conftest.py            | 1 -
 romancal/refpix/tests/test_refpix.py         | 2 --
 romancal/refpix/tests/test_step.py           | 8 ++------
 romancal/regtest/test_mos_pipeline.py        | 2 --
 romancal/saturation/tests/test_saturation.py | 1 -
 11 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/romancal/dark_current/tests/test_dark.py b/romancal/dark_current/tests/test_dark.py
index 7f03a045a..316063cb2 100644
--- a/romancal/dark_current/tests/test_dark.py
+++ b/romancal/dark_current/tests/test_dark.py
@@ -5,7 +5,6 @@
 import numpy as np
 import pytest
 import roman_datamodels as rdm
-from astropy import units as u
 from roman_datamodels import maker_utils
 from roman_datamodels.datamodels import DarkRefModel, RampModel
 
diff --git a/romancal/dq_init/tests/test_dq_init.py b/romancal/dq_init/tests/test_dq_init.py
index 2b24f8220..c7ed8b8c0 100644
--- a/romancal/dq_init/tests/test_dq_init.py
+++ b/romancal/dq_init/tests/test_dq_init.py
@@ -1,6 +1,5 @@
 import numpy as np
 import pytest
-from astropy import units as u
 from roman_datamodels import maker_utils, stnode
 from roman_datamodels.datamodels import MaskRefModel, ScienceRawModel
 from roman_datamodels.dqflags import pixel
diff --git a/romancal/flatfield/tests/test_flatfield.py b/romancal/flatfield/tests/test_flatfield.py
index 964aff33a..f246fae2e 100644
--- a/romancal/flatfield/tests/test_flatfield.py
+++ b/romancal/flatfield/tests/test_flatfield.py
@@ -1,6 +1,5 @@
 import numpy as np
 import pytest
-from astropy import units as u
 from astropy.time import Time
 from roman_datamodels import maker_utils, stnode
 from roman_datamodels.datamodels import FlatRefModel, ImageModel
@@ -26,7 +25,7 @@ def test_flatfield_step_interface(instrument, exptype):
     wfi_image.meta.instrument.detector = "WFI01"
     wfi_image.meta.instrument.optical_element = "F158"
     wfi_image.meta.exposure.type = exptype
-    wfi_image.data =  np.ones(shape, dtype=np.float32)
+    wfi_image.data = np.ones(shape, dtype=np.float32)
     wfi_image.dq = np.zeros(shape, dtype=np.uint32)
     wfi_image.err = np.zeros(shape, dtype=np.float32)
     wfi_image.var_poisson = np.zeros(shape, dtype=np.float32)
diff --git a/romancal/linearity/linearity_step.py b/romancal/linearity/linearity_step.py
index e78dc980f..217fa6795 100644
--- a/romancal/linearity/linearity_step.py
+++ b/romancal/linearity/linearity_step.py
@@ -3,7 +3,6 @@
 """
 
 import numpy as np
-from astropy import units as u
 from roman_datamodels import datamodels as rdd
 from roman_datamodels.dqflags import pixel
 from stcal.linearity.linearity import linearity_correction
diff --git a/romancal/linearity/tests/test_linearity.py b/romancal/linearity/tests/test_linearity.py
index ab11fb2d2..6b1fed838 100644
--- a/romancal/linearity/tests/test_linearity.py
+++ b/romancal/linearity/tests/test_linearity.py
@@ -6,7 +6,6 @@
 
 import numpy as np
 import pytest
-from astropy import units as u
 from roman_datamodels import maker_utils
 from roman_datamodels.datamodels import LinearityRefModel, ScienceRawModel
 
diff --git a/romancal/refpix/data.py b/romancal/refpix/data.py
index e735a0c35..0a4ee55a8 100644
--- a/romancal/refpix/data.py
+++ b/romancal/refpix/data.py
@@ -11,7 +11,6 @@
 from enum import IntEnum
 
 import numpy as np
-from astropy import units as u
 from scipy import fft
 
 
diff --git a/romancal/refpix/tests/conftest.py b/romancal/refpix/tests/conftest.py
index 9856094f1..f70795b9c 100644
--- a/romancal/refpix/tests/conftest.py
+++ b/romancal/refpix/tests/conftest.py
@@ -2,7 +2,6 @@
 
 import numpy as np
 import pytest
-from astropy import units as u
 from roman_datamodels.datamodels import RampModel, RefpixRefModel
 from roman_datamodels.maker_utils import mk_ramp, mk_refpix
 
diff --git a/romancal/refpix/tests/test_refpix.py b/romancal/refpix/tests/test_refpix.py
index c36bb46f0..5fd6e04f4 100644
--- a/romancal/refpix/tests/test_refpix.py
+++ b/romancal/refpix/tests/test_refpix.py
@@ -1,5 +1,3 @@
-from astropy import units as u
-
 from romancal.refpix.data import StandardView
 from romancal.refpix.refpix import run_steps
 
diff --git a/romancal/refpix/tests/test_step.py b/romancal/refpix/tests/test_step.py
index 3445c140a..7aaf28867 100644
--- a/romancal/refpix/tests/test_step.py
+++ b/romancal/refpix/tests/test_step.py
@@ -47,13 +47,9 @@ def test_refpix_step(
     # Check the amp33
     assert (result.amp33 == regression.amp33).all()
     # Check left ref pix
-    assert (
-        result.border_ref_pix_left == regression.border_ref_pix_left
-    ).all()
+    assert (result.border_ref_pix_left == regression.border_ref_pix_left).all()
     # Check right ref pix
-    assert (
-        result.border_ref_pix_right == regression.border_ref_pix_right
-    ).all()
+    assert (result.border_ref_pix_right == regression.border_ref_pix_right).all()
     #
     # Run the step with reffile = N/A
     result = RefPixStep.call(
diff --git a/romancal/regtest/test_mos_pipeline.py b/romancal/regtest/test_mos_pipeline.py
index 547a22e67..3cd150d74 100644
--- a/romancal/regtest/test_mos_pipeline.py
+++ b/romancal/regtest/test_mos_pipeline.py
@@ -19,7 +19,6 @@
 pytestmark = [pytest.mark.bigdata, pytest.mark.soctests]
 
 
-
 class RegtestFileModifier:
     # TODO: remove this entire class once the units
     #  have been removed from the regtest files
@@ -100,7 +99,6 @@ def prepare_regtest_input_files(self):
         self.update_rtdata()
 
 
-
 @pytest.fixture(scope="module")
 def run_mos(rtdata_module):
     rtdata = rtdata_module
diff --git a/romancal/saturation/tests/test_saturation.py b/romancal/saturation/tests/test_saturation.py
index 4d6c945fa..f6bbd3df9 100644
--- a/romancal/saturation/tests/test_saturation.py
+++ b/romancal/saturation/tests/test_saturation.py
@@ -6,7 +6,6 @@
 
 import numpy as np
 import pytest
-from astropy import units as u
 from roman_datamodels import maker_utils
 from roman_datamodels.datamodels import ScienceRawModel
 from roman_datamodels.dqflags import group, pixel

From 91155e65325a3eb7203ff1f0f907726a4a8d9f04 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 8 Oct 2024 15:24:30 -0400
Subject: [PATCH 50/74] Remove units (all unit tests passing).

---
 romancal/jump/jump_step.py                    |  5 +-
 romancal/jump/tests/test_jump_step.py         | 14 ++--
 romancal/outlier_detection/_fileio.py         |  2 +-
 .../tests/test_outlier_detection.py           | 24 +++----
 romancal/outlier_detection/utils.py           | 10 +--
 romancal/photom/photom.py                     | 12 ++--
 romancal/photom/tests/test_photom.py          | 66 +++++++++----------
 .../ramp_fitting/tests/test_ramp_fit_cas22.py |  2 +-
 romancal/regtest/test_regtestdata.py          |  4 +-
 romancal/resample/tests/test_resample.py      | 28 ++------
 romancal/resample/tests/test_resample_step.py |  9 ++-
 romancal/skymatch/tests/test_skymatch.py      | 54 +++++++--------
 12 files changed, 100 insertions(+), 130 deletions(-)

diff --git a/romancal/jump/jump_step.py b/romancal/jump/jump_step.py
index dfee1592f..74d555863 100644
--- a/romancal/jump/jump_step.py
+++ b/romancal/jump/jump_step.py
@@ -53,10 +53,10 @@ def process(self, input):
 
         # Extract the needed info from the Roman Data Model
         meta = input_model.meta
-        r_data = input_model.data.value
+        r_data = input_model.data
         r_gdq = input_model.groupdq
         r_pdq = input_model.pixeldq
-        r_err = input_model.err.value
+        r_err = input_model.err
         result = input_model
 
         # If the ramp fitting jump detection is enabled, then skip this step
@@ -106,6 +106,7 @@ def process(self, input):
             self.log.info("Maximum cores to use = %s", max_cores)
 
         # Get the gain and readnoise reference files
+        # TODO: remove units from gain and RN reference files
         gain_filename = self.get_reference_file(input_model, "gain")
         self.log.info("Using GAIN reference file: %s", gain_filename)
         gain_model = rdd.GainRefModel(gain_filename)
diff --git a/romancal/jump/tests/test_jump_step.py b/romancal/jump/tests/test_jump_step.py
index b34c63958..485bf255b 100644
--- a/romancal/jump/tests/test_jump_step.py
+++ b/romancal/jump/tests/test_jump_step.py
@@ -106,10 +106,10 @@ def _setup(
         dm_ramp.meta.instrument.name = "WFI"
         dm_ramp.meta.instrument.optical_element = "F158"
 
-        dm_ramp.data = u.Quantity(data + 6.0, u.DN, dtype=np.float32)
+        dm_ramp.data = data + 6.0
         dm_ramp.pixeldq = pixdq
         dm_ramp.groupdq = gdq
-        dm_ramp.err = u.Quantity(err, u.DN, dtype=np.float32)
+        dm_ramp.err = err
 
         dm_ramp.meta.exposure.type = "WFI_IMAGE"
         dm_ramp.meta.exposure.group_time = deltatime
@@ -148,7 +148,7 @@ def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs):
     )
 
     for i in range(ngroups):
-        model1.data[i, :, :] = deltaDN * i * model1.data.unit
+        model1.data[i, :, :] = deltaDN * i
 
     first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0]
 
@@ -162,7 +162,7 @@ def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs):
         CR_group = next(CR_pool)
         model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = (
             model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]]
-            + 5000.0 * model1.data.unit
+            + 5000.0
         )
 
     out_model = JumpStep.call(
@@ -203,7 +203,7 @@ def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs):
     )
 
     for i in range(ngroups):
-        model1.data[i, :, :] = deltaDN * i * model1.data.unit
+        model1.data[i, :, :] = deltaDN * i
 
     first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0]
     CR_locs = [x for x in range(xsize * ysize) if x % CR_fraction == 0]
@@ -215,11 +215,11 @@ def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs):
         CR_group = next(CR_pool)
 
         model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = (
-            model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000 * model1.data.unit
+            model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000
         )
         model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] = (
             model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]]
-            + 700 * model1.data.unit
+            + 700
         )
 
     out_model = JumpStep.call(
diff --git a/romancal/outlier_detection/_fileio.py b/romancal/outlier_detection/_fileio.py
index d1dde8489..48c942b71 100644
--- a/romancal/outlier_detection/_fileio.py
+++ b/romancal/outlier_detection/_fileio.py
@@ -20,7 +20,7 @@ def save_drizzled(drizzled_model, make_output_path):
 
 def _make_median_model(example_model, data, wcs):
     model = example_model.copy()
-    model.data = Quantity(data, unit=model.data.unit)
+    model.data = data
     model.meta.filename = "drizzled_median.asdf"
     model.meta.wcs = wcs
     return model
diff --git a/romancal/outlier_detection/tests/test_outlier_detection.py b/romancal/outlier_detection/tests/test_outlier_detection.py
index 0069fde88..a5a25f1c2 100644
--- a/romancal/outlier_detection/tests/test_outlier_detection.py
+++ b/romancal/outlier_detection/tests/test_outlier_detection.py
@@ -1,9 +1,7 @@
 import os
 
-import astropy.units as u
 import numpy as np
 import pytest
-from astropy.units import Quantity
 
 from romancal.datamodels import ModelLibrary
 from romancal.outlier_detection import OutlierDetectionStep
@@ -104,10 +102,10 @@ def test_outlier_do_detection_write_files_to_custom_location(tmp_path, base_imag
     """
     img_1 = base_image()
     img_1.meta.filename = "img1_cal.asdf"
-    img_1.meta.background.level = 0 * u.DN / u.s
+    img_1.meta.background.level = 0
     img_2 = base_image()
     img_2.meta.filename = "img2_cal.asdf"
-    img_2.meta.background.level = 0 * u.DN / u.s
+    img_2.meta.background.level = 0
     input_models = ModelLibrary([img_1, img_2])
 
     outlier_step = OutlierDetectionStep(
@@ -134,9 +132,9 @@ def test_find_outliers(tmp_path, base_image, on_disk):
     """
     Test that OutlierDetection can find outliers.
     """
-    cr_value = Quantity(100, "DN / s")
-    source_value = Quantity(10, "DN / s")
-    err_value = Quantity(10, "DN / s")  # snr=1
+    cr_value = 100
+    source_value = 10
+    err_value = 10  # snr=1
 
     imgs = []
     for i in range(3):
@@ -145,7 +143,7 @@ def test_find_outliers(tmp_path, base_image, on_disk):
         img.err[:] = err_value
         img.meta.filename = f"img{i}_suffix.asdf"
         img.meta.observation.exposure = i
-        img.meta.background.level = 0 * u.DN / u.s
+        img.meta.background.level = 0
         imgs.append(img)
 
     # add outliers
@@ -204,14 +202,12 @@ def test_identical_images(tmp_path, base_image, caplog):
     """
     img_1 = base_image()
     img_1.meta.filename = "img1_suffix.asdf"
-    img_1.meta.background.level = 0 * u.DN / u.s
+    img_1.meta.background.level = 0
     # add outliers
     img_1_input_coords = np.array(
         [(5, 45), (25, 25), (45, 85), (65, 65), (85, 5)], dtype=[("x", int), ("y", int)]
     )
-    img_1.data[img_1_input_coords["x"], img_1_input_coords["y"]] = Quantity(
-        100000, "DN / s"
-    )
+    img_1.data[img_1_input_coords["x"], img_1_input_coords["y"]] = 100000
 
     img_2 = img_1.copy()
     img_2.meta.filename = "img2_suffix.asdf"
@@ -258,10 +254,10 @@ def test_outlier_detection_always_returns_modelcontainer_with_updated_datamodels
     os.chdir(tmp_path)
     img_1 = base_image()
     img_1.meta.filename = "img_1.asdf"
-    img_1.data *= img_1.meta.photometry.conversion_megajanskys / img_1.data.unit
+    img_1.data *= img_1.meta.photometry.conversion_megajanskys / img_1.data
     img_2 = base_image()
     img_2.meta.filename = "img_2.asdf"
-    img_2.data *= img_2.meta.photometry.conversion_megajanskys / img_2.data.unit
+    img_2.data *= img_2.meta.photometry.conversion_megajanskys / img_2.data
 
     library = ModelLibrary([img_1, img_2])
     library._save(tmp_path)
diff --git a/romancal/outlier_detection/utils.py b/romancal/outlier_detection/utils.py
index 6e529d052..703bd6dbe 100644
--- a/romancal/outlier_detection/utils.py
+++ b/romancal/outlier_detection/utils.py
@@ -86,7 +86,7 @@ def _median_with_resampling(
 
             weight_threshold = compute_weight_threshold(drizzled_model.weight, maskpt)
             drizzled_model.data[drizzled_model.weight < weight_threshold] = np.nan
-            computer.append(drizzled_model.data.value, i)
+            computer.append(drizzled_model.data, i)
             del drizzled_model
 
     # Perform median combination on set of drizzled mosaics
@@ -173,7 +173,7 @@ def _median_without_resampling(
 
             weight_threshold = compute_weight_threshold(wht, maskpt)
 
-            data_copy = model.data.value.copy()
+            data_copy = model.data.copy()
             data_copy[wht < weight_threshold] = np.nan
             computer.append(data_copy, i)
 
@@ -211,13 +211,13 @@ def _flag_resampled_model_crs(
         and image.meta.background.subtracted is False
         and image.meta.background.level is not None
     ):
-        backg = image.meta.background.level.value
+        backg = image.meta.background.level
         log.debug(
             f"Adding background level {image.meta.background.level} to blotted image"
         )
 
     cr_mask = flag_resampled_crs(
-        image.data.value, image.err.value, blot, snr1, snr2, scale1, scale2, backg
+        image.data, image.err, blot, snr1, snr2, scale1, scale2, backg
     )
 
     # update the dq flags in-place
@@ -226,7 +226,7 @@ def _flag_resampled_model_crs(
 
 
 def _flag_model_crs(image, median_data, snr):
-    cr_mask = flag_crs(image.data.value, image.err.value, median_data, snr)
+    cr_mask = flag_crs(image.data, image.err, median_data, snr)
 
     # Update dq array in-place
     image.dq |= cr_mask * np.uint32(pixel.DO_NOT_USE | pixel.OUTLIER)
diff --git a/romancal/photom/photom.py b/romancal/photom/photom.py
index 75c6243e6..778777080 100644
--- a/romancal/photom/photom.py
+++ b/romancal/photom/photom.py
@@ -28,20 +28,20 @@ def photom_io(input_model, photom_metadata):
 
     # Store the conversion factor in the meta data
     log.info(f"photmjsr value: {conversion:.6g}")
-    input_model.meta.photometry.conversion_megajanskys = conversion
+    input_model.meta.photometry.conversion_megajanskys = conversion.value
     input_model.meta.photometry.conversion_microjanskys = conversion.to(
         u.microjansky / u.arcsecond**2
-    )
+    ).value
 
     # Get the scalar conversion uncertainty factor
     uncertainty_conv = photom_metadata["uncertainty"]
 
     # Store the uncertainty conversion factor in the meta data
     log.info(f"uncertainty value: {uncertainty_conv:.6g}")
-    input_model.meta.photometry.conversion_megajanskys_uncertainty = uncertainty_conv
+    input_model.meta.photometry.conversion_megajanskys_uncertainty = uncertainty_conv.value
     input_model.meta.photometry.conversion_microjanskys_uncertainty = (
         uncertainty_conv.to(u.microjansky / u.arcsecond**2)
-    )
+    ).value
 
     # Return updated input model
     return input_model
@@ -62,8 +62,8 @@ def save_area_info(input_model, photom_parameters):
     """
 
     # Load the average pixel area values from the photom reference file header
-    area_ster = photom_parameters["pixelareasr"]
-    area_a2 = photom_parameters["pixelareasr"].to(u.arcsecond**2)
+    area_ster = photom_parameters["pixelareasr"].value
+    area_a2 = photom_parameters["pixelareasr"].to(u.arcsecond**2).value
 
     # Copy the pixel area values to the input model
     log.debug(f"pixelarea_steradians = {area_ster}, pixelarea_arcsecsq = {area_a2}")
diff --git a/romancal/photom/tests/test_photom.py b/romancal/photom/tests/test_photom.py
index 044a3737a..1877e17ec 100644
--- a/romancal/photom/tests/test_photom.py
+++ b/romancal/photom/tests/test_photom.py
@@ -102,13 +102,9 @@ def test_no_photom_match():
     input_model.meta.instrument.optical_element = "F146"
 
     # Set bad values which would be overwritten by apply_photom
-    input_model.meta.photometry.pixelarea_steradians = -1.0 * u.sr
-    input_model.meta.photometry.conversion_megajanskys = (
-        -1.0 * u.megajansky / u.steradian
-    )
-    input_model.meta.photometry.conversion_microjanskys_uncertainty = (
-        -1.0 * u.microjansky / u.arcsecond**2
-    )
+    input_model.meta.photometry.pixelarea_steradians = -1.0
+    input_model.meta.photometry.conversion_megajanskys = -1.0
+    input_model.meta.photometry.conversion_microjanskys_uncertainty = -1.0
 
     with warnings.catch_warnings(record=True) as caught:
         # Look for now non existent F146 optical element
@@ -121,14 +117,12 @@ def test_no_photom_match():
         )
 
         # Assert that photom elements are not updated
-        assert output_model.meta.photometry.pixelarea_steradians == -1.0 * u.sr
+        assert output_model.meta.photometry.pixelarea_steradians == -1.0
         assert (
-            output_model.meta.photometry.conversion_megajanskys
-            == -1.0 * u.megajansky / u.steradian
+            output_model.meta.photometry.conversion_megajanskys == -1.0
         )
         assert (
-            output_model.meta.photometry.conversion_microjanskys_uncertainty
-            == -1.0 * u.microjansky / u.arcsecond**2
+            output_model.meta.photometry.conversion_microjanskys_uncertainty == -1.0
         )
 
 
@@ -153,17 +147,17 @@ def test_apply_photom1():
 
     # Tests for pixel areas
     assert np.isclose(
-        output_model.meta.photometry.pixelarea_steradians.value,
+        output_model.meta.photometry.pixelarea_steradians,
         area_ster.value,
         atol=1.0e-7,
     )
-    assert output_model.meta.photometry.pixelarea_steradians.unit == area_ster.unit
+    # assert output_model.meta.photometry.pixelarea_steradians.unit == area_ster.unit
     assert np.isclose(
-        output_model.meta.photometry.pixelarea_arcsecsq.value,
+        output_model.meta.photometry.pixelarea_arcsecsq,
         area_a2.value,
         atol=1.0e-7,
     )
-    assert output_model.meta.photometry.pixelarea_arcsecsq.unit == area_a2.unit
+    # assert output_model.meta.photometry.pixelarea_arcsecsq.unit == area_a2.unit
 
     # Set reference photometry
     phot_ster = 3.5 * u.megajansky / u.steradian
@@ -171,17 +165,17 @@ def test_apply_photom1():
 
     # Tests for photometry
     assert np.isclose(
-        output_model.meta.photometry.conversion_megajanskys.value,
+        output_model.meta.photometry.conversion_megajanskys,
         phot_ster.value,
         atol=1.0e-7,
     )
-    assert output_model.meta.photometry.conversion_megajanskys.unit == phot_ster.unit
+    # assert output_model.meta.photometry.conversion_megajanskys.unit == phot_ster.unit
     assert np.isclose(
-        output_model.meta.photometry.conversion_microjanskys.value,
+        output_model.meta.photometry.conversion_microjanskys,
         phot_a2.value,
         atol=1.0e-7,
     )
-    assert output_model.meta.photometry.conversion_microjanskys.unit == phot_a2.unit
+    # assert output_model.meta.photometry.conversion_microjanskys.unit == phot_a2.unit
 
     # Set reference photometric uncertainty
     muphot_ster = 0.175 * u.megajansky / u.steradian
@@ -189,23 +183,23 @@ def test_apply_photom1():
 
     # Tests for photometric uncertainty
     assert np.isclose(
-        output_model.meta.photometry.conversion_megajanskys_uncertainty.value,
+        output_model.meta.photometry.conversion_megajanskys_uncertainty,
         muphot_ster.value,
         atol=1.0e-7,
     )
-    assert (
-        output_model.meta.photometry.conversion_megajanskys_uncertainty.unit
-        == muphot_ster.unit
-    )
+    # assert (
+    #     output_model.meta.photometry.conversion_megajanskys_uncertainty.unit
+    #     == muphot_ster.unit
+    # )
     assert np.isclose(
-        output_model.meta.photometry.conversion_microjanskys_uncertainty.value,
+        output_model.meta.photometry.conversion_microjanskys_uncertainty,
         muphot_a2.value,
         atol=1.0e-7,
     )
-    assert (
-        output_model.meta.photometry.conversion_microjanskys_uncertainty.unit
-        == muphot_a2.unit
-    )
+    # assert (
+    #     output_model.meta.photometry.conversion_microjanskys_uncertainty.unit
+    #     == muphot_a2.unit
+    # )
 
 
 def test_apply_photom2():
@@ -287,22 +281,22 @@ def test_photom_step_interface_spectroscopic(instrument, exptype):
     wfi_image.meta.instrument.optical_element = "PRISM"
 
     # Set photometric values for spectroscopic data
-    wfi_image.meta.photometry.pixelarea_steradians = 2.31307642258977e-14 * u.steradian
+    wfi_image.meta.photometry.pixelarea_steradians = (2.31307642258977e-14 * u.steradian).value
     wfi_image.meta.photometry.pixelarea_arcsecsq = (
         0.000984102303070964 * u.arcsecond * u.arcsecond
-    )
+    ).value
     wfi_image.meta.photometry.conversion_megajanskys = (
         -99999 * u.megajansky / u.steradian
-    )
+    ).value
     wfi_image.meta.photometry.conversion_megajanskys_uncertainty = (
         -99999 * u.megajansky / u.steradian
-    )
+    ).value
     wfi_image.meta.photometry.conversion_microjanskys = (
         -99999 * u.microjansky / u.arcsecond**2
-    )
+    ).value
     wfi_image.meta.photometry.conversion_microjanskys_uncertainty = (
         -99999 * u.microjansky / u.arcsecond**2
-    )
+    ).value
 
     # Create input model
     wfi_image_model = ImageModel(wfi_image)
diff --git a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py
index ee0b677b2..419cd8ec6 100644
--- a/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py
+++ b/romancal/ramp_fitting/tests/test_ramp_fit_cas22.py
@@ -105,7 +105,7 @@ def test_fits(fit_ramps, attribute):
     """Check slopes"""
     image_model, expected = fit_ramps
 
-    value = getattr(image_model, attribute).value
+    value = getattr(image_model, attribute)
     np.testing.assert_allclose(value, expected[attribute], 1e-05)
 
 
diff --git a/romancal/regtest/test_regtestdata.py b/romancal/regtest/test_regtestdata.py
index 4b71c3f2f..43f7a64ce 100644
--- a/romancal/regtest/test_regtestdata.py
+++ b/romancal/regtest/test_regtestdata.py
@@ -33,9 +33,9 @@ def test_compare_asdf(tmp_path, modification):
     l2.save(file0)
     atol = 0.0001
     if modification == "small":
-        l2.data += (atol / 2) * l2.data.unit
+        l2.data += (atol / 2)
     elif modification == "large":
-        l2.data += (atol * 2) * l2.data.unit
+        l2.data += (atol * 2)
     l2.save(file1)
     diff = compare_asdf(file0, file1, atol=atol)
     if modification == "large":
diff --git a/romancal/resample/tests/test_resample.py b/romancal/resample/tests/test_resample.py
index b7eda3088..78c9f7fbc 100644
--- a/romancal/resample/tests/test_resample.py
+++ b/romancal/resample/tests/test_resample.py
@@ -66,7 +66,7 @@ def create_image(self):
         datamodels.ImageModel
             An L2 ImageModel datamodel.
         """
-        rng = np.random.default_rng()
+        rng = np.random.default_rng(seed=13)
         l2 = maker_utils.mk_level2_image(
             shape=self.shape,
             **{
@@ -94,26 +94,10 @@ def create_image(self):
                         "exposure": 1,
                     },
                 },
-                "data": u.Quantity(
-                    rng.poisson(2.5, size=self.shape).astype(np.float32),
-                    u.MJy / u.sr,
-                    dtype=np.float32,
-                ),
-                "var_rnoise": u.Quantity(
-                    rng.normal(1, 0.05, size=self.shape).astype(np.float32),
-                    u.MJy**2 / u.sr**2,
-                    dtype=np.float32,
-                ),
-                "var_poisson": u.Quantity(
-                    rng.poisson(1, size=self.shape).astype(np.float32),
-                    u.MJy**2 / u.sr**2,
-                    dtype=np.float32,
-                ),
-                "var_flat": u.Quantity(
-                    rng.uniform(0, 1, size=self.shape).astype(np.float32),
-                    u.MJy**2 / u.sr**2,
-                    dtype=np.float32,
-                ),
+                "data": rng.poisson(2.5, size=self.shape).astype(np.float32),
+                "var_rnoise": rng.normal(1, 0.05, size=self.shape).astype(np.float32),
+                "var_poisson": rng.poisson(1, size=self.shape).astype(np.float32),
+                "var_flat": rng.uniform(0, 1, size=self.shape).astype(np.float32),
             },
         )
         # data from WFISim simulation of SCA #01
@@ -634,7 +618,7 @@ def test_resample_variance_array(wfi_sca1, wfi_sca4, name):
     expected_combined_variance_value = np.nanmean(mean_data) / len(input_models)
 
     np.isclose(
-        np.nanmean(getattr(output_model, name)).value,
+        np.nanmean(getattr(output_model, name)),
         expected_combined_variance_value,
         atol=0.01,
     )
diff --git a/romancal/resample/tests/test_resample_step.py b/romancal/resample/tests/test_resample_step.py
index 74beb7a8e..a221cc1f0 100644
--- a/romancal/resample/tests/test_resample_step.py
+++ b/romancal/resample/tests/test_resample_step.py
@@ -4,7 +4,6 @@
 from astropy import coordinates as coord
 from astropy import units as u
 from astropy.modeling import models
-from astropy.units import Quantity
 from gwcs import WCS
 from gwcs import coordinate_frames as cf
 from roman_datamodels import datamodels, maker_utils
@@ -318,7 +317,7 @@ def test_build_driz_weight_multiple_good_bits(
     data_shape = dq_array.shape
     img1 = base_image()
     img1.dq = dq_array
-    img1.data = Quantity(np.ones(data_shape), unit="DN / s")
+    img1.data = np.ones(data_shape)
 
     result = resample_utils.build_driz_weight(
         img1, weight_type=None, good_bits=good_bits
@@ -346,7 +345,7 @@ def test_set_good_bits_in_resample_meta(base_image, good_bits):
 
     img = datamodels.ImageModel(model)
 
-    img.data *= img.meta.photometry.conversion_megajanskys / img.data.unit
+    img.data *= img.meta.photometry.conversion_megajanskys / img.data
 
     step = ResampleStep
 
@@ -361,14 +360,14 @@ def test_build_driz_weight_different_weight_type(base_image, weight_type):
     img1 = base_image()
     # update attributes that will be used in building the weight array
     img1.meta.exposure.exposure_time = 10
-    img1.var_rnoise = Quantity(rng.normal(1, 0.1, size=img1.shape), unit="DN2 / s2")
+    img1.var_rnoise = rng.normal(1, 0.1, size=img1.shape)
     # build the drizzle weight array
     result = resample_utils.build_driz_weight(
         img1, weight_type=weight_type, good_bits="~DO_NOT_USE+NON_SCIENCE"
     )
 
     expected_results = {
-        "ivm": img1.var_rnoise.value**-1,
+        "ivm": img1.var_rnoise**-1,
         "exptime": np.ones(img1.shape, dtype=img1.data.dtype)
         * img1.meta.exposure.exposure_time,
         None: np.ones(img1.shape, dtype=img1.data.dtype),
diff --git a/romancal/skymatch/tests/test_skymatch.py b/romancal/skymatch/tests/test_skymatch.py
index b9aa61fd5..672a06dc2 100644
--- a/romancal/skymatch/tests/test_skymatch.py
+++ b/romancal/skymatch/tests/test_skymatch.py
@@ -73,12 +73,9 @@ def mk_image_model(
 ):
     l2 = mk_level2_image(shape=image_shape)
     l2_im = ImageModel(l2)
-    l2_im.data = u.Quantity(
-        rng.normal(loc=rate_mean, scale=rate_std, size=l2_im.data.shape).astype(
+    l2_im.data = rng.normal(loc=rate_mean, scale=rate_std, size=l2_im.data.shape).astype(
             np.float32
-        ),
-        l2_im.data.unit,
-    )
+        )
 
     l2_im.meta["wcs"] = mk_gwcs(image_shape, sky_offset=sky_offset, rotate=rotation)
 
@@ -125,11 +122,10 @@ def _add_bad_pixels(im, sat_val, dont_use_val):
     mask = np.ones(im.data.shape, dtype=bool)
     # Add some "bad" pixels:
     # corners
-    im_unit = im.data.unit
-    im.data[:5, :5] = sat_val * im_unit
-    im.data[-5:, :5] = sat_val * im_unit
-    im.data[-5:, -5:] = sat_val * im_unit
-    im.data[:5, -5:] = sat_val * im_unit
+    im.data[:5, :5] = sat_val
+    im.data[-5:, :5] = sat_val
+    im.data[-5:, -5:] = sat_val
+    im.data[:5, -5:] = sat_val
 
     im.dq[:5, :5] = pixel.SATURATED
     im.dq[-5:, :5] = pixel.SATURATED
@@ -146,7 +142,7 @@ def _add_bad_pixels(im, sat_val, dont_use_val):
     cy -= 5
 
     # center
-    im.data[cx : cx + 10, cy : cy + 10] = dont_use_val * im_unit
+    im.data[cx : cx + 10, cy : cy + 10] = dont_use_val
     im.dq[cx : cx + 10, cy : cy + 10] = pixel.DO_NOT_USE
     mask[cx : cx + 10, cy : cy + 10] = False
 
@@ -183,7 +179,7 @@ def test_skymatch(wfi_rate, skymethod, subtract, skystat, match_down):
 
     with library:
         for i, (im, lev) in enumerate(zip(library, levels)):
-            im.data = rng.normal(loc=lev, scale=0.05, size=im.data.shape) * im.data.unit
+            im.data = rng.normal(loc=lev, scale=0.05, size=im.data.shape)
             library.shelve(im, i)
 
     # exclude central DO_NOT_USE and corner SATURATED pixels
@@ -219,14 +215,14 @@ def test_skymatch(wfi_rate, skymethod, subtract, skystat, match_down):
             assert im.meta.background.subtracted == subtract
 
             # test computed/measured sky values if level is set:
-            if not np.isclose(im.meta.background.level.value, 0):
-                assert abs(im.meta.background.level.value - rlev) < 0.01
+            if not np.isclose(im.meta.background.level, 0):
+                assert abs(im.meta.background.level - rlev) < 0.01
 
             # test
             if subtract:
-                assert abs(np.mean(im.data[dq_mask]).value - slev) < 0.01
+                assert abs(np.mean(im.data[dq_mask]) - slev) < 0.01
             else:
-                assert abs(np.mean(im.data[dq_mask]).value - lev) < 0.01
+                assert abs(np.mean(im.data[dq_mask]) - lev) < 0.01
             result.shelve(im, i, modify=False)
 
 
@@ -248,7 +244,7 @@ def test_skymatch_overlap(mk_sky_match_image_models, skymethod, subtract, skysta
 
     with library:
         for i, (im, lev) in enumerate(zip(library, levels)):
-            im.data = rng.normal(loc=lev, scale=0.01, size=im.data.shape) * im.data.unit
+            im.data = rng.normal(loc=lev, scale=0.01, size=im.data.shape)
             library.shelve(im, i)
 
     # We do not exclude SATURATED pixels. They should be ignored because
@@ -287,23 +283,23 @@ def test_skymatch_overlap(mk_sky_match_image_models, skymethod, subtract, skysta
                 # These two sky methods must fail because they do not take
                 # into account (do not compute) overlap regions and use
                 # entire images:
-                assert abs(im.meta.background.level.value - rlev) < 0.1
+                assert abs(im.meta.background.level - rlev) < 0.1
 
                 # test
                 if subtract:
-                    assert abs(np.mean(im.data[dq_mask]).value - slev) < 0.1
+                    assert abs(np.mean(im.data[dq_mask]) - slev) < 0.1
                 else:
-                    assert abs(np.mean(im.data[dq_mask]).value - lev) < 0.01
+                    assert abs(np.mean(im.data[dq_mask]) - lev) < 0.01
             else:
                 # test computed/measured sky values if level is nonzero:
-                if not np.isclose(im.meta.background.level.value, 0):
-                    assert abs(im.meta.background.level.value - rlev) < 0.01
+                if not np.isclose(im.meta.background.level, 0):
+                    assert abs(im.meta.background.level - rlev) < 0.01
 
                 # test
                 if subtract:
-                    assert abs(np.mean(im.data[dq_mask].value) - slev) < 0.01
+                    assert abs(np.mean(im.data[dq_mask]) - slev) < 0.01
                 else:
-                    assert abs(np.mean(im.data[dq_mask].value) - lev) < 0.01
+                    assert abs(np.mean(im.data[dq_mask]) - lev) < 0.01
             result.shelve(im, i, modify=False)
 
 
@@ -330,7 +326,7 @@ def test_skymatch_2x(wfi_rate, skymethod, subtract):
 
     with library:
         for i, (im, lev) in enumerate(zip(library, levels)):
-            im.data = rng.normal(loc=lev, scale=0.05, size=im.data.shape) * im.data.unit
+            im.data = rng.normal(loc=lev, scale=0.05, size=im.data.shape)
             library.shelve(im, i)
 
     # We do not exclude SATURATED pixels. They should be ignored because
@@ -385,14 +381,14 @@ def test_skymatch_2x(wfi_rate, skymethod, subtract):
             assert im.meta.background.subtracted == step.subtract
 
             # test computed/measured sky values:
-            if not np.isclose(im.meta.background.level.value, 0, atol=1e-6):
-                assert abs(im.meta.background.level.value - rlev) < 0.01
+            if not np.isclose(im.meta.background.level, 0, atol=1e-6):
+                assert abs(im.meta.background.level - rlev) < 0.01
 
             # test
             if subtract:
-                assert abs(np.mean(im.data[dq_mask]).value - slev) < 0.01
+                assert abs(np.mean(im.data[dq_mask]) - slev) < 0.01
             else:
-                assert abs(np.mean(im.data[dq_mask]).value - lev) < 0.01
+                assert abs(np.mean(im.data[dq_mask]) - lev) < 0.01
             result2.shelve(im, i, modify=False)
 
 

From dd04cff674154557ec3bb579ccc4d107e70d7721 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 8 Oct 2024 16:57:43 -0400
Subject: [PATCH 51/74] Point to rad+rdm @ RCAL-932.

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 89185451a..d0096e78d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,9 +24,9 @@ dependencies = [
     "pyparsing >=2.4.7",
     "requests >=2.26",
     # "rad>=0.21.0,<0.22.0",
-    "rad @ git+https://github.com/mairanteodoro/rad.git@RCAL-911-remove-units-from-mosaic-level-pipeline",
+    "rad @ git+https://github.com/mairanteodoro/rad.git@RCAL-932-remove-units-in-exposure-level-pipeline",
     # "roman_datamodels>=0.21.0,<0.22.0",
-    "roman_datamodels @ git+https://github.com/mairanteodoro/roman_datamodels.git@RCAL-911-remove-units-from-mosaic-level-pipeline",
+    "roman_datamodels @ git+https://github.com/mairanteodoro/roman_datamodels.git@RCAL-932-remove-units-in-exposure-level-pipeline",
     "scipy >=1.11",
 #    "stcal>=1.8.0,<1.9.0",
     "stcal @ git+https://github.com/spacetelescope/stcal.git@main",

From 4ba2a7f42e58b0a511405dfffe3b844a0d9ad844 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 8 Oct 2024 20:58:30 +0000
Subject: [PATCH 52/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/jump/tests/test_jump_step.py    |  6 ++----
 romancal/outlier_detection/_fileio.py    |  2 --
 romancal/photom/photom.py                |  4 +++-
 romancal/photom/tests/test_photom.py     | 12 +++++-------
 romancal/regtest/test_regtestdata.py     |  4 ++--
 romancal/skymatch/tests/test_skymatch.py |  6 +++---
 6 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/romancal/jump/tests/test_jump_step.py b/romancal/jump/tests/test_jump_step.py
index 485bf255b..14d48947b 100644
--- a/romancal/jump/tests/test_jump_step.py
+++ b/romancal/jump/tests/test_jump_step.py
@@ -161,8 +161,7 @@ def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs):
     for i in range(len(CR_x_locs)):
         CR_group = next(CR_pool)
         model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = (
-            model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]]
-            + 5000.0
+            model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000.0
         )
 
     out_model = JumpStep.call(
@@ -218,8 +217,7 @@ def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs):
             model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000
         )
         model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] = (
-            model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]]
-            + 700
+            model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] + 700
         )
 
     out_model = JumpStep.call(
diff --git a/romancal/outlier_detection/_fileio.py b/romancal/outlier_detection/_fileio.py
index 48c942b71..0b6fdc857 100644
--- a/romancal/outlier_detection/_fileio.py
+++ b/romancal/outlier_detection/_fileio.py
@@ -1,7 +1,5 @@
 import logging
 
-from astropy.units import Quantity
-
 log = logging.getLogger(__name__)
 log.setLevel(logging.DEBUG)
 
diff --git a/romancal/photom/photom.py b/romancal/photom/photom.py
index 778777080..86ce7618d 100644
--- a/romancal/photom/photom.py
+++ b/romancal/photom/photom.py
@@ -38,7 +38,9 @@ def photom_io(input_model, photom_metadata):
 
     # Store the uncertainty conversion factor in the meta data
     log.info(f"uncertainty value: {uncertainty_conv:.6g}")
-    input_model.meta.photometry.conversion_megajanskys_uncertainty = uncertainty_conv.value
+    input_model.meta.photometry.conversion_megajanskys_uncertainty = (
+        uncertainty_conv.value
+    )
     input_model.meta.photometry.conversion_microjanskys_uncertainty = (
         uncertainty_conv.to(u.microjansky / u.arcsecond**2)
     ).value
diff --git a/romancal/photom/tests/test_photom.py b/romancal/photom/tests/test_photom.py
index 1877e17ec..198c77034 100644
--- a/romancal/photom/tests/test_photom.py
+++ b/romancal/photom/tests/test_photom.py
@@ -118,12 +118,8 @@ def test_no_photom_match():
 
         # Assert that photom elements are not updated
         assert output_model.meta.photometry.pixelarea_steradians == -1.0
-        assert (
-            output_model.meta.photometry.conversion_megajanskys == -1.0
-        )
-        assert (
-            output_model.meta.photometry.conversion_microjanskys_uncertainty == -1.0
-        )
+        assert output_model.meta.photometry.conversion_megajanskys == -1.0
+        assert output_model.meta.photometry.conversion_microjanskys_uncertainty == -1.0
 
 
 def test_apply_photom1():
@@ -281,7 +277,9 @@ def test_photom_step_interface_spectroscopic(instrument, exptype):
     wfi_image.meta.instrument.optical_element = "PRISM"
 
     # Set photometric values for spectroscopic data
-    wfi_image.meta.photometry.pixelarea_steradians = (2.31307642258977e-14 * u.steradian).value
+    wfi_image.meta.photometry.pixelarea_steradians = (
+        2.31307642258977e-14 * u.steradian
+    ).value
     wfi_image.meta.photometry.pixelarea_arcsecsq = (
         0.000984102303070964 * u.arcsecond * u.arcsecond
     ).value
diff --git a/romancal/regtest/test_regtestdata.py b/romancal/regtest/test_regtestdata.py
index 43f7a64ce..d3d768be7 100644
--- a/romancal/regtest/test_regtestdata.py
+++ b/romancal/regtest/test_regtestdata.py
@@ -33,9 +33,9 @@ def test_compare_asdf(tmp_path, modification):
     l2.save(file0)
     atol = 0.0001
     if modification == "small":
-        l2.data += (atol / 2)
+        l2.data += atol / 2
     elif modification == "large":
-        l2.data += (atol * 2)
+        l2.data += atol * 2
     l2.save(file1)
     diff = compare_asdf(file0, file1, atol=atol)
     if modification == "large":
diff --git a/romancal/skymatch/tests/test_skymatch.py b/romancal/skymatch/tests/test_skymatch.py
index 672a06dc2..fc272bb91 100644
--- a/romancal/skymatch/tests/test_skymatch.py
+++ b/romancal/skymatch/tests/test_skymatch.py
@@ -73,9 +73,9 @@ def mk_image_model(
 ):
     l2 = mk_level2_image(shape=image_shape)
     l2_im = ImageModel(l2)
-    l2_im.data = rng.normal(loc=rate_mean, scale=rate_std, size=l2_im.data.shape).astype(
-            np.float32
-        )
+    l2_im.data = rng.normal(
+        loc=rate_mean, scale=rate_std, size=l2_im.data.shape
+    ).astype(np.float32)
 
     l2_im.meta["wcs"] = mk_gwcs(image_shape, sky_offset=sky_offset, rotate=rotation)
 

From 2e37c9884702251078a24594d2dd12218e74af6c Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 8 Oct 2024 17:06:48 -0400
Subject: [PATCH 53/74] Remove LV2_UNITS.

---
 romancal/flux/tests/test_flux_step.py | 31 ++++++---------------------
 1 file changed, 7 insertions(+), 24 deletions(-)

diff --git a/romancal/flux/tests/test_flux_step.py b/romancal/flux/tests/test_flux_step.py
index dcb04dd06..dcfabc1f9 100644
--- a/romancal/flux/tests/test_flux_step.py
+++ b/romancal/flux/tests/test_flux_step.py
@@ -7,7 +7,6 @@
 
 from romancal.datamodels import ModelLibrary
 from romancal.flux import FluxStep
-from romancal.flux.flux_step import LV2_UNITS
 
 
 @pytest.mark.parametrize(
@@ -23,7 +22,7 @@
 def test_attributes(flux_step, attr, factor):
     """Test that the attribute has been scaled by the right factor"""
     original, result = flux_step
-    c_unit = 1.0 / LV2_UNITS
+    c_unit = 1.0
 
     # Handle difference between just a single image and a list.
     if isinstance(original, datamodels.ImageModel):
@@ -90,27 +89,11 @@ def image_model():
     rng = np.random.default_rng()
     shape = (10, 10)
     image_model = maker_utils.mk_datamodel(datamodels.ImageModel, shape=shape)
-    image_model.data = u.Quantity(
-        rng.poisson(2.5, size=shape).astype(np.float32),
-        LV2_UNITS,
-        dtype=np.float32,
-    )
-    image_model.var_rnoise = u.Quantity(
-        rng.normal(1, 0.05, size=shape).astype(np.float32),
-        LV2_UNITS**2,
-        dtype=np.float32,
-    )
-    image_model.var_poisson = u.Quantity(
-        rng.poisson(1, size=shape).astype(np.float32),
-        LV2_UNITS**2,
-        dtype=np.float32,
-    )
-    image_model.var_flat = u.Quantity(
-        rng.uniform(0, 1, size=shape).astype(np.float32),
-        LV2_UNITS**2,
-        dtype=np.float32,
-    )
-    image_model.meta.photometry.conversion_megajanskys = 2.0 * u.MJy / u.sr
+    image_model.data = rng.poisson(2.5, size=shape).astype(np.float32),
+    image_model.var_rnoise = rng.normal(1, 0.05, size=shape).astype(np.float32),
+    image_model.var_poisson = rng.poisson(1, size=shape).astype(np.float32),
+    image_model.var_flat = rng.uniform(0, 1, size=shape).astype(np.float32),
+    image_model.meta.photometry.conversion_megajanskys = (2.0 * u.MJy / u.sr).value
 
     return image_model
 
@@ -129,6 +112,6 @@ def input_modellibrary(image_model):
     # Create and return a ModelLibrary
     image_model1 = image_model.copy()
     image_model2 = image_model.copy()
-    image_model2.meta.photometry.conversion_megajanskys = 0.5 * u.MJy / u.sr
+    image_model2.meta.photometry.conversion_megajanskys = (0.5 * u.MJy / u.sr).value
     container = ModelLibrary([image_model1, image_model2])
     return container

From 75f3eb497835514226df03f3a0196cd221114932 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 8 Oct 2024 21:07:20 +0000
Subject: [PATCH 54/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/flux/tests/test_flux_step.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/romancal/flux/tests/test_flux_step.py b/romancal/flux/tests/test_flux_step.py
index dcfabc1f9..e8ec44cca 100644
--- a/romancal/flux/tests/test_flux_step.py
+++ b/romancal/flux/tests/test_flux_step.py
@@ -89,10 +89,10 @@ def image_model():
     rng = np.random.default_rng()
     shape = (10, 10)
     image_model = maker_utils.mk_datamodel(datamodels.ImageModel, shape=shape)
-    image_model.data = rng.poisson(2.5, size=shape).astype(np.float32),
-    image_model.var_rnoise = rng.normal(1, 0.05, size=shape).astype(np.float32),
-    image_model.var_poisson = rng.poisson(1, size=shape).astype(np.float32),
-    image_model.var_flat = rng.uniform(0, 1, size=shape).astype(np.float32),
+    image_model.data = (rng.poisson(2.5, size=shape).astype(np.float32),)
+    image_model.var_rnoise = (rng.normal(1, 0.05, size=shape).astype(np.float32),)
+    image_model.var_poisson = (rng.poisson(1, size=shape).astype(np.float32),)
+    image_model.var_flat = (rng.uniform(0, 1, size=shape).astype(np.float32),)
     image_model.meta.photometry.conversion_megajanskys = (2.0 * u.MJy / u.sr).value
 
     return image_model

From 17eed0ae17f537d6f56ada6a896802de56e6a71a Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 8 Oct 2024 23:14:46 -0400
Subject: [PATCH 55/74] Bug fix.

---
 romancal/flux/tests/test_flux_step.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/romancal/flux/tests/test_flux_step.py b/romancal/flux/tests/test_flux_step.py
index e8ec44cca..f7209241d 100644
--- a/romancal/flux/tests/test_flux_step.py
+++ b/romancal/flux/tests/test_flux_step.py
@@ -85,14 +85,14 @@ def flux_step(request):
 @pytest.fixture(scope="module")
 def image_model():
     """Product a basic ImageModel"""
-    # Create a random image and specify a conversion.
+    # Create a random image and specify a conversion
     rng = np.random.default_rng()
     shape = (10, 10)
     image_model = maker_utils.mk_datamodel(datamodels.ImageModel, shape=shape)
-    image_model.data = (rng.poisson(2.5, size=shape).astype(np.float32),)
-    image_model.var_rnoise = (rng.normal(1, 0.05, size=shape).astype(np.float32),)
-    image_model.var_poisson = (rng.poisson(1, size=shape).astype(np.float32),)
-    image_model.var_flat = (rng.uniform(0, 1, size=shape).astype(np.float32),)
+    image_model.data = rng.poisson(2.5, size=shape).astype(np.float32)
+    image_model.var_rnoise = rng.normal(1, 0.05, size=shape).astype(np.float32)
+    image_model.var_poisson = rng.poisson(1, size=shape).astype(np.float32)
+    image_model.var_flat = rng.uniform(0, 1, size=shape).astype(np.float32)
     image_model.meta.photometry.conversion_megajanskys = (2.0 * u.MJy / u.sr).value
 
     return image_model

From 9bd0f90205f277b599719841881981359727d282 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Wed, 9 Oct 2024 10:09:25 -0400
Subject: [PATCH 56/74] Fix failing unit tests.

---
 romancal/lib/tests/test_psf.py                     | 14 ++++----------
 romancal/source_catalog/source_catalog.py          |  8 --------
 .../source_catalog/tests/test_source_catalog.py    | 10 ++--------
 3 files changed, 6 insertions(+), 26 deletions(-)

diff --git a/romancal/lib/tests/test_psf.py b/romancal/lib/tests/test_psf.py
index 0738534d1..4eb81d43d 100644
--- a/romancal/lib/tests/test_psf.py
+++ b/romancal/lib/tests/test_psf.py
@@ -29,21 +29,15 @@ def setup_inputs(
     Return ImageModel of level 2 image.
     """
     wfi_image = testutil.mk_level2_image(shape=shape)
-    wfi_image.data = u.Quantity(
-        np.ones(shape, dtype=np.float32), u.DN / u.s, dtype=np.float32
-    )
+    wfi_image.data = np.ones(shape, dtype=np.float32)
     wfi_image.meta.filename = "filename"
     wfi_image.meta.instrument["optical_element"] = "F087"
 
     # add noise to data
     if noise is not None:
         setup_rng = np.random.default_rng(seed or 19)
-        wfi_image.data = u.Quantity(
-            setup_rng.normal(scale=noise, size=shape),
-            u.DN / u.s,
-            dtype=np.float32,
-        )
-        wfi_image.err = noise * np.ones(shape, dtype=np.float32) * u.DN / u.s
+        wfi_image.data = setup_rng.normal(scale=noise, size=shape).astype("float32")
+        wfi_image.err = noise * (np.ones(shape, dtype=np.float32) * u.DN / u.s).value
 
     # add dq array
     wfi_image.dq = np.zeros(shape, dtype=np.uint32)
@@ -106,7 +100,7 @@ def setup_method(self):
     def test_psf_fit(self, dx, dy, true_flux):
         # generate an ImageModel
         image_model = deepcopy(self.image_model)
-        init_data_stddev = np.std(image_model.data.value)
+        init_data_stddev = np.std(image_model.data)
 
         # add synthetic sources to the ImageModel:
         true_x = image_model_shape[0] / 2 + dx
diff --git a/romancal/source_catalog/source_catalog.py b/romancal/source_catalog/source_catalog.py
index b0112349f..e0e5ae0e3 100644
--- a/romancal/source_catalog/source_catalog.py
+++ b/romancal/source_catalog/source_catalog.py
@@ -167,10 +167,6 @@ def convert_l2_to_sb(self):
         Convert a level-2 image from units of DN/s to MJy/sr (surface
         brightness).
         """
-        if self.model.data.unit != self.l2_unit or self.model.err.unit != self.l2_unit:
-            raise ValueError(
-                f"data and err are expected to be in units of {self.l2_unit}"
-            )
 
         # the conversion in done in-place to avoid making copies of the data;
         # use a dictionary to set the value to avoid on-the-fly validation
@@ -185,10 +181,6 @@ def convert_sb_to_l2(self):
 
         This is the inverse operation of `convert_l2_to_sb`.
         """
-        if self.model.data.unit != self.sb_unit or self.model.err.unit != self.sb_unit:
-            raise ValueError(
-                f"data and err are expected to be in units of {self.sb_unit}"
-            )
 
         # the conversion in done in-place to avoid making copies of the data;
         # use a dictionary to set the value to avoid on-the-fly validation
diff --git a/romancal/source_catalog/tests/test_source_catalog.py b/romancal/source_catalog/tests/test_source_catalog.py
index f1dfd15bf..13580f474 100644
--- a/romancal/source_catalog/tests/test_source_catalog.py
+++ b/romancal/source_catalog/tests/test_source_catalog.py
@@ -73,12 +73,9 @@ def mosaic_model():
     wfi_mosaic = mk_level3_mosaic(shape=(101, 101))
     model = MosaicModel(wfi_mosaic)
     data, err = make_test_image()
-    units = u.MJy / u.sr
-    data <<= units
-    err <<= units
     model.data = data
     model.err = err
-    model.weight = 1.0 / err.value
+    model.weight = 1.0 / err
     return model
 
 
@@ -87,12 +84,9 @@ def image_model():
     wfi_image = mk_level2_image(shape=(101, 101))
     model = ImageModel(wfi_image)
     data, err = make_test_image()
-    units = u.DN / u.s
-    data <<= units
-    err <<= units
     model.data = data
     model.err = err
-    model.meta.photometry.conversion_megajanskys = 0.3324 * u.MJy / u.sr
+    model.meta.photometry.conversion_megajanskys = (0.3324 * u.MJy / u.sr).value
     return model
 
 

From 3226fa53d08535c4051255dc1b3f1dd8c77a56d2 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Thu, 10 Oct 2024 08:39:05 -0400
Subject: [PATCH 57/74] Remove regtest helper function.

---
 romancal/regtest/test_mos_pipeline.py     | 87 +----------------------
 romancal/source_catalog/source_catalog.py |  4 +-
 2 files changed, 3 insertions(+), 88 deletions(-)

diff --git a/romancal/regtest/test_mos_pipeline.py b/romancal/regtest/test_mos_pipeline.py
index 3cd150d74..86c206d38 100644
--- a/romancal/regtest/test_mos_pipeline.py
+++ b/romancal/regtest/test_mos_pipeline.py
@@ -19,86 +19,6 @@
 pytestmark = [pytest.mark.bigdata, pytest.mark.soctests]
 
 
-class RegtestFileModifier:
-    # TODO: remove this entire class once the units
-    #  have been removed from the regtest files
-
-    def __init__(self, rtdata):
-        self.rtdata = rtdata
-        self.updated_asn_fname = None
-        self.truth_parent = Path(rtdata.truth).parent
-        self.input_parent = Path(rtdata.input).parent
-        self.truth_relative_path = Path(self.truth_parent).relative_to(
-            self.input_parent
-        )
-        self.truth_path = self.truth_relative_path / f"{Path(self.rtdata.truth).name}"
-
-    @staticmethod
-    def create_unitless_file(input_filename: str, output_filename: str) -> None:
-        with asdf.config_context() as cfg:
-            cfg.validate_on_read = False
-            cfg.validate_on_save = False
-            af = asdf.open(input_filename)
-
-            for attr in af.tree["roman"]:
-                item = getattr(af.tree["roman"], attr)
-                if isinstance(item, Quantity):
-                    setattr(af.tree["roman"], attr, item.value)
-
-            for attr in af.tree["roman"].meta.photometry:
-                item = getattr(af.tree["roman"].meta.photometry, attr)
-                if isinstance(item, Quantity):
-                    setattr(af.tree["roman"].meta.photometry, attr, item.value)
-
-            af.write_to(output_filename)
-
-    def create_new_asn_file(self, output_filename_list: list):
-        updated_asn = asn_from_list(
-            output_filename_list,
-            product_name=f"{self.rtdata.asn['products'][0]['name']}_no_units",
-        )
-        updated_asn["target"] = "none"
-
-        current_asn_fname = Path(self.rtdata.input)
-        self.updated_asn_fname = (
-            f"{current_asn_fname.stem}_no_units{current_asn_fname.suffix}"
-        )
-
-        _, serialized_updated_asn = asn_json.dump(updated_asn)
-        with open(self.updated_asn_fname, "w") as f:
-            json.dump(
-                json.loads(serialized_updated_asn), f, indent=4, separators=(",", ": ")
-            )
-
-    def update_rtdata(self):
-        rtdata_root_path = Path(self.rtdata.input).parent
-        self.rtdata.input = f"{rtdata_root_path}/{Path(self.updated_asn_fname)}"
-        # r0099101001001001001_F158_visit_no_units_i2d.asdf
-        self.rtdata.output = f"{rtdata_root_path}/{Path(self.rtdata.output.split('_i2d')[0]).stem}_no_units_i2d{Path(self.rtdata.output).suffix}"
-
-    def prepare_regtest_input_files(self):
-        input_filenames = [
-            x["expname"] for x in self.rtdata.asn["products"][0]["members"]
-        ]
-        input_filenames.append(str(self.truth_path))
-        output_filename_list = []
-        # include truth file
-        for input_filename in input_filenames:
-            fname = Path(input_filename)
-            if str(fname).startswith(str(self.truth_relative_path)):
-                output_filename = Path(
-                    f"{str(fname).split('_i2d.asdf')[0]}_no_units_i2d{fname.suffix}"
-                )
-                self.rtdata.truth = str(self.truth_parent / output_filename.name)
-            else:
-                output_filename = f"{fname.stem}_no_units{fname.suffix}"
-                output_filename_list.append(output_filename)
-            self.create_unitless_file(input_filename, output_filename)
-
-        self.create_new_asn_file(output_filename_list)
-        self.update_rtdata()
-
-
 @pytest.fixture(scope="module")
 def run_mos(rtdata_module):
     rtdata = rtdata_module
@@ -109,18 +29,13 @@ def run_mos(rtdata_module):
     output = "r0099101001001001001_F158_visit_i2d.asdf"
     rtdata.output = output
 
-    rtdata.get_truth(f"truth/WFI/image/{output}")
-
-    fixer = RegtestFileModifier(rtdata)
-    fixer.prepare_regtest_input_files()
-
     args = [
         "roman_mos",
         rtdata.input,
     ]
     MosaicPipeline.from_cmdline(args)
 
-    # rtdata.get_truth(f"truth/WFI/image/{output}")
+    rtdata.get_truth(f"truth/WFI/image/{output}")
     return rtdata
 
 
diff --git a/romancal/source_catalog/source_catalog.py b/romancal/source_catalog/source_catalog.py
index e0e5ae0e3..7e3b6243d 100644
--- a/romancal/source_catalog/source_catalog.py
+++ b/romancal/source_catalog/source_catalog.py
@@ -159,8 +159,8 @@ def pixel_area(self):
         """
         pixel_area = self.model.meta.photometry.pixelarea_steradians
         if pixel_area < 0:
-            pixel_area = (self._pixel_scale**2).to(u.sr)
-        return pixel_area.value
+            pixel_area = (self._pixel_scale**2).to(u.sr).value
+        return pixel_area
 
     def convert_l2_to_sb(self):
         """

From f0f1435504266b817d6640a8e30ba91cc7b8a8dc Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 10 Oct 2024 12:39:40 +0000
Subject: [PATCH 58/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/regtest/test_mos_pipeline.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/romancal/regtest/test_mos_pipeline.py b/romancal/regtest/test_mos_pipeline.py
index 86c206d38..1e22438ac 100644
--- a/romancal/regtest/test_mos_pipeline.py
+++ b/romancal/regtest/test_mos_pipeline.py
@@ -1,18 +1,12 @@
 """ Roman tests for the High Level Pipeline """
 
-import json
 import os
-from pathlib import Path
 
-import asdf
 import pytest
 import roman_datamodels as rdm
-from astropy.units import Quantity
 
-from romancal.associations.asn_from_list import asn_from_list
 from romancal.pipeline.mosaic_pipeline import MosaicPipeline
 
-from ..associations.association_io import json as asn_json
 from .regtestdata import compare_asdf
 
 # mark all tests in this module

From 5921a2cb3d6771ef2371307ff6e51b1f84f04c5e Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Thu, 10 Oct 2024 19:24:13 -0400
Subject: [PATCH 59/74] Fixes for the regression tests.

---
 romancal/regtest/test_mos_skycell_pipeline.py |  2 +-
 romancal/regtest/test_tweakreg.py             |  2 +
 romancal/regtest/test_wfi_photom.py           | 65 ++++++-------------
 romancal/scripts/make_regtestdata.sh          |  8 +--
 4 files changed, 26 insertions(+), 51 deletions(-)

diff --git a/romancal/regtest/test_mos_skycell_pipeline.py b/romancal/regtest/test_mos_skycell_pipeline.py
index 712a69f55..da542af3b 100644
--- a/romancal/regtest/test_mos_skycell_pipeline.py
+++ b/romancal/regtest/test_mos_skycell_pipeline.py
@@ -15,7 +15,7 @@ def run_mos(rtdata_module):
 
     # Test Pipeline
     rtdata.get_asn("WFI/image/L3_mosaic_asn.json")
-    output = "r0099101001001001001_r274dp63x31y81_prompt_F158_i2d.asdf"
+    output = "mosaic_i2d.asdf"
     rtdata.output = output
     args = [
         "roman_mos",
diff --git a/romancal/regtest/test_tweakreg.py b/romancal/regtest/test_tweakreg.py
index 7adcc2ce3..1eb51d711 100644
--- a/romancal/regtest/test_tweakreg.py
+++ b/romancal/regtest/test_tweakreg.py
@@ -17,11 +17,13 @@ def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path):
     # the wcsinfo is perturbed, and AssignWCS is run to update the WCS with the
     # perturbed information
     orig_uncal = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
+    orig_catfile = "r0000101001001001001_01101_0001_WFI01_cat.asdf"
     input_data = "r0000101001001001001_01101_0001_WFI01_shift_cal.asdf"
     output_data = "r0000101001001001001_01101_0001_WFI01_shift_tweakregstep.asdf"
     truth_data = "r0000101001001001001_01101_0001_WFI01_shift_tweakregstep.asdf"
 
     rtdata.get_data(f"WFI/image/{input_data}")
+    rtdata.get_data(f"WFI/image/{orig_catfile}")
     rtdata.get_data(f"WFI/image/{orig_uncal}")
     rtdata.get_truth(f"truth/WFI/image/{truth_data}")
 
diff --git a/romancal/regtest/test_wfi_photom.py b/romancal/regtest/test_wfi_photom.py
index 81f888dc1..5731be785 100644
--- a/romancal/regtest/test_wfi_photom.py
+++ b/romancal/regtest/test_wfi_photom.py
@@ -60,40 +60,33 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Photom megajansky conversion calculated? : "
         + str(
-            (photom_out.meta.photometry.conversion_megajanskys.unit == u.MJy / u.sr)
-            and math.isclose(
-                photom_out.meta.photometry.conversion_megajanskys.value,
-                0.3324,
-                abs_tol=0.0001,
+            (
+                math.isclose(
+                    photom_out.meta.photometry.conversion_megajanskys,
+                    0.3324,
+                    abs_tol=0.0001,
+                    )
             )
         )
     )
-    assert photom_out.meta.photometry.conversion_megajanskys.unit == u.MJy / u.sr
     assert math.isclose(
-        photom_out.meta.photometry.conversion_megajanskys.value, 0.3324, abs_tol=0.0001
+        photom_out.meta.photometry.conversion_megajanskys, 0.3324, abs_tol=0.0001
     )
 
     step.log.info(
         "DMS140 MSG: Photom microjanskys conversion calculated? : "
         + str(
             (
-                photom_out.meta.photometry.conversion_microjanskys.unit
-                == u.uJy / u.arcsec**2
-            )
-            and (
                 math.isclose(
-                    photom_out.meta.photometry.conversion_microjanskys.value,
+                    photom_out.meta.photometry.conversion_microjanskys,
                     7.81320,
                     abs_tol=0.0001,
                 )
             )
         )
     )
-    assert (
-        photom_out.meta.photometry.conversion_microjanskys.unit == u.uJy / u.arcsec**2
-    )
     assert math.isclose(
-        photom_out.meta.photometry.conversion_microjanskys.value,
+        photom_out.meta.photometry.conversion_microjanskys,
         7.81320,
         abs_tol=0.0001,
     )
@@ -101,19 +94,17 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Pixel area in steradians calculated? : "
         + str(
-            (photom_out.meta.photometry.pixelarea_steradians.unit == u.sr)
-            and (
+            (
                 math.isclose(
-                    photom_out.meta.photometry.pixelarea_steradians.value,
+                    photom_out.meta.photometry.pixelarea_steradians,
                     2.8083e-13,
                     abs_tol=1.0e-17,
                 )
             )
         )
     )
-    assert photom_out.meta.photometry.pixelarea_steradians.unit == u.sr
     assert math.isclose(
-        photom_out.meta.photometry.pixelarea_steradians.value,
+        photom_out.meta.photometry.pixelarea_steradians,
         2.8083e-13,
         abs_tol=1.0e-17,
     )
@@ -121,43 +112,33 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Pixel area in square arcseconds calculated? : "
         + str(
-            (photom_out.meta.photometry.pixelarea_arcsecsq.unit == u.arcsec**2)
-            and (
+            (
                 math.isclose(
-                    photom_out.meta.photometry.pixelarea_arcsecsq.value,
+                    photom_out.meta.photometry.pixelarea_arcsecsq,
                     0.011948,
                     abs_tol=1.0e-6,
                 )
             )
         )
     )
-    assert photom_out.meta.photometry.pixelarea_arcsecsq.unit == u.arcsec**2
     assert math.isclose(
-        photom_out.meta.photometry.pixelarea_arcsecsq.value, 0.011948, abs_tol=1.0e-6
+        photom_out.meta.photometry.pixelarea_arcsecsq, 0.011948, abs_tol=1.0e-6
     )
 
     step.log.info(
         "DMS140 MSG: Photom megajansky conversion uncertainty calculated? : "
         + str(
             (
-                photom_out.meta.photometry.conversion_megajanskys_uncertainty.unit
-                == u.MJy / u.sr
-            )
-            and (
                 math.isclose(
-                    photom_out.meta.photometry.conversion_megajanskys_uncertainty.value,
+                    photom_out.meta.photometry.conversion_megajanskys_uncertainty,
                     0.0,
                     abs_tol=1.0e-6,
                 )
             )
         )
     )
-    assert (
-        photom_out.meta.photometry.conversion_megajanskys_uncertainty.unit
-        == u.MJy / u.sr
-    )
     assert math.isclose(
-        photom_out.meta.photometry.conversion_megajanskys_uncertainty.value,
+        photom_out.meta.photometry.conversion_megajanskys_uncertainty,
         0.0,
         abs_tol=1.0e-6,
     )
@@ -166,24 +147,16 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
         "DMS140 MSG: Photom megajansky conversion uncertainty calculated? : "
         + str(
             (
-                photom_out.meta.photometry.conversion_microjanskys_uncertainty.unit
-                == u.uJy / u.arcsec**2
-            )
-            and (
                 math.isclose(
-                    photom_out.meta.photometry.conversion_microjanskys_uncertainty.value,
+                    photom_out.meta.photometry.conversion_microjanskys_uncertainty,
                     0.0,
                     abs_tol=1.0e-6,
                 )
             )
         )
     )
-    assert (
-        photom_out.meta.photometry.conversion_microjanskys_uncertainty.unit
-        == u.uJy / u.arcsec**2
-    )
     assert math.isclose(
-        photom_out.meta.photometry.conversion_microjanskys_uncertainty.value,
+        photom_out.meta.photometry.conversion_microjanskys_uncertainty,
         0.0,
         abs_tol=1.0e-6,
     )
diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh
index 348e1269a..15b81e7cb 100644
--- a/romancal/scripts/make_regtestdata.sh
+++ b/romancal/scripts/make_regtestdata.sh
@@ -57,9 +57,9 @@ cp r0000101001001001001_01101_0002_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI
 # resample regtest; needs r0000101001001001001_01101_000{1,2}_WFI01_cal.asdf
 # builds the appropriate asn file and calls strun with it
 echo "Creating regtest files for resample..."
-asn_from_list r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf -o mosaic_asn.json --product-name mosaic
-strun romancal.step.ResampleStep mosaic_asn.json --rotation=0 --output_file=mosaic.asdf
-cp mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/
+asn_from_list r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic
+strun romancal.step.ResampleStep L3_mosaic_asn.json --rotation=0 --output_file=mosaic.asdf
+cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/
 cp mosaic_resamplestep.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 
@@ -187,7 +187,7 @@ cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 strun romancal.step.SourceCatalogStep r0000101001001001001_01101_0001_WFI01_cal.asdf
 cp r0000101001001001001_01101_0001_WFI01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
-l3name="r0099101001001001001_F158_visit_r274dp63x31y81"
+l3name="mosaic"
 asn_from_list --product-name=$l3name r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf r0000101001001001001_01101_0003_WFI01_cal.asdf -o L3_m1_asn.json
 strun roman_mos L3_m1_asn.json
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/

From ef289fac8c75bfe326bd98299e9e0e2acdd6be83 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 10 Oct 2024 23:24:46 +0000
Subject: [PATCH 60/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/regtest/test_wfi_photom.py | 61 ++++++++++++-----------------
 1 file changed, 24 insertions(+), 37 deletions(-)

diff --git a/romancal/regtest/test_wfi_photom.py b/romancal/regtest/test_wfi_photom.py
index 5731be785..f7c064c87 100644
--- a/romancal/regtest/test_wfi_photom.py
+++ b/romancal/regtest/test_wfi_photom.py
@@ -4,7 +4,6 @@
 
 import pytest
 import roman_datamodels as rdm
-from astropy import units as u
 
 from romancal.step import PhotomStep
 from romancal.stpipe import RomanStep
@@ -60,12 +59,10 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Photom megajansky conversion calculated? : "
         + str(
-            (
-                math.isclose(
-                    photom_out.meta.photometry.conversion_megajanskys,
-                    0.3324,
-                    abs_tol=0.0001,
-                    )
+            math.isclose(
+                photom_out.meta.photometry.conversion_megajanskys,
+                0.3324,
+                abs_tol=0.0001,
             )
         )
     )
@@ -76,12 +73,10 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Photom microjanskys conversion calculated? : "
         + str(
-            (
-                math.isclose(
-                    photom_out.meta.photometry.conversion_microjanskys,
-                    7.81320,
-                    abs_tol=0.0001,
-                )
+            math.isclose(
+                photom_out.meta.photometry.conversion_microjanskys,
+                7.81320,
+                abs_tol=0.0001,
             )
         )
     )
@@ -94,12 +89,10 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Pixel area in steradians calculated? : "
         + str(
-            (
-                math.isclose(
-                    photom_out.meta.photometry.pixelarea_steradians,
-                    2.8083e-13,
-                    abs_tol=1.0e-17,
-                )
+            math.isclose(
+                photom_out.meta.photometry.pixelarea_steradians,
+                2.8083e-13,
+                abs_tol=1.0e-17,
             )
         )
     )
@@ -112,12 +105,10 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Pixel area in square arcseconds calculated? : "
         + str(
-            (
-                math.isclose(
-                    photom_out.meta.photometry.pixelarea_arcsecsq,
-                    0.011948,
-                    abs_tol=1.0e-6,
-                )
+            math.isclose(
+                photom_out.meta.photometry.pixelarea_arcsecsq,
+                0.011948,
+                abs_tol=1.0e-6,
             )
         )
     )
@@ -128,12 +119,10 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Photom megajansky conversion uncertainty calculated? : "
         + str(
-            (
-                math.isclose(
-                    photom_out.meta.photometry.conversion_megajanskys_uncertainty,
-                    0.0,
-                    abs_tol=1.0e-6,
-                )
+            math.isclose(
+                photom_out.meta.photometry.conversion_megajanskys_uncertainty,
+                0.0,
+                abs_tol=1.0e-6,
             )
         )
     )
@@ -146,12 +135,10 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     step.log.info(
         "DMS140 MSG: Photom megajansky conversion uncertainty calculated? : "
         + str(
-            (
-                math.isclose(
-                    photom_out.meta.photometry.conversion_microjanskys_uncertainty,
-                    0.0,
-                    abs_tol=1.0e-6,
-                )
+            math.isclose(
+                photom_out.meta.photometry.conversion_microjanskys_uncertainty,
+                0.0,
+                abs_tol=1.0e-6,
             )
         )
     )

From a75564027dc6be76d843f6865afd9f7bd927b61d Mon Sep 17 00:00:00 2001
From: Eddie Schlafly <eschlafly@stsci.edu>
Date: Fri, 11 Oct 2024 09:25:06 -0400
Subject: [PATCH 61/74] Remove ggsaa from file names.

---
 romancal/regtest/test_catalog.py              |  2 +-
 romancal/regtest/test_dark_current.py         | 14 ++--
 romancal/regtest/test_jump_det.py             |  4 +-
 romancal/regtest/test_linearity.py            |  6 +-
 romancal/regtest/test_ramp_fitting.py         |  8 +-
 romancal/regtest/test_refpix.py               |  4 +-
 romancal/regtest/test_tweakreg.py             | 10 +--
 romancal/regtest/test_wfi_dq_init.py          |  8 +-
 romancal/regtest/test_wfi_flat_field.py       | 16 ++--
 .../regtest/test_wfi_grism_16resultants.py    |  4 +-
 romancal/regtest/test_wfi_grism_pipeline.py   |  4 +-
 .../regtest/test_wfi_image_16resultants.py    |  4 +-
 romancal/regtest/test_wfi_image_pipeline.py   | 16 ++--
 romancal/regtest/test_wfi_photom.py           |  4 +-
 romancal/regtest/test_wfi_saturation.py       |  8 +-
 romancal/scripts/make_regtestdata.sh          | 75 +++++++++++--------
 16 files changed, 100 insertions(+), 87 deletions(-)

diff --git a/romancal/regtest/test_catalog.py b/romancal/regtest/test_catalog.py
index 6ba270577..3c461af1c 100644
--- a/romancal/regtest/test_catalog.py
+++ b/romancal/regtest/test_catalog.py
@@ -13,7 +13,7 @@
     scope="module",
     params=[
         "r0099101001001001001_F158_visit_i2d.asdf",
-        "r0000101001001001001_01101_0001_WFI01_cal.asdf",
+        "r0000101001001001001_0001_WFI01_cal.asdf",
     ],
     ids=["L3", "L2"],
 )
diff --git a/romancal/regtest/test_dark_current.py b/romancal/regtest/test_dark_current.py
index 3e31501a4..86200bc71 100644
--- a/romancal/regtest/test_dark_current.py
+++ b/romancal/regtest/test_dark_current.py
@@ -12,13 +12,13 @@ def test_dark_current_subtraction_step(rtdata, ignore_asdf_paths):
     """Function to run and compare Dark Current subtraction files. Note: This
     should include tests for overrides etc."""
 
-    input_datafile = "r0000101001001001001_01101_0001_WFI01_linearity.asdf"
+    input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf"
     rtdata.get_data(f"WFI/image/{input_datafile}")
     rtdata.input = input_datafile
 
     args = ["romancal.step.DarkCurrentStep", rtdata.input]
     RomanStep.from_cmdline(args)
-    output = "r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf"
+    output = "r0000101001001001001_0001_WFI01_darkcurrent.asdf"
     rtdata.output = output
     rtdata.get_truth(f"truth/WFI/image/{output}")
     diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths)
@@ -29,7 +29,7 @@ def test_dark_current_subtraction_step(rtdata, ignore_asdf_paths):
 def test_dark_current_outfile_step(rtdata, ignore_asdf_paths):
     """Function to run and compare Dark Current subtraction files. Here the
     test is for renaming the output file."""
-    input_datafile = "r0000101001001001001_01101_0001_WFI01_linearity.asdf"
+    input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf"
     rtdata.get_data(f"WFI/image/{input_datafile}")
     rtdata.input = input_datafile
 
@@ -50,7 +50,7 @@ def test_dark_current_outfile_step(rtdata, ignore_asdf_paths):
 def test_dark_current_outfile_suffix(rtdata, ignore_asdf_paths):
     """Function to run and compare Dark Current subtraction files. Here the
     test is for renaming the output file."""
-    input_datafile = "r0000101001001001001_01101_0001_WFI01_linearity.asdf"
+    input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf"
     rtdata.get_data(f"WFI/image/{input_datafile}")
     rtdata.input = input_datafile
 
@@ -73,10 +73,10 @@ def test_dark_current_output(rtdata, ignore_asdf_paths):
     """Function to run and compare Dark Current subtraction files. Here the
     test for overriding the CRDS dark reference file."""
 
-    input_datafile = "r0000101001001001001_01101_0001_WFI01_linearity.asdf"
+    input_datafile = "r0000101001001001001_0001_WFI01_linearity.asdf"
     rtdata.get_data(f"WFI/image/{input_datafile}")
     rtdata.input = input_datafile
-    dark_output_name = "r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf"
+    dark_output_name = "r0000101001001001001_0001_WFI01_darkcurrent.asdf"
 
     args = [
         "romancal.step.DarkCurrentStep",
@@ -84,7 +84,7 @@ def test_dark_current_output(rtdata, ignore_asdf_paths):
         f"--dark_output={dark_output_name}",
     ]
     RomanStep.from_cmdline(args)
-    output = "r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf"
+    output = "r0000101001001001001_0001_WFI01_darkcurrent.asdf"
     rtdata.output = output
     rtdata.get_truth(f"truth/WFI/image/{output}")
     diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths)
diff --git a/romancal/regtest/test_jump_det.py b/romancal/regtest/test_jump_det.py
index 7d892de1f..4c014d161 100644
--- a/romancal/regtest/test_jump_det.py
+++ b/romancal/regtest/test_jump_det.py
@@ -13,7 +13,7 @@ def test_jump_detection_step(rtdata, ignore_asdf_paths):
     """Function to run and compare Jump Detection files. Note: This should
     include tests for overrides etc."""
 
-    input_file = "r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf"
+    input_file = "r0000101001001001001_0001_WFI01_darkcurrent.asdf"
     rtdata.get_data(f"WFI/image/{input_file}")
     rtdata.input = input_file
 
@@ -29,7 +29,7 @@ def test_jump_detection_step(rtdata, ignore_asdf_paths):
         "--run_ramp_jump_detection=False",
     ]
     RomanStep.from_cmdline(args)
-    output = "r0000101001001001001_01101_0001_WFI01_jump.asdf"
+    output = "r0000101001001001001_0001_WFI01_jump.asdf"
     rtdata.output = output
     rtdata.get_truth(f"truth/WFI/image/{output}")
     diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths)
diff --git a/romancal/regtest/test_linearity.py b/romancal/regtest/test_linearity.py
index aad0a88a2..b6c9b5cdc 100644
--- a/romancal/regtest/test_linearity.py
+++ b/romancal/regtest/test_linearity.py
@@ -10,13 +10,13 @@
 @pytest.mark.bigdata
 def test_linearity_step(rtdata, ignore_asdf_paths):
     """Function to run and compare linearity correction files."""
-    input_file = "r0000101001001001001_01101_0001_WFI01_refpix.asdf"
+    input_file = "r0000101001001001001_0001_WFI01_refpix.asdf"
     rtdata.get_data(f"WFI/image/{input_file}")
     rtdata.input = input_file
 
     args = ["romancal.step.LinearityStep", rtdata.input]
     RomanStep.from_cmdline(args)
-    output = "r0000101001001001001_01101_0001_WFI01_linearity.asdf"
+    output = "r0000101001001001001_0001_WFI01_linearity.asdf"
     rtdata.output = output
     rtdata.get_truth(f"truth/WFI/image/{output}")
     diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths)
@@ -27,7 +27,7 @@ def test_linearity_step(rtdata, ignore_asdf_paths):
 def test_linearity_outfile_step(rtdata, ignore_asdf_paths):
     """Function to run and linearity correction files. Here the
     test is for renaming the output file."""
-    input_file = "r0000101001001001001_01101_0001_WFI01_refpix.asdf"
+    input_file = "r0000101001001001001_0001_WFI01_refpix.asdf"
     rtdata.get_data(f"WFI/image/{input_file}")
     rtdata.input = input_file
 
diff --git a/romancal/regtest/test_ramp_fitting.py b/romancal/regtest/test_ramp_fitting.py
index e1e79e767..6b76f8d20 100644
--- a/romancal/regtest/test_ramp_fitting.py
+++ b/romancal/regtest/test_ramp_fitting.py
@@ -100,22 +100,22 @@ def cond_science_verification(
     params=[
         (
             "DMS362",
-            Path("WFI/image/r0000101001001001001_01101_0001_WFI01_darkcurrent.asdf"),
+            Path("WFI/image/r0000101001001001001_0001_WFI01_darkcurrent.asdf"),
             CONDITIONS_FULL,
         ),
         (
             "DMS366",
-            Path("WFI/grism/r0000201001001001001_01101_0001_WFI01_darkcurrent.asdf"),
+            Path("WFI/grism/r0000201001001001001_0001_WFI01_darkcurrent.asdf"),
             CONDITIONS_FULL,
         ),
         (
             "DMS363",
-            Path("WFI/image/r0000101001001001001_01101_0003_WFI01_darkcurrent.asdf"),
+            Path("WFI/image/r0000101001001001001_0003_WFI01_darkcurrent.asdf"),
             CONDITIONS_TRUNC,
         ),
         (
             "DMS367",
-            Path("WFI/grism/r0000201001001001001_01101_0003_WFI01_darkcurrent.asdf"),
+            Path("WFI/grism/r0000201001001001001_0003_WFI01_darkcurrent.asdf"),
             CONDITIONS_TRUNC,
         ),
     ],
diff --git a/romancal/regtest/test_refpix.py b/romancal/regtest/test_refpix.py
index ec87d0cbc..b8c7c44d3 100644
--- a/romancal/regtest/test_refpix.py
+++ b/romancal/regtest/test_refpix.py
@@ -10,7 +10,7 @@
 @pytest.mark.bigdata
 def test_refpix_step(rtdata, ignore_asdf_paths):
     # I have no idea what this is supposed to be
-    input_datafile = "r0000101001001001001_01101_0001_WFI01_saturation.asdf"
+    input_datafile = "r0000101001001001001_0001_WFI01_saturation.asdf"
     rtdata.get_data(f"WFI/image/{input_datafile}")
     rtdata.input = input_datafile
 
@@ -18,7 +18,7 @@ def test_refpix_step(rtdata, ignore_asdf_paths):
     RomanStep.from_cmdline(args)
 
     # Again I have no idea here
-    output = "r0000101001001001001_01101_0001_WFI01_refpix.asdf"
+    output = "r0000101001001001001_0001_WFI01_refpix.asdf"
     rtdata.output = output
     rtdata.get_truth(f"truth/WFI/image/{output}")
 
diff --git a/romancal/regtest/test_tweakreg.py b/romancal/regtest/test_tweakreg.py
index 1eb51d711..9402c16d7 100644
--- a/romancal/regtest/test_tweakreg.py
+++ b/romancal/regtest/test_tweakreg.py
@@ -16,11 +16,11 @@ def test_tweakreg(rtdata, ignore_asdf_paths, tmp_path):
     # ``shifted'' version is created in make_regtestdata.sh; cal file is taken,
     # the wcsinfo is perturbed, and AssignWCS is run to update the WCS with the
     # perturbed information
-    orig_uncal = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
-    orig_catfile = "r0000101001001001001_01101_0001_WFI01_cat.asdf"
-    input_data = "r0000101001001001001_01101_0001_WFI01_shift_cal.asdf"
-    output_data = "r0000101001001001001_01101_0001_WFI01_shift_tweakregstep.asdf"
-    truth_data = "r0000101001001001001_01101_0001_WFI01_shift_tweakregstep.asdf"
+    orig_uncal = "r0000101001001001001_0001_WFI01_uncal.asdf"
+    orig_catfile = "r0000101001001001001_0001_WFI01_cat.asdf"
+    input_data = "r0000101001001001001_0001_WFI01_shift_cal.asdf"
+    output_data = "r0000101001001001001_0001_WFI01_shift_tweakregstep.asdf"
+    truth_data = "r0000101001001001001_0001_WFI01_shift_tweakregstep.asdf"
 
     rtdata.get_data(f"WFI/image/{input_data}")
     rtdata.get_data(f"WFI/image/{orig_catfile}")
diff --git a/romancal/regtest/test_wfi_dq_init.py b/romancal/regtest/test_wfi_dq_init.py
index b05eabce9..5f9ba08dd 100644
--- a/romancal/regtest/test_wfi_dq_init.py
+++ b/romancal/regtest/test_wfi_dq_init.py
@@ -16,7 +16,7 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths):
     """DMS25 Test: Testing retrieval of best ref file for image data,
     and creation of a ramp file with CRDS selected mask file applied."""
 
-    input_file = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
+    input_file = "r0000101001001001001_0001_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/image/{input_file}")
     rtdata.input = input_file
 
@@ -40,7 +40,7 @@ def test_dq_init_image_step(rtdata, ignore_asdf_paths):
     assert "roman_wfi_mask" in ref_file_name
 
     # Test DQInitStep
-    output = "r0000101001001001001_01101_0001_WFI01_dqinit.asdf"
+    output = "r0000101001001001001_0001_WFI01_dqinit.asdf"
     rtdata.output = output
     args = ["romancal.step.DQInitStep", rtdata.input]
     step.log.info(
@@ -71,7 +71,7 @@ def test_dq_init_grism_step(rtdata, ignore_asdf_paths):
     """DMS25 Test: Testing retrieval of best ref file for grism data,
     and creation of a ramp file with CRDS selected mask file applied."""
 
-    input_file = "r0000201001001001001_01101_0001_WFI01_uncal.asdf"
+    input_file = "r0000201001001001001_0001_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/grism/{input_file}")
     rtdata.input = input_file
 
@@ -95,7 +95,7 @@ def test_dq_init_grism_step(rtdata, ignore_asdf_paths):
     assert "roman_wfi_mask" in ref_file_name
 
     # Test DQInitStep
-    output = "r0000201001001001001_01101_0001_WFI01_dqinit.asdf"
+    output = "r0000201001001001001_0001_WFI01_dqinit.asdf"
     rtdata.output = output
     args = ["romancal.step.DQInitStep", rtdata.input]
     step.log.info(
diff --git a/romancal/regtest/test_wfi_flat_field.py b/romancal/regtest/test_wfi_flat_field.py
index 323e2d17e..f9b01a1a1 100644
--- a/romancal/regtest/test_wfi_flat_field.py
+++ b/romancal/regtest/test_wfi_flat_field.py
@@ -14,7 +14,7 @@
 def test_flat_field_image_step(rtdata, ignore_asdf_paths):
     """Test for the flat field step using imaging data."""
 
-    input_data = "r0000101001001001001_01101_0001_WFI01_assignwcs.asdf"
+    input_data = "r0000101001001001001_0001_WFI01_assignwcs.asdf"
     rtdata.get_data(f"WFI/image/{input_data}")
     rtdata.input = input_data
 
@@ -25,7 +25,7 @@ def test_flat_field_image_step(rtdata, ignore_asdf_paths):
     ref_file_name = os.path.split(ref_file_path)[-1]
     assert "roman_wfi_flat" in ref_file_name
     # Test FlatFieldStep
-    output = "r0000101001001001001_01101_0001_WFI01_flat.asdf"
+    output = "r0000101001001001001_0001_WFI01_flat.asdf"
     rtdata.output = output
     args = ["romancal.step.FlatFieldStep", rtdata.input]
     RomanStep.from_cmdline(args)
@@ -40,7 +40,7 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths):
     the grism and prism data should be None, only testing the grism
     case here."""
 
-    input_file = "r0000201001001001001_01101_0001_WFI01_uncal.asdf"
+    input_file = "r0000201001001001001_0001_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/grism/{input_file}")
     rtdata.input = input_file
 
@@ -59,7 +59,7 @@ def test_flat_field_grism_step(rtdata, ignore_asdf_paths):
     assert ref_file_name == "N/A"
 
     # Test FlatFieldStep
-    output = "r0000201001001001001_01101_0001_WFI01_flat.asdf"
+    output = "r0000201001001001001_0001_WFI01_flat.asdf"
     rtdata.output = output
     args = ["romancal.step.FlatFieldStep", rtdata.input]
     RomanStep.from_cmdline(args)
@@ -75,7 +75,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths):
     flat files and successfully make level 2 output"""
 
     # First file
-    input_l2_file = "r0000101001001001001_01101_0001_WFI01_assignwcs.asdf"
+    input_l2_file = "r0000101001001001001_0001_WFI01_assignwcs.asdf"
     rtdata.get_data(f"WFI/image/{input_l2_file}")
     rtdata.input = input_l2_file
 
@@ -102,7 +102,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths):
     )
 
     # Test FlatFieldStep
-    output = "r0000101001001001001_01101_0001_WFI01_flat.asdf"
+    output = "r0000101001001001001_0001_WFI01_flat.asdf"
     rtdata.output = output
     args = ["romancal.step.FlatFieldStep", rtdata.input]
     step.log.info(
@@ -126,7 +126,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths):
     #  to separate flat files in CRDS.
 
     # Second file
-    input_file = "r0000101001001001001_01101_0001_WFI01_changetime_assignwcs.asdf"
+    input_file = "r0000101001001001001_0001_WFI01_changetime_assignwcs.asdf"
     rtdata.get_data(f"WFI/image/{input_file}")
     rtdata.input = input_file
 
@@ -149,7 +149,7 @@ def test_flat_field_crds_match_image_step(rtdata, ignore_asdf_paths):
     )
 
     # Test FlatFieldStep
-    output = "r0000101001001001001_01101_0001_WFI01_changetime_flat.asdf"
+    output = "r0000101001001001001_0001_WFI01_changetime_flat.asdf"
     rtdata.output = output
     args = ["romancal.step.FlatFieldStep", rtdata.input]
     step.log.info(
diff --git a/romancal/regtest/test_wfi_grism_16resultants.py b/romancal/regtest/test_wfi_grism_16resultants.py
index b76c38e38..4e222ed09 100644
--- a/romancal/regtest/test_wfi_grism_16resultants.py
+++ b/romancal/regtest/test_wfi_grism_16resultants.py
@@ -16,12 +16,12 @@ def run_elp(rtdata_module):
     # The input data is from INS for stress testing at some point this should be generated
     # by INS every time new data is needed.
 
-    input_data = "r0000201001001001001_01101_0004_WFI01_uncal.asdf"
+    input_data = "r0000201001001001001_0004_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/grism/{input_data}")
     rtdata.input = input_data
 
     # Test Pipeline
-    output = "r0000201001001001001_01101_0004_WFI01_cal.asdf"
+    output = "r0000201001001001001_0004_WFI01_cal.asdf"
     rtdata.output = output
     args = [
         "roman_elp",
diff --git a/romancal/regtest/test_wfi_grism_pipeline.py b/romancal/regtest/test_wfi_grism_pipeline.py
index c993c25f1..d547240c5 100644
--- a/romancal/regtest/test_wfi_grism_pipeline.py
+++ b/romancal/regtest/test_wfi_grism_pipeline.py
@@ -19,12 +19,12 @@
 def run_elp(rtdata_module):
     rtdata = rtdata_module
 
-    input_data = "r0000201001001001001_01101_0001_WFI01_uncal.asdf"
+    input_data = "r0000201001001001001_0001_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/grism/{input_data}")
     rtdata.input = input_data
 
     # Test Pipeline
-    output = "r0000201001001001001_01101_0001_WFI01_cal.asdf"
+    output = "r0000201001001001001_0001_WFI01_cal.asdf"
     rtdata.output = output
     args = [
         "roman_elp",
diff --git a/romancal/regtest/test_wfi_image_16resultants.py b/romancal/regtest/test_wfi_image_16resultants.py
index 4e7add78e..8b2a997a5 100644
--- a/romancal/regtest/test_wfi_image_16resultants.py
+++ b/romancal/regtest/test_wfi_image_16resultants.py
@@ -16,12 +16,12 @@ def run_elp(rtdata_module):
     # The input data is from INS for stress testing at some point this should be generated
     # every time new data is needed.
 
-    input_data = "r0000101001001001001_01101_0004_WFI01_uncal.asdf"
+    input_data = "r0000101001001001001_0004_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/image/{input_data}")
     rtdata.input = input_data
 
     # Test Pipeline
-    output = "r0000101001001001001_01101_0004_WFI01_cal.asdf"
+    output = "r0000101001001001001_0004_WFI01_cal.asdf"
     rtdata.output = output
     args = [
         "roman_elp",
diff --git a/romancal/regtest/test_wfi_image_pipeline.py b/romancal/regtest/test_wfi_image_pipeline.py
index 71e612c8a..b821393c7 100644
--- a/romancal/regtest/test_wfi_image_pipeline.py
+++ b/romancal/regtest/test_wfi_image_pipeline.py
@@ -20,12 +20,12 @@
 def run_elp(rtdata_module):
     rtdata = rtdata_module
 
-    input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
+    input_data = "r0000101001001001001_0001_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/image/{input_data}")
     rtdata.input = input_data
 
     # Test Pipeline
-    output = "r0000101001001001001_01101_0001_WFI01_cal.asdf"
+    output = "r0000101001001001001_0001_WFI01_cal.asdf"
     rtdata.output = output
     args = [
         "roman_elp",
@@ -205,12 +205,12 @@ def test_repointed_wcs_differs(repointed_filename_and_delta, output_model):
 
 def test_elp_input_dm(rtdata, ignore_asdf_paths):
     """Test for input roman Datamodel to exposure level pipeline"""
-    input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
+    input_data = "r0000101001001001001_0001_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/image/{input_data}")
     dm_input = rdm.open(rtdata.input)
 
     # Test Pipeline with input datamodel
-    output = "r0000101001001001001_01101_0001_WFI01_cal.asdf"
+    output = "r0000101001001001001_0001_WFI01_cal.asdf"
     rtdata.output = output
     ExposurePipeline.call(dm_input, save_results=True)
     rtdata.get_truth(f"truth/WFI/image/{output}")
@@ -234,12 +234,12 @@ def test_processing_pipeline_all_saturated(rtdata, ignore_asdf_paths):
     Note that this test mimics how the pipeline is run in OPS.
     Any changes to this test should be coordinated with OPS.
     """
-    input_data = "r0000101001001001001_01101_0001_WFI01_ALL_SATURATED_uncal.asdf"
+    input_data = "r0000101001001001001_0001_WFI01_ALL_SATURATED_uncal.asdf"
     rtdata.get_data(f"WFI/image/{input_data}")
     rtdata.input = input_data
 
     # Test Pipeline
-    output = "r0000101001001001001_01101_0001_WFI01_ALL_SATURATED_cal.asdf"
+    output = "r0000101001001001001_0001_WFI01_ALL_SATURATED_cal.asdf"
     rtdata.output = output
     args = [
         "roman_elp",
@@ -272,10 +272,10 @@ def test_pipeline_suffix(rtdata, ignore_asdf_paths):
 
     Any changes to this test should be coordinated with OPS.
     """
-    input_data = "r0000101001001001001_01101_0001_WFI01_uncal.asdf"
+    input_data = "r0000101001001001001_0001_WFI01_uncal.asdf"
     rtdata.get_data(f"WFI/image/{input_data}")
 
-    output = "r0000101001001001001_01101_0001_WFI01_star.asdf"
+    output = "r0000101001001001001_0001_WFI01_star.asdf"
     rtdata.output = output
 
     args = [
diff --git a/romancal/regtest/test_wfi_photom.py b/romancal/regtest/test_wfi_photom.py
index f7c064c87..7af8734c7 100644
--- a/romancal/regtest/test_wfi_photom.py
+++ b/romancal/regtest/test_wfi_photom.py
@@ -16,7 +16,7 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     """DMS140 Test: Testing application of photometric correction using
     CRDS selected photom file."""
 
-    input_data = "r0000101001001001001_01101_0001_WFI01_flat.asdf"
+    input_data = "r0000101001001001001_0001_WFI01_flat.asdf"
     rtdata.get_data(f"WFI/image/{input_data}")
     rtdata.input = input_data
 
@@ -39,7 +39,7 @@ def test_absolute_photometric_calibration(rtdata, ignore_asdf_paths):
     # photom match from CRDS. Values come from roman_wfi_photom_0034.asdf
 
     # Test PhotomStep
-    output = "r0000101001001001001_01101_0001_WFI01_photom.asdf"
+    output = "r0000101001001001001_0001_WFI01_photom.asdf"
     rtdata.output = output
     args = ["romancal.step.PhotomStep", rtdata.input]
     step.log.info(
diff --git a/romancal/regtest/test_wfi_saturation.py b/romancal/regtest/test_wfi_saturation.py
index 0180008c1..1484af520 100644
--- a/romancal/regtest/test_wfi_saturation.py
+++ b/romancal/regtest/test_wfi_saturation.py
@@ -16,7 +16,7 @@ def test_saturation_image_step(rtdata, ignore_asdf_paths):
     """Testing retrieval of best ref file for image data,
     and creation of a ramp file with CRDS selected saturation file applied."""
 
-    input_file = "r0000101001001001001_01101_0001_WFI01_dqinit.asdf"
+    input_file = "r0000101001001001001_0001_WFI01_dqinit.asdf"
     rtdata.get_data(f"WFI/image/{input_file}")
     rtdata.input = input_file
 
@@ -30,7 +30,7 @@ def test_saturation_image_step(rtdata, ignore_asdf_paths):
     assert "roman_wfi_saturation" in ref_file_name
 
     # Test SaturationStep
-    output = "r0000101001001001001_01101_0001_WFI01_saturation.asdf"
+    output = "r0000101001001001001_0001_WFI01_saturation.asdf"
     rtdata.output = output
 
     args = ["romancal.step.SaturationStep", rtdata.input]
@@ -49,7 +49,7 @@ def test_saturation_grism_step(rtdata, ignore_asdf_paths):
     """Testing retrieval of best ref file for grism data,
     and creation of a ramp file with CRDS selected saturation file applied."""
 
-    input_file = "r0000201001001001001_01101_0001_WFI01_dqinit.asdf"
+    input_file = "r0000201001001001001_0001_WFI01_dqinit.asdf"
     rtdata.get_data(f"WFI/grism/{input_file}")
     rtdata.input = input_file
 
@@ -63,7 +63,7 @@ def test_saturation_grism_step(rtdata, ignore_asdf_paths):
     assert "roman_wfi_saturation" in ref_file_name
 
     # Test SaturationStep
-    output = "r0000201001001001001_01101_0001_WFI01_saturation.asdf"
+    output = "r0000201001001001001_0001_WFI01_saturation.asdf"
     rtdata.output = output
 
     args = ["romancal.step.SaturationStep", rtdata.input]
diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh
index 15b81e7cb..c406c4895 100644
--- a/romancal/scripts/make_regtestdata.sh
+++ b/romancal/scripts/make_regtestdata.sh
@@ -3,15 +3,15 @@
 # takes one argument: the directory into which to start putting the regtest files.
 
 # input files needed:
-# r0000101001001001001_01101_0001_WFI01 - default for most steps
-# r0000201001001001001_01101_0001_WFI01 - equivalent for spectroscopic data
-# r0000101001001001001_01101_0002_WFI01 - a second resample exposure, only cal step needed
-# r0000101001001001001_01101_0003_WFI01 - for ramp fitting; truncated image
-# r0000201001001001001_01101_0003_WFI01 - for ramp fitting; truncated spectroscopic
+# r0000101001001001001_0001_WFI01 - default for most steps
+# r0000201001001001001_0001_WFI01 - equivalent for spectroscopic data
+# r0000101001001001001_0002_WFI01 - a second resample exposure, only cal step needed
+# r0000101001001001001_0003_WFI01 - for ramp fitting; truncated image
+# r0000201001001001001_0003_WFI01 - for ramp fitting; truncated spectroscopic
 #                                         we need only darkcurrent & ramp fit for these
 #
-# r00r1601001001001001_01101_0001_WFI01 - special 16 resultant file, imaging, only need cal file
-# r10r1601001001001001_01101_0001_WFI01 - special 16 resultant file, spectroscopy, only need cal file
+# r00r1601001001001001_0001_WFI01 - special 16 resultant file, imaging, only need cal file
+# r10r1601001001001001_0001_WFI01 - special 16 resultant file, spectroscopy, only need cal file
 # roman_dark_WFI01_IMAGE_STRESS_TEST_16_MA_TABLE_998_D1 - special dark for 16 resultant file
 
 outdir=$1
@@ -24,7 +24,7 @@ mkdir -p $outdir/roman-pipeline/dev/truth/WFI/grism
 
 
 # most regtests; run the pipeline and save a lot of results.
-for fn in r0000101001001001001_01101_0001_WFI01 r0000201001001001001_01101_0001_WFI01
+for fn in r0000101001001001001_0001_WFI01 r0000201001001001001_0001_WFI01
 do
     echo "Running pipeline on ${fn}..."
     strun roman_elp ${fn}_uncal.asdf --steps.dq_init.save_results True --steps.saturation.save_results True --steps.linearity.save_results True --steps.dark_current.save_results True --steps.rampfit.save_results True --steps.assign_wcs.save_results True --steps.photom.save_results True --steps.refpix.save_results True --steps.flatfield.save_results True --steps.assign_wcs.save_results True
@@ -38,7 +38,7 @@ done
 
 
 # truncated files for ramp fit regtests
-for fn in r0000101001001001001_01101_0003_WFI01 r0000201001001001001_01101_0003_WFI01
+for fn in r0000101001001001001_0003_WFI01 r0000201001001001001_0003_WFI01
 do
     echo "Running pipeline on ${fn}..."
     strun roman_elp ${fn}_uncal.asdf --steps.dark_current.save_results True --steps.rampfit.save_results True
@@ -46,29 +46,35 @@ do
     cp ${fn}_darkcurrent.asdf $outdir/roman-pipeline/dev/WFI/$dirname/
     cp ${fn}_rampfit.asdf $outdir/roman-pipeline/dev/truth/WFI/$dirname/
 done
-cp r0000101001001001001_01101_0003_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/
+cp r0000101001001001001_0003_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/
 
 
 # second imaging exposure
-strun roman_elp r0000101001001001001_01101_0002_WFI01_uncal.asdf
-cp r0000101001001001001_01101_0002_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/
+strun roman_elp r0000101001001001001_0002_WFI01_uncal.asdf
+cp r0000101001001001001_0002_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image/
 
 
-# resample regtest; needs r0000101001001001001_01101_000{1,2}_WFI01_cal.asdf
+# resample regtest; needs r0000101001001001001_000{1,2}_WFI01_cal.asdf
 # builds the appropriate asn file and calls strun with it
 echo "Creating regtest files for resample..."
+<<<<<<< Updated upstream
 asn_from_list r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic
 strun romancal.step.ResampleStep L3_mosaic_asn.json --rotation=0 --output_file=mosaic.asdf
 cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/
+=======
+asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf -o mosaic_asn.json --product-name mosaic
+strun romancal.step.ResampleStep mosaic_asn.json --rotation=0 --output_file=mosaic.asdf
+cp mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/
+>>>>>>> Stashed changes
 cp mosaic_resamplestep.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 
 
-# CRDS test needs the "usual" r00001..._01101_0001_WFI01 files.
-# It also needs a hacked r00001..._01101_0001_WFI01 file, with the time changed.
+# CRDS test needs the "usual" r00001..._0001_WFI01 files.
+# It also needs a hacked r00001..._0001_WFI01 file, with the time changed.
 # this makes the hacked version.
 echo "Creating regtest files for CRDS tests..."
-basename="r0000101001001001001_01101_0001_WFI01"
+basename="r0000101001001001001_0001_WFI01"
 python -c "
 import asdf
 from roman_datamodels import stnode
@@ -86,14 +92,14 @@ cp ${basename}_changetime_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/image
 
 # need to make a special ALL_SATURATED file for the all saturated test.
 echo "Creating regtest files for all saturated tests..."
-basename="r0000101001001001001_01101_0001_WFI01"
+basename="r0000101001001001001_0001_WFI01"
 python -c "
 import asdf
 from roman_datamodels import stnode
 basename = '$basename'
 f = asdf.open(f'{basename}_uncal.asdf')
 data = f['roman']['data'].copy()
-data[...] = 65535 * f['roman']['data'].unit
+data[...] = 65535
 f['roman']['data'] = data
 f['roman']['meta']['filename'] = stnode.Filename(f'{basename}_ALL_SATURATED_uncal.asdf')
 f.write_to(f'{basename}_ALL_SATURATED_uncal.asdf')"
@@ -103,19 +109,19 @@ cp ${basename}_ALL_SATURATED_cal.asdf $outdir/roman-pipeline/dev/truth/WFI/image
 
 
 # make a special file dark file with a different name
-strun romancal.step.DarkCurrentStep r0000101001001001001_01101_0001_WFI01_linearity.asdf --output_file=Test_dark
+strun romancal.step.DarkCurrentStep r0000101001001001001_0001_WFI01_linearity.asdf --output_file=Test_dark
 cp Test_darkcurrent.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 
 # make a special linearity file with a different suffix
-strun romancal.step.LinearityStep r0000101001001001001_01101_0001_WFI01_refpix.asdf --output_file=Test_linearity
+strun romancal.step.LinearityStep r0000101001001001001_0001_WFI01_refpix.asdf --output_file=Test_linearity
 cp Test_linearity.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 
 # we have a test that runs the flat field step directly on an _L1_ spectroscopic
 # file and verifies that it gets skipped.
 # I don't really understand that but we can duplicate it for now.
-basename="r0000201001001001001_01101_0001_WFI01"
+basename="r0000201001001001001_0001_WFI01"
 strun romancal.step.FlatFieldStep ${basename}_uncal.asdf
 cp ${basename}_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/grism/
 
@@ -124,7 +130,7 @@ cp ${basename}_flat.asdf $outdir/roman-pipeline/dev/truth/WFI/grism/
 # we haven't updated the filename in these files, but the regtest mechanism
 # also doesn't
 # update them, and we need to match.
-for basename in r0000101001001001001_01101_0001_WFI01 r0000201001001001001_01101_0001_WFI01
+for basename in r0000101001001001001_0001_WFI01 r0000201001001001001_0001_WFI01
 do
     python -c "
 import asdf
@@ -146,7 +152,7 @@ model.to_asdf(f'${basename}_cal_repoint.asdf')"
 done
 
 # Test tweakreg with repointed file, only shifted by 1"
-for basename in r0000101001001001001_01101_0001_WFI01
+for basename in r0000101001001001001_0001_WFI01
 do
     python -c "
 import asdf
@@ -167,27 +173,34 @@ model.to_asdf(f'${basename}_shift_cal.asdf')"
 done
 
 
-strun roman_elp r0000101001001001001_01101_0004_WFI01_uncal.asdf
-strun roman_elp r0000201001001001001_01101_0004_WFI01_uncal.asdf
-cp r0000101001001001001_01101_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/image/
-cp r0000201001001001001_01101_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/grism/
+strun roman_elp r0000101001001001001_0004_WFI01_uncal.asdf
+strun roman_elp r0000201001001001001_0004_WFI01_uncal.asdf
+cp r0000101001001001001_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/image/
+cp r0000201001001001001_0004_WFI01_uncal.asdf $outdir/roman-pipeline/dev/WFI/grism/
 
 l3name="r0099101001001001001_F158_visit"
-asn_from_list r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf r0000101001001001001_01101_0003_WFI01_cal.asdf -o L3_regtest_asn.json --product-name $l3name
+asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_regtest_asn.json --product-name $l3name
 strun roman_mos L3_regtest_asn.json
 cp L3_regtest_asn.json $outdir/roman-pipeline/dev/WFI/image/
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/WFI/image/
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
+l3name="r0099101001001001001_r274dp63x31y81_prompt_F158"
+asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name $l3name --target r274dp63x31y81
+strun roman_mos L3_mosaic_asn.json
+cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/
+cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/WFI/image/
+cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
+
 # L3 catalog
 strun romancal.step.SourceCatalogStep ${l3name}_i2d.asdf
 cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 # L2 catalog
-strun romancal.step.SourceCatalogStep r0000101001001001001_01101_0001_WFI01_cal.asdf
-cp r0000101001001001001_01101_0001_WFI01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
+strun romancal.step.SourceCatalogStep r0000101001001001001_0001_WFI01_cal.asdf
+cp r0000101001001001001_0001_WFI01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 l3name="mosaic"
-asn_from_list --product-name=$l3name r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf r0000101001001001001_01101_0003_WFI01_cal.asdf -o L3_m1_asn.json
+asn_from_list --product-name=$l3name r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_m1_asn.json
 strun roman_mos L3_m1_asn.json
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/

From 68cea78008379b37c6143b89b9177cdd8eb9b6e0 Mon Sep 17 00:00:00 2001
From: Eddie Schlafly <eschlafly@stsci.edu>
Date: Fri, 11 Oct 2024 09:56:29 -0400
Subject: [PATCH 62/74] Remove ggsaa from file names.

---
 romancal/scripts/make_regtestdata.sh | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh
index c406c4895..de2a01c1b 100644
--- a/romancal/scripts/make_regtestdata.sh
+++ b/romancal/scripts/make_regtestdata.sh
@@ -57,19 +57,12 @@ cp r0000101001001001001_0002_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image
 # resample regtest; needs r0000101001001001001_000{1,2}_WFI01_cal.asdf
 # builds the appropriate asn file and calls strun with it
 echo "Creating regtest files for resample..."
-<<<<<<< Updated upstream
-asn_from_list r0000101001001001001_01101_0001_WFI01_cal.asdf r0000101001001001001_01101_0002_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic
+asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic
 strun romancal.step.ResampleStep L3_mosaic_asn.json --rotation=0 --output_file=mosaic.asdf
 cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/
-=======
-asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf -o mosaic_asn.json --product-name mosaic
-strun romancal.step.ResampleStep mosaic_asn.json --rotation=0 --output_file=mosaic.asdf
-cp mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/
->>>>>>> Stashed changes
 cp mosaic_resamplestep.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 
-
 # CRDS test needs the "usual" r00001..._0001_WFI01 files.
 # It also needs a hacked r00001..._0001_WFI01 file, with the time changed.
 # this makes the hacked version.

From 5c9e4cefd4a20515c1ba03dbf3aaf7817491c48c Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Sat, 12 Oct 2024 09:51:34 -0400
Subject: [PATCH 63/74] More fixes.

---
 romancal/regtest/test_mos_skycell_pipeline.py |  2 +-
 romancal/regtest/test_resample.py             |  2 +-
 romancal/scripts/make_regtestdata.sh          | 14 ++++++++++++--
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/romancal/regtest/test_mos_skycell_pipeline.py b/romancal/regtest/test_mos_skycell_pipeline.py
index da542af3b..712a69f55 100644
--- a/romancal/regtest/test_mos_skycell_pipeline.py
+++ b/romancal/regtest/test_mos_skycell_pipeline.py
@@ -15,7 +15,7 @@ def run_mos(rtdata_module):
 
     # Test Pipeline
     rtdata.get_asn("WFI/image/L3_mosaic_asn.json")
-    output = "mosaic_i2d.asdf"
+    output = "r0099101001001001001_r274dp63x31y81_prompt_F158_i2d.asdf"
     rtdata.output = output
     args = [
         "roman_mos",
diff --git a/romancal/regtest/test_resample.py b/romancal/regtest/test_resample.py
index bdbf48170..d669eecc6 100644
--- a/romancal/regtest/test_resample.py
+++ b/romancal/regtest/test_resample.py
@@ -12,7 +12,7 @@
 def test_resample_single_file(rtdata, ignore_asdf_paths):
     output_data = "mosaic_resamplestep.asdf"
 
-    rtdata.get_asn("WFI/image/mosaic_asn.json")
+    rtdata.get_asn("WFI/image/L3_mosaic_asn.json")
     rtdata.get_truth(f"truth/WFI/image/{output_data}")
 
     rtdata.output = output_data
diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh
index de2a01c1b..5b145ec7c 100644
--- a/romancal/scripts/make_regtestdata.sh
+++ b/romancal/scripts/make_regtestdata.sh
@@ -12,9 +12,12 @@
 #
 # r00r1601001001001001_0001_WFI01 - special 16 resultant file, imaging, only need cal file
 # r10r1601001001001001_0001_WFI01 - special 16 resultant file, spectroscopy, only need cal file
-# roman_dark_WFI01_IMAGE_STRESS_TEST_16_MA_TABLE_998_D1 - special dark for 16 resultant file
 
 outdir=$1
+logfile=$outdir/make_regtestdata.log
+
+# Redirect all output to the logfile
+exec > $logfile 2>&1
 
 # set up the directory structure
 mkdir -p $outdir/roman-pipeline/dev/WFI/image
@@ -178,6 +181,11 @@ cp L3_regtest_asn.json $outdir/roman-pipeline/dev/WFI/image/
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/WFI/image/
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
+# L3 catalog
+strun romancal.step.SourceCatalogStep ${l3name}_i2d.asdf
+cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
+
+
 l3name="r0099101001001001001_r274dp63x31y81_prompt_F158"
 asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name $l3name --target r274dp63x31y81
 strun roman_mos L3_mosaic_asn.json
@@ -189,11 +197,13 @@ cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 strun romancal.step.SourceCatalogStep ${l3name}_i2d.asdf
 cp ${l3name}_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
+
 # L2 catalog
 strun romancal.step.SourceCatalogStep r0000101001001001001_0001_WFI01_cal.asdf
 cp r0000101001001001001_0001_WFI01_cat.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
-l3name="mosaic"
+
+l3name="r0099101001001001001_F158_visit_r274dp63x31y81"
 asn_from_list --product-name=$l3name r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_m1_asn.json
 strun roman_mos L3_m1_asn.json
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/

From 4214f85cd9ff8c8950f691b0f1bd517e758b3b80 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Mon, 14 Oct 2024 13:45:40 -0400
Subject: [PATCH 64/74] Update make_regtestdata.sh

---
 romancal/scripts/make_regtestdata.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh
index 5b145ec7c..19681f449 100644
--- a/romancal/scripts/make_regtestdata.sh
+++ b/romancal/scripts/make_regtestdata.sh
@@ -60,7 +60,7 @@ cp r0000101001001001001_0002_WFI01_cal.asdf $outdir/roman-pipeline/dev/WFI/image
 # resample regtest; needs r0000101001001001001_000{1,2}_WFI01_cal.asdf
 # builds the appropriate asn file and calls strun with it
 echo "Creating regtest files for resample..."
-asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic
+asn_from_list r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_mosaic_asn.json --product-name mosaic
 strun romancal.step.ResampleStep L3_mosaic_asn.json --rotation=0 --output_file=mosaic.asdf
 cp L3_mosaic_asn.json $outdir/roman-pipeline/dev/WFI/image/
 cp mosaic_resamplestep.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
@@ -207,3 +207,7 @@ l3name="r0099101001001001001_F158_visit_r274dp63x31y81"
 asn_from_list --product-name=$l3name r0000101001001001001_0001_WFI01_cal.asdf r0000101001001001001_0002_WFI01_cal.asdf r0000101001001001001_0003_WFI01_cal.asdf -o L3_m1_asn.json
 strun roman_mos L3_m1_asn.json
 cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
+
+# tests passing suffix to the pipeline
+strun roman_elp r0000101001001001001_0001_WFI01_uncal.asdf --steps.tweakreg.skip=True --suffix=star
+cp r0000101001001001001_0001_WFI01_star.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
\ No newline at end of file

From 8b6a7fb9cd37ef9ff69872a1ae4c93d2b1f6fadf Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 15 Oct 2024 13:29:13 -0400
Subject: [PATCH 65/74] Update docstrings.

---
 romancal/source_catalog/detection.py      |  4 ++--
 romancal/source_catalog/source_catalog.py | 11 +++++------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/romancal/source_catalog/detection.py b/romancal/source_catalog/detection.py
index c86982bee..f83421296 100644
--- a/romancal/source_catalog/detection.py
+++ b/romancal/source_catalog/detection.py
@@ -22,8 +22,8 @@ def convolve_data(data, kernel_fwhm, size=None, mask=None):
 
     Parameters
     ----------
-    data : 2D `astropy.units.Quantity`
-        The input 2D Quantity array. The data is assumed to be
+    data : 2D `numpy.ndarray`
+        The input 2D array. The data is assumed to be
         background subtracted.
     kernel_fwhm : float
         The FWHM of the Gaussian kernel.
diff --git a/romancal/source_catalog/source_catalog.py b/romancal/source_catalog/source_catalog.py
index 7e3b6243d..66dadaf82 100644
--- a/romancal/source_catalog/source_catalog.py
+++ b/romancal/source_catalog/source_catalog.py
@@ -176,7 +176,7 @@ def convert_l2_to_sb(self):
 
     def convert_sb_to_l2(self):
         """
-        Convert the data and error Quantity arrays from MJy/sr (surface
+        Convert the data and error arrays from MJy/sr (surface
         brightness) to DN/s (level-2 units).
 
         This is the inverse operation of `convert_l2_to_sb`.
@@ -190,7 +190,7 @@ def convert_sb_to_l2(self):
 
     def convert_sb_to_flux_density(self):
         """
-        Convert the data and error Quantity arrays from MJy/sr (surface
+        Convert the data and error arrays from MJy/sr (surface
         brightness) to flux density units.
 
         The flux density unit is defined by self.flux_unit.
@@ -206,7 +206,7 @@ def convert_sb_to_flux_density(self):
 
     def convert_flux_density_to_sb(self):
         """
-        Convert the data and error Quantity arrays from flux density units to
+        Convert the data and error arrays from flux density units to
         MJy/sr (surface brightness).
 
         This is the inverse operation of `convert_sb_to_flux_density`.
@@ -222,7 +222,7 @@ def convert_flux_to_abmag(self, flux, flux_err):
 
         Parameters
         ----------
-        flux, flux_err : `~astropy.unit.Quantity`
+        flux, flux_err : `~numpy.ndarray`
             The input flux and error arrays.
 
         Returns
@@ -1190,8 +1190,7 @@ def catalog(self):
         self._update_metadata()
         catalog.meta.update(self.meta)
 
-        # convert QTable to Table to change Quantity columns to regular
-        # columns with units
+        # convert QTable to Table
         catalog = Table(catalog)
 
         # split SkyCoord columns into separate RA and Dec columns

From 7b46909dbd33293fddd50d3b33747673ce509e7c Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 15 Oct 2024 17:29:45 +0000
Subject: [PATCH 66/74] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 romancal/scripts/make_regtestdata.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/romancal/scripts/make_regtestdata.sh b/romancal/scripts/make_regtestdata.sh
index 19681f449..ec0d82098 100644
--- a/romancal/scripts/make_regtestdata.sh
+++ b/romancal/scripts/make_regtestdata.sh
@@ -210,4 +210,4 @@ cp ${l3name}_i2d.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
 
 # tests passing suffix to the pipeline
 strun roman_elp r0000101001001001001_0001_WFI01_uncal.asdf --steps.tweakreg.skip=True --suffix=star
-cp r0000101001001001001_0001_WFI01_star.asdf $outdir/roman-pipeline/dev/truth/WFI/image/
\ No newline at end of file
+cp r0000101001001001001_0001_WFI01_star.asdf $outdir/roman-pipeline/dev/truth/WFI/image/

From bc6877cd3e5d7943158b13a95005d2c028ae5e44 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Tue, 15 Oct 2024 14:36:51 -0400
Subject: [PATCH 67/74] Add changelog entry.

---
 changes/1445.general.rst | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changes/1445.general.rst

diff --git a/changes/1445.general.rst b/changes/1445.general.rst
new file mode 100644
index 000000000..4471074d0
--- /dev/null
+++ b/changes/1445.general.rst
@@ -0,0 +1 @@
+Remove units from romancal.

From f3070d0018a27cd6a0ea319c5c11f72bc4fda574 Mon Sep 17 00:00:00 2001
From: mairan <mteodoro@stsci.edu>
Date: Tue, 15 Oct 2024 16:59:19 -0400
Subject: [PATCH 68/74] Update test_photom.py

---
 romancal/photom/tests/test_photom.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/romancal/photom/tests/test_photom.py b/romancal/photom/tests/test_photom.py
index 198c77034..d5b222a3e 100644
--- a/romancal/photom/tests/test_photom.py
+++ b/romancal/photom/tests/test_photom.py
@@ -153,7 +153,6 @@ def test_apply_photom1():
         area_a2.value,
         atol=1.0e-7,
     )
-    # assert output_model.meta.photometry.pixelarea_arcsecsq.unit == area_a2.unit
 
     # Set reference photometry
     phot_ster = 3.5 * u.megajansky / u.steradian

From e3b204c0f8afdd603397bc0067e54ac5b55c3d6f Mon Sep 17 00:00:00 2001
From: mairan <mteodoro@stsci.edu>
Date: Tue, 15 Oct 2024 17:03:18 -0400
Subject: [PATCH 69/74] Update skystatistics.py

---
 romancal/skymatch/skystatistics.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/romancal/skymatch/skystatistics.py b/romancal/skymatch/skystatistics.py
index 5032d3291..ce22eee2b 100644
--- a/romancal/skymatch/skystatistics.py
+++ b/romancal/skymatch/skystatistics.py
@@ -127,13 +127,7 @@ def calc_sky(self, data):
 
         """
         imstat = ImageStats(image=data, fields=self._fields, **(self._kwargs))
-        stat = self._skystat(imstat)  # dict or scalar
-
-        # re-attach units:
-        if hasattr(stat, "__len__"):
-            self.skyval = {k: value for k, value in stat.items()}
-        else:
-            self.skyval = stat
+        self.skyval = self._skystat(imstat)  # dict or scalar
 
         self.npix = imstat.npix
         return self.skyval, self.npix

From 6521c27057e30eb2ed8eeb9e14a85625592a10eb Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Wed, 16 Oct 2024 16:18:45 -0400
Subject: [PATCH 70/74] Disable  in conf.py.

---
 docs/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/conf.py b/docs/conf.py
index c390a66fd..4222f3afa 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -273,7 +273,7 @@ def check_sphinx_version(expected_version):
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-html_theme_options = {"collapse_navigation": True, "display_version": True}
+html_theme_options = {"collapse_navigation": True}
 #        "nosidebar": "false",
 #        "sidebarbgcolor": "#4db8ff",
 #        "sidebartextcolor": "black",

From d1dd3b4cfa8ebe21dfd6925fcdf84aac64223124 Mon Sep 17 00:00:00 2001
From: "M. Teodoro" <mteodoro@stsci.edu>
Date: Thu, 17 Oct 2024 06:27:13 -0400
Subject: [PATCH 71/74] Remove ggsaa from filename used by new regtest.

---
 romancal/regtest/test_skycell_generation.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/romancal/regtest/test_skycell_generation.py b/romancal/regtest/test_skycell_generation.py
index 60e943ba0..cc08de547 100644
--- a/romancal/regtest/test_skycell_generation.py
+++ b/romancal/regtest/test_skycell_generation.py
@@ -13,13 +13,13 @@ def test_skycell_asn_generation(rtdata):
 
     # This test should generate seven json files
     args = [
-        "r0000101001001001001_01101_0002_WFI01_cal.asdf",
-        "r0000101001001001001_01101_0002_WFI10_cal.asdf",
+        "r0000101001001001001_0002_WFI01_cal.asdf",
+        "r0000101001001001001_0002_WFI10_cal.asdf",
         "-o",
         "r512",
     ]
-    rtdata.get_data("WFI/image/r0000101001001001001_01101_0002_WFI01_cal.asdf")
-    rtdata.get_data("WFI/image/r0000101001001001001_01101_0002_WFI10_cal.asdf")
+    rtdata.get_data("WFI/image/r0000101001001001001_0002_WFI01_cal.asdf")
+    rtdata.get_data("WFI/image/r0000101001001001001_0002_WFI10_cal.asdf")
 
     skycell_asn.Main(args)
 

From 64674c00b313c6a8c4ed7b9cf46d21c9bd0eb76b Mon Sep 17 00:00:00 2001
From: Paul Huwe <42071634+PaulHuwe@users.noreply.github.com>
Date: Thu, 17 Oct 2024 10:31:31 -0400
Subject: [PATCH 72/74] Update pyproject.toml

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index d0096e78d..b85b9c065 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,7 +24,7 @@ dependencies = [
     "pyparsing >=2.4.7",
     "requests >=2.26",
     # "rad>=0.21.0,<0.22.0",
-    "rad @ git+https://github.com/mairanteodoro/rad.git@RCAL-932-remove-units-in-exposure-level-pipeline",
+    "rad @ git+https://github.com/spacetelescope/rad.git",
     # "roman_datamodels>=0.21.0,<0.22.0",
     "roman_datamodels @ git+https://github.com/mairanteodoro/roman_datamodels.git@RCAL-932-remove-units-in-exposure-level-pipeline",
     "scipy >=1.11",

From 920fb462ce238db713c7754a7999345189f11151 Mon Sep 17 00:00:00 2001
From: Paul Huwe <42071634+PaulHuwe@users.noreply.github.com>
Date: Thu, 17 Oct 2024 10:32:47 -0400
Subject: [PATCH 73/74] Update pyproject.toml

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index b85b9c065..0624c00d9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,7 +26,7 @@ dependencies = [
     # "rad>=0.21.0,<0.22.0",
     "rad @ git+https://github.com/spacetelescope/rad.git",
     # "roman_datamodels>=0.21.0,<0.22.0",
-    "roman_datamodels @ git+https://github.com/mairanteodoro/roman_datamodels.git@RCAL-932-remove-units-in-exposure-level-pipeline",
+    "roman_datamodels @ git+https://github.com/spacetelescope/roman_datamodels.git",
     "scipy >=1.11",
 #    "stcal>=1.8.0,<1.9.0",
     "stcal @ git+https://github.com/spacetelescope/stcal.git@main",

From 2cf8dbc11ce40c16199298a097e0a4d1ca65ab18 Mon Sep 17 00:00:00 2001
From: Paul Huwe <42071634+PaulHuwe@users.noreply.github.com>
Date: Thu, 17 Oct 2024 11:14:39 -0400
Subject: [PATCH 74/74] Update skymatch_step.py

---
 romancal/skymatch/skymatch_step.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/romancal/skymatch/skymatch_step.py b/romancal/skymatch/skymatch_step.py
index 833f5a320..309f99fbc 100644
--- a/romancal/skymatch/skymatch_step.py
+++ b/romancal/skymatch/skymatch_step.py
@@ -179,7 +179,7 @@ def _set_sky_background(self, sky_image, step_status):
         image.meta.background.subtracted = self.subtract
         # In numpy 2, the dtypes are more carefully controlled, so to match the
         # schema the data type needs to be re-cast to float64.
-        image.meta.background.level = sky.astype(np.float64)
+        image.meta.background.level = sky
 
         if step_status == "COMPLETE" and self.subtract:
             image.data[...] = sky_image.image[...]