From b50a7db89f3a4e05a019da00f661c43ecdfac4a4 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Wed, 20 Nov 2024 13:04:13 -0500 Subject: [PATCH 01/19] rcal-954 Initial removal of jump code --- romancal/jump/__init__.py | 3 - romancal/jump/jump_step.py | 176 -------------- romancal/jump/tests/__init__.py | 0 romancal/jump/tests/test_jump_step.py | 322 -------------------------- romancal/regtest/test_jump_det.py | 36 --- 5 files changed, 537 deletions(-) delete mode 100644 romancal/jump/__init__.py delete mode 100644 romancal/jump/jump_step.py delete mode 100644 romancal/jump/tests/__init__.py delete mode 100644 romancal/jump/tests/test_jump_step.py delete mode 100644 romancal/regtest/test_jump_det.py diff --git a/romancal/jump/__init__.py b/romancal/jump/__init__.py deleted file mode 100644 index 87f39f59a..000000000 --- a/romancal/jump/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .jump_step import JumpStep - -__all__ = ["JumpStep"] diff --git a/romancal/jump/jump_step.py b/romancal/jump/jump_step.py deleted file mode 100644 index 2dc1a3d80..000000000 --- a/romancal/jump/jump_step.py +++ /dev/null @@ -1,176 +0,0 @@ -""" -Detect jumps in a science image -""" - -import logging -import time - -import numpy as np -from roman_datamodels import datamodels as rdd -from roman_datamodels.dqflags import group, pixel -from stcal.jump.jump import detect_jumps - -from romancal.stpipe import RomanStep - -log = logging.getLogger(__name__) -log.setLevel(logging.DEBUG) - - -__all__ = ["JumpStep"] - - -class JumpStep(RomanStep): - """ - JumpStep: Performs CR/jump detection. The 2-point difference method is - applied. - """ - - class_alias = "jump" - - spec = """ - rejection_threshold = float(default=180.0,min=0) # CR sigma rej thresh - three_group_rejection_threshold = float(default=185.0,min=0) # CR sigma rej thresh - four_group_rejection_threshold = float(default=190.0,min=0) # CR sigma rej thresh - maximum_cores = option('none', 'quarter', 'half', 'all', default='none') # max number of processes to create - flag_4_neighbors = boolean(default=True) # flag the four perpendicular neighbors of each CR - max_jump_to_flag_neighbors = float(default=1000) # maximum jump sigma that will trigger neighbor flagging - min_jump_to_flag_neighbors = float(default=10) # minimum jump sigma that will trigger neighbor flagging - min_sat_area = float(default=1.0) # minimum required area for the central saturation of snowballs - min_jump_area = float(default=5.0) # minimum area to trigger large events processing - expand_factor = float(default=2.0) # The expansion factor for the enclosing circles or ellipses - use_ellipses = boolean(default=False) # Use an enclosing ellipse rather than a circle for MIRI showers - sat_required_snowball = boolean(default=True) # Require the center of snowballs to be saturated - expand_large_events = boolean(default=False) # must be True to trigger snowball and shower flagging - use_ramp_jump_detection = boolean(default=True) # Use jump detection during ramp fitting - """ # noqa: E501 - - reference_file_types = ["gain", "readnoise"] - - def process(self, input): - # Open input as a Roman DataModel (single integration; 3D arrays) - if isinstance(input, rdd.DataModel): - input_model = input - else: - input_model = rdd.open(input) - - # Extract the needed info from the Roman Data Model - r_data = input_model.data - r_gdq = input_model.groupdq - r_pdq = input_model.pixeldq - r_err = input_model.err - result = input_model - - # If the ramp fitting jump detection is enabled, then skip this step - if self.use_ramp_jump_detection: - result.meta.cal_step.jump = "SKIPPED" - return result - - # FIXME: since frames_per_group => meta.exposure.nframes has been removed, - # we need to fix stcal.jump.jump to remove it from there too - frames_per_group = 1 - - # Modify the arrays for input into the 'common' jump (4D) - data = np.copy(r_data[np.newaxis, :]) - gdq = r_gdq[np.newaxis, :] - pdq = r_pdq[np.newaxis, :] - err = np.copy(r_err[np.newaxis, :]) - - tstart = time.time() - - # Check for an input model with nresultants<=2 - nresultants = data.shape[1] - - if nresultants <= 2: - self.log.warning("Cannot apply jump detection as nresultants<=2;") - self.log.warning("Jump step will be skipped") - - result = input_model - - result.meta.cal_step.jump = "SKIPPED" - return result - - # Retrieve the parameter values - rej_thresh = self.rejection_threshold - three_grp_rej_thresh = self.three_group_rejection_threshold - four_grp_rej_thresh = self.four_group_rejection_threshold - max_cores = self.maximum_cores - max_jump_to_flag_neighbors = self.max_jump_to_flag_neighbors - min_jump_to_flag_neighbors = self.min_jump_to_flag_neighbors - flag_4_neighbors = self.flag_4_neighbors - min_sat_area = self.min_sat_area - min_jump_area = self.min_jump_area - expand_factor = self.expand_factor - use_ellipses = self.use_ellipses - sat_required_snowball = self.sat_required_snowball - expand_large_events = self.expand_large_events - - self.log.info("CR rejection threshold = %g sigma", rej_thresh) - if self.maximum_cores != "none": - self.log.info("Maximum cores to use = %s", max_cores) - - # Get the gain and readnoise reference files - # TODO: remove units from gain and RN reference files - gain_filename = self.get_reference_file(input_model, "gain") - self.log.info("Using GAIN reference file: %s", gain_filename) - gain_model = rdd.GainRefModel(gain_filename) - gain_2d = gain_model.data.value - - readnoise_filename = self.get_reference_file(input_model, "readnoise") - self.log.info("Using READNOISE reference file: %s", readnoise_filename) - readnoise_model = rdd.ReadnoiseRefModel(readnoise_filename) - # This is to clear the WRITEABLE=False flag? - readnoise_2d = np.copy(readnoise_model.data.value) - - # DG 0810/21: leave for now; make dqflags changes in a later, - # separate PR - dqflags_d = {} # Dict of DQ flags - dqflags_d = { - "GOOD": group.GOOD, - "DO_NOT_USE": group.DO_NOT_USE, - "SATURATED": group.SATURATED, - "JUMP_DET": group.JUMP_DET, - "NO_GAIN_VALUE": pixel.NO_GAIN_VALUE, - } - - gdq, pdq, *_ = detect_jumps( - frames_per_group, - data, - gdq, - pdq, - err, - gain_2d, - readnoise_2d, - rej_thresh, - three_grp_rej_thresh, - four_grp_rej_thresh, - max_cores, - max_jump_to_flag_neighbors, - min_jump_to_flag_neighbors, - flag_4_neighbors, - dqflags_d, - min_sat_area=min_sat_area, - min_jump_area=min_jump_area, - expand_factor=expand_factor, - use_ellipses=use_ellipses, - sat_required_snowball=sat_required_snowball, - expand_large_events=expand_large_events, - ) - - gdq = gdq[0, :, :, :] - pdq = pdq[0, :, :] - result.groupdq = gdq - result.pixeldq = pdq - gain_model.close() - readnoise_model.close() - tstop = time.time() - self.log.info("The execution time in seconds: %f", tstop - tstart) - - result.meta.cal_step.jump = "COMPLETE" - - if self.save_results: - try: - self.suffix = "jump" - except AttributeError: - self["suffix"] = "jump" - - return result diff --git a/romancal/jump/tests/__init__.py b/romancal/jump/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/romancal/jump/tests/test_jump_step.py b/romancal/jump/tests/test_jump_step.py deleted file mode 100644 index dc06be40b..000000000 --- a/romancal/jump/tests/test_jump_step.py +++ /dev/null @@ -1,322 +0,0 @@ -""" - Unit tests for the Roman jump step code -""" - -from itertools import cycle - -import numpy as np -import pytest -import roman_datamodels.stnode as rds -from astropy import units as u -from astropy.time import Time -from roman_datamodels import datamodels as rdm -from roman_datamodels import maker_utils -from roman_datamodels.datamodels import GainRefModel, ReadnoiseRefModel - -from romancal.jump import JumpStep - -RNG = np.random.default_rng(42) - -MAXIMUM_CORES = ["none", "quarter", "half", "all"] - - -@pytest.fixture(scope="module") -def generate_wfi_reffiles(tmp_path_factory): - gainfile = str(tmp_path_factory.mktemp("ndata") / "gain.asdf") - readnoisefile = str(tmp_path_factory.mktemp("ndata") / "readnoise.asdf") - - ingain = 6 - xsize = 20 - ysize = 20 - - shape = (ysize, xsize) - - # Create temporary gain reference file - gain_ref = rds.GainRef() - meta = maker_utils.mk_ref_common("GAIN") - meta["instrument"]["detector"] = "WFI01" - meta["instrument"]["name"] = "WFI" - meta["author"] = "John Doe" - meta["pedigree"] = "DUMMY" - meta["description"] = "DUMMY" - meta["useafter"] = Time("2022-01-01T11:11:11.111") - - gain_ref["meta"] = meta - gain_ref["data"] = u.Quantity( - np.ones(shape, dtype=np.float32) * ingain, u.electron / u.DN, dtype=np.float32 - ) - gain_ref["dq"] = np.zeros(shape, dtype=np.uint16) - gain_ref["err"] = u.Quantity( - (RNG.uniform(size=shape) * 0.05).astype(np.float64), - u.electron / u.DN, - dtype=np.float64, - ) - - gain_ref_model = GainRefModel(gain_ref) - gain_ref_model.save(gainfile) - - # Create temporary readnoise reference file - rn_ref = rds.ReadnoiseRef() - meta = maker_utils.mk_ref_common("READNOISE") - meta["instrument"]["detector"] = "WFI01" - meta["instrument"]["name"] = "WFI" - meta["author"] = "John Doe" - meta["pedigree"] = "DUMMY" - meta["description"] = "DUMMY" - meta["useafter"] = Time("2022-01-01T11:11:11.111") - meta["exposure"] = {} - meta["exposure"]["type"] = "WFI_IMAGE" - meta["exposure"]["p_exptype"] = "WFI_IMAGE|WFI_GRISM|WFI_PRISM|" - - rn_ref["meta"] = meta - rn_ref["data"] = u.Quantity( - np.ones(shape, dtype=np.float32), u.DN, dtype=np.float32 - ) - rn_ref["dq"] = np.zeros(shape, dtype=np.uint16) - rn_ref["err"] = u.Quantity( - (RNG.uniform(size=shape) * 0.05).astype(np.float64), u.DN, dtype=np.float64 - ) - - rn_ref_model = ReadnoiseRefModel(rn_ref) - rn_ref_model.save(readnoisefile) - - return gainfile, readnoisefile - - -@pytest.fixture -def setup_inputs(): - def _setup( - nresultants=10, - readnoise=10, - nrows=20, - ncols=20, - gain=1, - deltatime=1, - ): - err = np.ones(shape=(nresultants, nrows, ncols), dtype=np.float32) - data = np.zeros(shape=(nresultants, nrows, ncols), dtype=np.float32) - gdq = np.zeros(shape=(nresultants, nrows, ncols), dtype=np.uint8) - pixdq = np.zeros(shape=(nrows, ncols), dtype=np.uint32) - - csize = (nresultants, nrows, ncols) - dm_ramp = rdm.RampModel(maker_utils.mk_ramp(shape=csize)) - - dm_ramp.meta.instrument.name = "WFI" - dm_ramp.meta.instrument.optical_element = "F158" - - dm_ramp.data = data + 6.0 - dm_ramp.pixeldq = pixdq - dm_ramp.groupdq = gdq - dm_ramp.err = err - - dm_ramp.meta.exposure.type = "WFI_IMAGE" - dm_ramp.meta.exposure.exposure_time = deltatime - dm_ramp.meta.exposure.frame_time = deltatime - dm_ramp.meta.exposure.nresultants = nresultants - dm_ramp.meta.cal_step["dq_init"] = "INCOMPLETE" - dm_ramp.meta.cal_step["jump"] = "INCOMPLETE" - - return dm_ramp - - return _setup - - -@pytest.mark.parametrize("max_cores", MAXIMUM_CORES) -def test_one_CR(generate_wfi_reffiles, max_cores, setup_inputs): - override_gain, override_readnoise = generate_wfi_reffiles - - grouptime = 3.0 - deltaDN = 5 - ingain = 200 - inreadnoise = 7.0 - nresultants = 100 - CR_fraction = 3 - xsize = 20 - ysize = 20 - - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - deltatime=grouptime, - ) - - for i in range(nresultants): - model1.data[i, :, :] = deltaDN * i - - first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0] - - CR_locs = [x for x in range(xsize * ysize) if x % CR_fraction == 0] - CR_x_locs = [x % ysize for x in CR_locs] - CR_y_locs = [int(x / xsize) for x in CR_locs] - CR_pool = cycle(first_CR_group_locs) - - # Add CRs to the SCI data - for i in range(len(CR_x_locs)): - CR_group = next(CR_pool) - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000.0 - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores=max_cores, - use_ramp_jump_detection=False, - ) - - CR_pool = cycle(first_CR_group_locs) - for i in range(len(CR_x_locs)): - CR_group = next(CR_pool) - - assert 4 == np.max(out_model.groupdq[CR_group, CR_y_locs[i], CR_x_locs[i]]) - - -@pytest.mark.parametrize("max_cores", MAXIMUM_CORES) -def test_two_CRs(generate_wfi_reffiles, max_cores, setup_inputs): - override_gain, override_readnoise = generate_wfi_reffiles - - grouptime = 3.0 - deltaDN = 5 - ingain = 6 - inreadnoise = 7.0 - nresultants = 100 - CR_fraction = 5 - xsize = 20 - ysize = 20 - - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - deltatime=grouptime, - ) - - for i in range(nresultants): - model1.data[i, :, :] = deltaDN * i - - first_CR_group_locs = [x for x in range(1, 7) if x % 5 == 0] - CR_locs = [x for x in range(xsize * ysize) if x % CR_fraction == 0] - CR_x_locs = [x % ysize for x in CR_locs] - CR_y_locs = [int(x / xsize) for x in CR_locs] - CR_pool = cycle(first_CR_group_locs) - - for i in range(len(CR_x_locs)): - CR_group = next(CR_pool) - - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group:, CR_y_locs[i], CR_x_locs[i]] + 5000 - ) - model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] = ( - model1.data[CR_group + 8 :, CR_y_locs[i], CR_x_locs[i]] + 700 - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores=max_cores, - use_ramp_jump_detection=False, - ) - - CR_pool = cycle(first_CR_group_locs) - for i in range(len(CR_x_locs)): - CR_group = next(CR_pool) - - assert 4 == np.max(out_model.groupdq[CR_group, CR_y_locs[i], CR_x_locs[i]]) - assert 4 == np.max(out_model.groupdq[CR_group + 8, CR_y_locs[i], CR_x_locs[i]]) - - -@pytest.mark.parametrize("max_cores", MAXIMUM_CORES) -def test_two_group_integration(generate_wfi_reffiles, max_cores, setup_inputs): - override_gain, override_readnoise = generate_wfi_reffiles - grouptime = 3.0 - ingain = 6 - inreadnoise = 7 - nresultants = 2 - xsize = 20 - ysize = 20 - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - deltatime=grouptime, - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores=max_cores, - ) - - assert out_model.meta.cal_step["jump"] == "SKIPPED" - - -@pytest.mark.parametrize("use_jump", [False, True]) -def test_four_group_integration(generate_wfi_reffiles, setup_inputs, use_jump): - override_gain, override_readnoise = generate_wfi_reffiles - ingain = 6 - inreadnoise = 7 - nresultants = 4 - xsize = 20 - ysize = 20 - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores="none", - use_ramp_jump_detection=use_jump, - ) - - if use_jump: - assert out_model.meta.cal_step["jump"] == "SKIPPED" - else: - assert out_model.meta.cal_step["jump"] == "COMPLETE" - - -@pytest.mark.parametrize("use_jump", [False, True]) -def test_three_group_integration(generate_wfi_reffiles, setup_inputs, use_jump): - override_gain, override_readnoise = generate_wfi_reffiles - grouptime = 3.0 - ingain = 6 - inreadnoise = 7 - nresultants = 3 - xsize = 20 - ysize = 20 - model1 = setup_inputs( - nresultants=nresultants, - nrows=ysize, - ncols=xsize, - gain=ingain, - readnoise=inreadnoise, - deltatime=grouptime, - ) - - out_model = JumpStep.call( - model1, - override_gain=override_gain, - override_readnoise=override_readnoise, - maximum_cores="none", - use_ramp_jump_detection=use_jump, - ) - - if use_jump: - assert out_model.meta.cal_step.jump == "SKIPPED" - else: - assert out_model.meta.cal_step.jump == "COMPLETE" diff --git a/romancal/regtest/test_jump_det.py b/romancal/regtest/test_jump_det.py deleted file mode 100644 index 768ee6888..000000000 --- a/romancal/regtest/test_jump_det.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Regression test for the Jump detection step.""" - -import pytest - -from romancal.stpipe import RomanStep - -from .regtestdata import compare_asdf - - -@pytest.mark.bigdata -@pytest.mark.skip(reason="Jump detection step is being removed from the pipeline") -def test_jump_detection_step(rtdata, ignore_asdf_paths): - """Function to run and compare Jump Detection files. Note: This should - include tests for overrides etc.""" - - input_file = "r0000101001001001001_0001_wfi01_darkcurrent.asdf" - rtdata.get_data(f"WFI/image/{input_file}") - rtdata.input = input_file - - # Note: the thresholds should be reset to the defaults once we have better - # input data - args = [ - "romancal.step.JumpStep", - rtdata.input, - "--rejection_threshold=180.", - "--three_group_rejection_threshold=185.", - "--four_group_rejection_threshold=190.", - # disable the jump detection in the rampfit step so this one will run - "--run_ramp_jump_detection=False", - ] - RomanStep.from_cmdline(args) - output = "r0000101001001001001_0001_wfi01_jump.asdf" - rtdata.output = output - rtdata.get_truth(f"truth/WFI/image/{output}") - diff = compare_asdf(rtdata.output, rtdata.truth, **ignore_asdf_paths) - assert diff.identical, diff.report() From 531a27a7104a176ead20c85ef34b7ff42b45256c Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Thu, 21 Nov 2024 15:59:56 -0500 Subject: [PATCH 02/19] rcal-954 Update git ignore for emacs backup files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 75d400001..5312b3528 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,8 @@ distribute-*.tar.gz .tox .*.sw[op] *~ +#* +.#* .project .pydevproject .settings From 9c143f298c42f985c8e2122541e3065ee778f390 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Fri, 22 Nov 2024 09:02:26 -0500 Subject: [PATCH 03/19] rcal-954 Add change text --- changes/1534.general.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/1534.general.rst diff --git a/changes/1534.general.rst b/changes/1534.general.rst new file mode 100644 index 000000000..1a8fc3458 --- /dev/null +++ b/changes/1534.general.rst @@ -0,0 +1 @@ +Remove the jump code for the depricated jump detection for even ramps and update the documentation From eadd35c3ad36331e1f471ecddad9a6d7f705393c Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Fri, 22 Nov 2024 09:11:17 -0500 Subject: [PATCH 04/19] rcal-954 Change congif file name in user pipeline docs --- docs/roman/stpipe/user_pipeline.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/roman/stpipe/user_pipeline.rst b/docs/roman/stpipe/user_pipeline.rst index 9176baf8c..455490c27 100644 --- a/docs/roman/stpipe/user_pipeline.rst +++ b/docs/roman/stpipe/user_pipeline.rst @@ -123,9 +123,9 @@ is equivalent to: - class: romancal.jump.jump_step.JumpStep name: jump parameters: - config_file = myjump.asdf + config_file = myconfig.asdf -with the file ``myjump.asdf.`` in the same directory: +with the file ``myconfig.asdf.`` in the same directory: .. code-block:: yaml From 3b9db97fb965c0333eb8e325ab12f9343f491b9a Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Fri, 22 Nov 2024 15:21:47 -0500 Subject: [PATCH 05/19] rcal-954 updates for run --- docs/roman/pipeline_run.rst | 12 ++++++++---- docs/roman/stpipe/user_pipeline.rst | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/roman/pipeline_run.rst b/docs/roman/pipeline_run.rst index 2697fdb74..62f266756 100644 --- a/docs/roman/pipeline_run.rst +++ b/docs/roman/pipeline_run.rst @@ -82,10 +82,12 @@ For the exposure pipeline and steps, :: from romancal.pipeline import ExposurePipeline - result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + elp = ExposurePipeline() + result = elp('r0000101001001001001_0001_wfi01_uncal.asdf') from romancal.linearity import LinearityStep - result = LinearityStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + linearity = LinearityStep() + result = linearity('r0000101001001001001_0001_wfi01_uncal.asdf') One difference between the mosaic level pipeline and the exposure level pipeline is that the mosaic level pipeline is generally designed to run on multiple overlapping exposures. To achieve @@ -95,10 +97,12 @@ For the mosaic level pipeline and steps, :: from romancal.pipeline import MosaicPipeline - result = ExposurePipeline.call('r0000101001001001001_asn.json') + mosp = MosaicPipeline() + result = mosp('r0000101001001001001_asn.json') from romancal.skymatch import SkyMatchStep - result = SkyMatchStep.call('r0000101001001001001_asn.json') + skymatch = SkyMatchStep() + result = skymatch('r0000101001001001001_asn.json') For more information, see :ref:`Execute via call()` diff --git a/docs/roman/stpipe/user_pipeline.rst b/docs/roman/stpipe/user_pipeline.rst index 455490c27..fddd785c1 100644 --- a/docs/roman/stpipe/user_pipeline.rst +++ b/docs/roman/stpipe/user_pipeline.rst @@ -186,7 +186,7 @@ From Python Once the pipeline has been configured (as above) it can be executed using run. - pipe.run(input_data) + pipe(input_data) Caching details --------------- From a02a18d485ec6e32be4354655afd9674021ebe44 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Mon, 2 Dec 2024 13:58:27 -0500 Subject: [PATCH 06/19] rcal-954 remove unused ref in docs --- docs/roman/stpipe/call_via_run.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/roman/stpipe/call_via_run.rst b/docs/roman/stpipe/call_via_run.rst index 014ef62f8..2f83b12d6 100644 --- a/docs/roman/stpipe/call_via_run.rst +++ b/docs/roman/stpipe/call_via_run.rst @@ -6,8 +6,7 @@ Executing a pipeline or pipeline step directly, or via run() When calling a pipeline or step instance directly, or using the ``run`` method, you can specify individual parameter values manually. In this case, parameter files are not used. If you use ``run`` after instantiating with a parameter -file (as is done when using the :ref:`call` method), the -parameter file will be ignored. +file, the parameter file will be ignored. :: From 3564dea5bde8b1be785422b967f671ea55f09765 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 10 Dec 2024 09:15:44 -0500 Subject: [PATCH 07/19] rcal-954 Resolve merge conflicts --- ...> r0000501001001001001_0001_wfi01_cal.png} | Bin ...0501001001001001_0001_wfi01_cal_thumb.png} | 0 docs/roman/associations/asn_from_list.rst | 8 +- docs/roman/data_products/file_naming.rst | 8 +- docs/roman/datamodels/#datamodels_asdf.rst# | 313 ++++++++++++++++++ docs/roman/datamodels/.#datamodels_asdf.rst | 1 + docs/roman/datamodels/datamodels_asdf.rst | 8 +- docs/roman/datamodels/models.rst | 6 +- docs/roman/jump/arguments.rst | 50 --- docs/roman/jump/description.rst | 77 ----- docs/roman/jump/even_ramp.rst | 17 - docs/roman/jump/index.rst | 11 - docs/roman/jump/reference_files.rst | 15 - docs/roman/jump/uneven_ramp.rst | 13 - docs/roman/outlier_detection/main.rst | 12 +- docs/roman/package_index.rst | 1 - docs/roman/pipeline/exposure_pipeline.rst | 6 +- docs/roman/pipeline/mosaic_pipeline.rst | 2 +- docs/roman/pipeline_naming_conventions.rst | 4 +- docs/roman/pipeline_parameters.rst | 8 +- docs/roman/pipeline_run.rst | 10 +- docs/roman/pipeline_static_preview.rst | 8 +- docs/roman/pipeline_steps.rst | 1 - .../references_general/references_general.rst | 12 - docs/roman/stpipe/call_via_call.rst | 72 ---- docs/roman/stpipe/call_via_run.rst | 11 +- docs/roman/stpipe/config_asdf.rst | 108 +++--- docs/roman/stpipe/index.rst | 1 - docs/roman/stpipe/parameter_files.rst | 4 +- docs/roman/stpipe/user_pipeline.rst | 52 +-- docs/roman/stpipe/user_step.rst | 53 +-- romancal/lib/suffix.py | 7 +- romancal/pipeline/exposure_pipeline.py | 55 ++- romancal/step.py | 2 - romancal/stpipe/integration.py | 1 - 35 files changed, 505 insertions(+), 452 deletions(-) rename docs/images/{r0000501001001001001_01101_0001_WFI01_cal.png => r0000501001001001001_0001_wfi01_cal.png} (100%) rename docs/images/{r0000501001001001001_01101_0001_WFI01_cal_thumb.png => r0000501001001001001_0001_wfi01_cal_thumb.png} (100%) create mode 100644 docs/roman/datamodels/#datamodels_asdf.rst# create mode 120000 docs/roman/datamodels/.#datamodels_asdf.rst delete mode 100644 docs/roman/jump/arguments.rst delete mode 100644 docs/roman/jump/description.rst delete mode 100644 docs/roman/jump/even_ramp.rst delete mode 100644 docs/roman/jump/index.rst delete mode 100644 docs/roman/jump/reference_files.rst delete mode 100644 docs/roman/jump/uneven_ramp.rst delete mode 100644 docs/roman/stpipe/call_via_call.rst diff --git a/docs/images/r0000501001001001001_01101_0001_WFI01_cal.png b/docs/images/r0000501001001001001_0001_wfi01_cal.png similarity index 100% rename from docs/images/r0000501001001001001_01101_0001_WFI01_cal.png rename to docs/images/r0000501001001001001_0001_wfi01_cal.png diff --git a/docs/images/r0000501001001001001_01101_0001_WFI01_cal_thumb.png b/docs/images/r0000501001001001001_0001_wfi01_cal_thumb.png similarity index 100% rename from docs/images/r0000501001001001001_01101_0001_WFI01_cal_thumb.png rename to docs/images/r0000501001001001001_0001_wfi01_cal_thumb.png diff --git a/docs/roman/associations/asn_from_list.rst b/docs/roman/associations/asn_from_list.rst index 3b848a7f5..d32ff2fa7 100644 --- a/docs/roman/associations/asn_from_list.rst +++ b/docs/roman/associations/asn_from_list.rst @@ -19,7 +19,7 @@ To create an association, use the following command: import romancal.associations.asn_from_list as asn_from_list product_name = 'test_product' - items = {'r0000101001001001001_01101_0001_WFI01_uncal.asdf': 'science', 'r0000101001001001001_3_01101_0001_WFI01_uncal.asdf': 'guide_window', 'c': 'somethingelse'} + items = {'r0000101001001001001_0001_wfi01_uncal.asdf': 'science', 'r0000101001001001001_3_0001_wfi01_uncal.asdf': 'guide_window', 'c': 'somethingelse'} asn = asn_from_list.asn_from_list([(item, type_) for item, type_ in items.items()], product_name=product_name, with_exptype=True) asn['asn_rule'] 'DMS_ELPP_Base' @@ -29,9 +29,9 @@ an example product that has both a science and guide window exposures would look like the following:: asn['products'] - [ { 'members': [ { 'expname': 'r0000101001001001001_01101_0001_WFI01_uncal.asdf', + [ { 'members': [ { 'expname': 'r0000101001001001001_0001_wfi01_uncal.asdf', 'exptype': 'science'}, - { 'expname': 'r0000101001001001001_3_01101_0001_WFI01_uncal.asdf', + { 'expname': 'r0000101001001001001_3_0001_wfi01_uncal.asdf', 'exptype': 'guide_window'}, {'expname': 'c', 'exptype': 'somethingelse'}], 'name': 'test_product'}] @@ -41,7 +41,7 @@ To create a association with all the detectors for a given exposure from the com .. code-block:: python - asn_from_list -o detector_asn.json --product-name r0000101001001001001_01101_0001_WFI data/*_cal.asdf + asn_from_list -o detector_asn.json --product-name r0000101001001001001_0001_wfi data/*_cal.asdf where the individual calibrated detector files are in a data subdirectory. diff --git a/docs/roman/data_products/file_naming.rst b/docs/roman/data_products/file_naming.rst index 5df0fde3e..dfe30270a 100644 --- a/docs/roman/data_products/file_naming.rst +++ b/docs/roman/data_products/file_naming.rst @@ -11,7 +11,7 @@ The names of the exposure level data are constructed with information from the science header of the exposure, allowing users to map it to the observation in their corresponding APT files. The ASDF file naming scheme for the Exposure products is:: - r____.asdf + r___.asdf where @@ -25,7 +25,7 @@ where - s: sequence ID (1=prime, >1 parallel) - aa: activity number (base 36) - eeee: exposure number - - detector: detector name (e.g. WFI01, WFI02, ...) + - detector: detector name (e.g. wfi01, wfi02, ...) - prodType: product type identifier (e.g. 'uncal', 'cal') The standard for the pipeline are uncal and cal, for the input products and resulting @@ -35,8 +35,8 @@ linearity, jump, darkcurrent, rampfit, assignwcs, flat, and photom. An example Exposure Level 1 upcalibrated ASDF file name is:: - r0000101001001001001_01101_0001_WFI01_uncal.asdf + r0000101001001001001_0001_wfi01_uncal.asdf An example Exposure Level 2 product ASDF file name is:: - r0000101001001001001_01101_0001_WFI01_cal.asdf + r0000101001001001001_0001_wfi01_cal.asdf diff --git a/docs/roman/datamodels/#datamodels_asdf.rst# b/docs/roman/datamodels/#datamodels_asdf.rst# new file mode 100644 index 000000000..03cc2a481 --- /dev/null +++ b/docs/roman/datamodels/#datamodels_asdf.rst# @@ -0,0 +1,313 @@ +.. _datamodels_asdf: + + +Working with Roman datamodels and ASDF files +============================================ + +Please refer to `Roman Documentation `_ +for more details about `roman_datamodels`. + +This section assumes that you are familiar with the ASDF standard format. +If that's not the case, a good starting point would be to go over the `ASDF Documentation `_. + +If you have installed the roman calibration pipeline you should also have access +to the standalone tool asdfinfo which allows access to `ASDF `_ (and roman) files +from the terminal prompt,:: + + asdftool info r0000101001001001001_0001_wfi16_cal.asdf + root (AsdfObject) + ├─asdf_library (Software) + │ ├─author (str): The ASDF Developers + ... + +also useful is:: + + asdftool help + usage: asdftool [-h] [--verbose] {help,explode,implode,extract,defragment,diff,edit,remove-hdu,info,extensions,tags,to_yaml} ... + + Commandline utilities for managing ASDF files. + + optional arguments: + -h, --help show this help message and exit + --verbose, -v Increase verbosity + + subcommands: + + {help,explode,implode,extract,defragment,diff,edit,remove-hdu,info,extensions,tags,to_yaml} + help Display usage information + explode Explode a ASDF file. + implode Implode a ASDF file. + extract Extract ASDF extensions in ASDF-in-FITS files into pure ASDF files + defragment Defragment an ASDF file.. + diff Report differences between two ASDF files + remove-hdu Remove ASDF extension from ASDF-in-FITS file + info Print a rendering of an ASDF tree. + extensions Show information about installed extensions + tags List currently available tags + to_yaml Convert as ASDF file to pure YAML. + + +which gives a list of possible actions one of the more useful can be:: + + asdftool edit file.asdf + +Which will open the file in an editor you have set via the EDITOR environment variable. +A more complete description of the options can be found in the +`asdftool `_ +documentation. + +To access the files via a python session, + +.. code-block:: python + + import roman_datamodels as rdm + import asdf + with rdm.open('r0000101001001001001_0001_wfi16_cal.asdf') as model: + + + with asdf.open('r0000101001001001001_0001_wfi16_cal.asdf', memmap=False) as model: + + +Once the files are loaded you can access various attributes. Below is a table +showing how to access various properties using the roman_datamodels and the +asdf.open methods, + ++--------------------------------------+---------------------------------------------------------------+ +| Roman Datamodels | ASDF | ++--------------------------------------+---------------------------------------------------------------+ +| .. code-block:: python | .. code-block:: python | +| | | +| rdm_a.meta | asdf_a.tree['roman']['meta'] | +| rdm_a.meta.aperture | asdf_a.tree['roman']['meta']['aperture'] | +| rdm_a.meta.aperture.position_angle | asdf_a.tree['roman']['meta']['aperture']['position_angle'] | +| 120 | 120 | ++--------------------------------------+---------------------------------------------------------------+ + +You can also update or modify the metadata in Roman datamodels + +.. code-block:: python + + rdm_a.meta.aperture.position_angle = 120.21 + rdm_a.meta.aperture.position_angle + 120.21 + +The ASDF equivalent is + +.. code-block:: python + + asdf_a.tree['roman']['meta']['aperture']['position_angle'] = 120.05 + asdf_a.tree['roman']['meta']['aperture']['position_angle'] + 120.05 + +.. HINT:: + + If you trigger an error, + "ValueError: assignment destination is read-only" + make sure the asdf file was opened with memmap=False, or + with mode='rw' + +You can also access and modify the data arrays + +.. code-block:: python + :caption: Roman Datamodels + + rdm_a.data + + + rdm_a.data[10,11] + 0.0 + + rdm_a.data[10,11] = 122.1 + rdm_a.data[10,11] + 122.1 + +or by modifying the ASDF tree, + +.. code-block:: python + :caption: ASDF + + asdf_a.tree['roman']['data'] + + + asdf_a.tree['roman']['data'][10,11] + 0.0 + + asdf_a.tree['roman']['data'][10,11] = 3.14159 + asdf_a.tree['roman']['data'][10,11] + 3.14159 + +Using the info method +--------------------- + +You can examine a roman data model using the info and search methods provided +from the asdf package. The info function will print a representation of the +asdf tree. + +.. code:: python + + >>> from roman_datamodels import datamodels as rdm # doctest: +SKIP + >>> d_uncal = rdm.open('r0000101001001001001 _0001_wfi01_uncal.asdf') # doctest: +SKIP + >>> d_uncal.info() # doctest: +SKIP + root (AsdfObject) + ├─asdf_library (Software) + │ ├─author (str): The ASDF Developers + │ ├─homepage (str): http://github.com/asdf-format/asdf + │ ├─name (str): asdf + │ └─version (str): 2.8.1 + ├─history (dict) + │ └─extensions (list) + │ ├─[0] (ExtensionMetadata) ... + │ ├─[1] (ExtensionMetadata) ... + │ └─[2] (ExtensionMetadata) ... + └─roman (WfiScienceRaw) + ├─meta (dict) + │ ├─aperture (Aperture) ... + │ ├─cal_step (L2CalStep) ... + │ ├─calibration_software_version (str): 0.4.3.dev89+gca5771d + │ ├─coordinates (Coordinates) ... + │ ├─crds_context_used (str): roman_0020.pmap + │ ├─crds_software_version (str): 11.5.0 + │ ├─ephemeris (Ephemeris) ... + │ ├─exposure (Exposure) ... + │ └─17 not shown + └─data (NDArrayType): shape=(8, 4096, 4096), dtype=uint16 + Some nodes not shown. + +The info command also gives you control over the number of lines displayed +by passing the argument ``max_rows``. As an integer, ``max_rows`` +will be interpreted as an overall limit on the number of displayed lines. +If ``max_rows`` is a tuple, then each member limits lines per node at the +depth corresponding to its tuple index. +For example, to show all top-level nodes and 5 of each's children: + +.. code:: python + + >>> d_uncal.info(max_rows=(None,5)) # doctest: +SKIP + root (AsdfObject) + ├─asdf_library (Software) + │ ├─author (str): The ASDF Developers + │ ├─homepage (str): http://github.com/asdf-format/asdf + │ ├─name (str): asdf + │ └─version (str): 2.8.1 + ├─history (dict) + │ └─extensions (list) ... + └─roman (WfiScienceRaw) + ├─meta (dict) ... + └─data (NDArrayType): shape=(8, 4096, 4096), dtype=uint16 + Some nodes not shown. + +Or you can use the asdf.info method to view the contents of the tree + +.. code:: python + + >> asdf.info(d_uncal) + +Will print the same information as the above `d_uncal.info` command but also +gives you enhanced capabilities. For instance you can display the first three +lines for each of the meta entries, + +.. code:: python + + >>> asdf.info(d_uncal.meta, max_rows=(None, 3)) # doctest: +SKIP + root (DNode) + ├─aperture (Aperture) + │ ├─name (str): WFI_CEN + │ └─position_angle (int): 120 + ├─cal_step (L2CalStep) + │ ├─assign_wcs (str): INCOMPLETE + │ ├─flat_field (str): INCOMPLETE + │ └─6 not shown + ├─calibration_software_version (str): 0.4.3.dev89+gca5771d + ├─coordinates (Coordinates) + │ └─reference_frame (str): ICRS + ├─crds_context_used (str): roman_0020.pmap + ├─crds_software_version (str): 11.5.0 + ├─ephemeris (Ephemeris) + │ ├─earth_angle (float): 3.3161255787892263 + │ ├─moon_angle (float): 3.3196162372932148 + │ └─10 not shown + ... + +or you can concentrate on a given attribute. To list all the attributes +in `cal_step` without listing the values, + +.. code:: python + + >>> asdf.info(d_uncal.meta.cal_step,max_rows=(None, 3),show_values=False) # doctest: +SKIP + root (L2CalStep) + ├─assign_wcs (str) + ├─flat_field (str) + ├─dark (str) + ├─dq_init (str) + ├─jump (str) + ├─linearity (str) + ├─ramp_fit (str) + └─saturation (str) + +More information on the info method can be found in the ASDF documentation at +`rendering the ASDF trees. `_ + +Using the search method +----------------------- + +You can also use the search method to find attributes, + +.. code:: python + + >>> d_uncal.search('cal_step') # doctest: +SKIP + root (AsdfObject) + └─roman (WfiScienceRaw) + └─meta (dict) + └─cal_step (L2CalStep) + +or a a general search for all attributes with cal in the name + +.. code:: python + + >>> d_uncal.search('cal') # doctest: +SKIP + root (AsdfObject) + └─roman (WfiScienceRaw) + └─meta (dict) + ├─cal_step (L2CalStep) + ├─calibration_software_version (str): 0.4.3.dev89+gca5771d + ├─instrument (WfiMode) + │ └─optical_element (str): F158 + └─velocity_aberration (VelocityAberration) + └─scale_factor (float): 0.9999723133902021 + +This will do a regular expression search for `cal` in the attribute name. More +information on using regular expressions in the search method can be found +in the ASDF documentation linked below. + +To search only within the meta tree, + +.. code:: python + + >>> d_uncal.search('cal_')['roman']['meta'] # doctest: +SKIP + meta (dict) + ├─cal_step (L2CalStep) + └─instrument (WfiMode) + └─optical_element (str): F158 + +You can also use the search method to find attributes by type in the asdf tree. +For instance, you can find all integers, floats, or booleans by using the type +keyword, + +.. code:: python + + >>> d_uncal.search(type=bool) # doctest: +SKIP + root (AsdfObject) + └─roman (WfiScienceRaw) + └─meta (dict) + ├─exposure (Exposure) + │ └─data_problem (bool): False + └─visit (Visit) + ├─internal_target (bool): False + └─target_of_opportunity (bool): False + + >>> d_uncal.search(type=bool, value=True) # doctest: +SKIP + No results found. + +More information and options for the search method can be found in the +ASDF documentation +`here. `_ diff --git a/docs/roman/datamodels/.#datamodels_asdf.rst b/docs/roman/datamodels/.#datamodels_asdf.rst new file mode 120000 index 000000000..8cb62536e --- /dev/null +++ b/docs/roman/datamodels/.#datamodels_asdf.rst @@ -0,0 +1 @@ +ddavis@rugila.fios-router.home.stsci.edu.12833 \ No newline at end of file diff --git a/docs/roman/datamodels/datamodels_asdf.rst b/docs/roman/datamodels/datamodels_asdf.rst index b75971ad1..eb76a0561 100644 --- a/docs/roman/datamodels/datamodels_asdf.rst +++ b/docs/roman/datamodels/datamodels_asdf.rst @@ -14,7 +14,7 @@ If you have installed the roman calibration pipeline you should also have access to the standalone tool asdfinfo which allows access to `ASDF `_ (and roman) files from the terminal prompt,:: - asdftool info r0000101001001001001_01101_0001_WFI16_cal.asdf + asdftool info r0000101001001001001_0001_wfi16_cal.asdf root (AsdfObject) ├─asdf_library (Software) │ ├─author (str): The ASDF Developers @@ -62,10 +62,10 @@ To access the files via a python session, import roman_datamodels as rdm import asdf - with rdm.open('r0000101001001001001_01101_0001_WFI16_cal.asdf') as model: + with rdm.open('r0000101001001001001_0001_wfi16_cal.asdf') as model: - with asdf.open('r0000101001001001001_01101_0001_WFI16_cal.asdf', memmap=False) as model: + with asdf.open('r0000101001001001001_0001_wfi16_cal.asdf', memmap=False) as model: Once the files are loaded you can access various attributes. Below is a table @@ -146,7 +146,7 @@ asdf tree. .. code:: python >>> from roman_datamodels import datamodels as rdm # doctest: +SKIP - >>> d_uncal = rdm.open('r0000101001001001001_01101_0001_WFI01_uncal.asdf') # doctest: +SKIP + >>> d_uncal = rdm.open('r0000101001001001001_0001_wfi01_uncal.asdf') # doctest: +SKIP >>> d_uncal.info() # doctest: +SKIP root (AsdfObject) ├─asdf_library (Software) diff --git a/docs/roman/datamodels/models.rst b/docs/roman/datamodels/models.rst index d46f30161..47beb3260 100644 --- a/docs/roman/datamodels/models.rst +++ b/docs/roman/datamodels/models.rst @@ -36,7 +36,7 @@ using python. .. testsetup:: >>> from roman_datamodels import datamodels as rdm >>> from roman_datamodels.maker_utils import mk_datamodel - >>> fn = 'r0019106003005004023_03203_0034_WFI01_cal.asdf' + >>> fn = 'r0019106003005004023_0034_wfi01_cal.asdf' >>> image_model = mk_datamodel(rdm.ImageModel) # set some metadata for the below tests @@ -46,7 +46,7 @@ using python. .. code-block:: python >>> from roman_datamodels import datamodels as rdm - >>> fn = 'r0019106003005004023_03203_0034_WFI01_cal.asdf' + >>> fn = 'r0019106003005004023_0034_wfi01_cal.asdf' >>> image_model = rdm.open(fn) # doctest: +SKIP >>> type(image_model) @@ -145,7 +145,7 @@ You can examine the contents of your model from within python using meta.calibration_software_name RomanCAL meta.calibration_software_version 9.9.0 meta.product_type l2 - meta.filename r0019106003005004023_03203_0034_WFI01_cal.asdf + meta.filename r0019106003005004023_0034_wfi01_cal.asdf meta.file_date 2020-01-01T00:00:00.000 meta.model_type ImageModel meta.origin STSCI/SOC diff --git a/docs/roman/jump/arguments.rst b/docs/roman/jump/arguments.rst deleted file mode 100644 index f77fafe80..000000000 --- a/docs/roman/jump/arguments.rst +++ /dev/null @@ -1,50 +0,0 @@ -Arguments -========= - -The ``jump`` step has five optional arguments that can be set by the user: - -* ``--rejection_threshold``: A floating-point value that sets the sigma - threshold for jump detection for ramps having 5 or more groups.. In the code - sigma is determined using the read noise from the read noise reference file - and the Poisson noise (based on the median difference between samples, and the - gain reference file). Note that any noise source beyond these two that may be - present in the data will lead to an increase in the false positive rate and - thus may require an increase in the value of this parameter. The default value - of 4.0 for the rejection threshold will yield 6200 false positives for every - million pixels, if the noise model is correct. - -* ``--three_group_rejection_threshold``: A floating-point value that sets the - sigma threshold for jump detection for ramps having exactly 3 groups. The - default value is 6.0 - -* ``--four_group_rejection_threshold``: A floating-point value that sets the - sigma threshold for jump detection for ramps having exactly 4 groups. The - default value is 5.0 - -* ``--maximum_cores``: The fraction of available cores that will be - used for multi-processing in this step. The default value is 'none' which does not use - multi-processing. The other options are 'quarter', 'half', and 'all'. Note that these - fractions refer to the total available cores and on most CPUs these include physical - and virtual cores. The clock time for the step is reduced - almost linearly by the number of physical cores used on all machines. For example, on an Intel CPU with - six real cores and 6 virtual cores setting maximum_cores to 'half' results in a - decrease of a factor of six in the clock time for the step to run. Depending on the system - the clock time can also decrease even more with maximum_cores is set to 'all'. - -* ``--flag_4_neighbors``: If set to True (default is True) it will cause the four perpendicular - neighbors of all detected jumps to be flagged as a jump. This is needed because of - the inter-pixel capacitance (IPC) causing a small jump in the neighbors. The small jump - might be below the rejection threshold but will affect the slope determination of - the pixel. The step will take about 40% longer to run when this is set to True. - -* ``--max_jump_to_flag_neighbors``: A floating point value in units of sigma that limits - the flagging of neighbors. Any jump above this cutoff will not have its neighbors flagged. - The concept is that the jumps in neighbors will be above the rejection-threshold and thus - be flagged as primary jumps. The default value is 1000. - -* ``--min_jump_to_flag_neighbors``: A floating point value in units of sigma that limits - the flagging of neighbors of marginal detections. Any primary jump below this value will - not have its neighbors flagged. The goal is to prevent flagging jumps that would be too - small to significantly affect the slope determination. The default value is 10. - -* ``--use_ramp_jump_detection``: See the description in :ref:`ramp fitting `. diff --git a/docs/roman/jump/description.rst b/docs/roman/jump/description.rst deleted file mode 100644 index 295863460..000000000 --- a/docs/roman/jump/description.rst +++ /dev/null @@ -1,77 +0,0 @@ -Description -=========== - -Assumptions ------------ -We assume that the ``saturation`` step has already been applied to the input -science exposure, so that saturated values are appropriately flagged in the -input GROUPDQ array. We also assume that steps such as the reference pixel -correction (``refpix``) and non-linearity correction (``linearity``) have been applied, so -that the input data ramps do not have any non-linearities or noise above the modeled Poission -and read noise due to instrumental effects. The absence of any of these preceding corrections -or residual non-linearities or noise can lead to the false detection of jumps in the ramps, -due to departure from linearity. - -The ``jump`` step will automatically skip execution if the input data contain fewer -than 3 groups in the integration, because the baseline algorithm requires two first -differences to work. - -Algorithm ---------- -This routine detects jumps in an exposure by looking for outliers -in the up-the-ramp signal for each pixel in the integration within -an input exposure. On output, the GROUPDQ array is updated with the DQ flag -"JUMP_DET" to indicate the location of each jump that was found. -In addition, any pixels that have non-positive or NaN values in the gain -reference file will have DQ flags "NO_GAIN_VALUE" and "DO_NOT_USE" set in the -output PIXELDQ array. -The SCI and ERR arrays of the input data are not modified. - -The current implementation uses the two-point difference method described -in Anderson&Gordon2011_. - -Two-Point Difference Method -^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The two-point difference method is applied to the integration as follows: - -* Compute the first differences for each pixel (the difference between - adjacent groups) -* Compute the clipped (dropping the largest difference) median of the first differences for each pixel. -* Use the median to estimate the Poisson noise for each group and combine it - with the read noise to arrive at an estimate of the total expected noise for - each difference. -* Compute the "difference ratio" as the difference between the first differences - of each group and the median, divided by the expected noise. -* If the largest "difference ratio" is greater than the rejection threshold, - flag the group corresponding to that ratio as having a jump. -* If a jump is found in a given pixel, iterate the above steps with the - jump-impacted group excluded, looking for additional lower-level jumps - that still exceed the rejection threshold. -* Stop iterating on a given pixel when no new jumps are found or only one difference remains. -* If the there are only three differences (four groups), the standard median is used rather than - the clipped median. -* If there are only two differences (three groups), the smallest one is compared to the larger - one and if the larger one is above a threshold, it is flagged as a jump. - -Note that any ramp values flagged as SATURATED in the input GROUPDQ array -are not used in any of the above calculations and hence will never be -marked as containing a jump. - -Multiprocessing -=============== -This step has the option of running in multiprocessing mode. In that mode it will -split the input data cube into a number of row slices based on the number of available -cores on the host computer and the value of the max_cores input parameter. By -default the step runs on a single processor. At the other extreme if max_cores is -set to 'all', it will use all available cores (real and virtual). Testing has shown -a reduction in the elapsed time for the step proportional to the number of real -cores used. Using the virtual cores also reduces the elapsed time but at a slightly -lower rate than the real cores. - -If multiprocessing is requested the input cube will be divided into a number of -slices in the row dimension (with the last slice being slightly larger, if needed). -The slices are then sent to twopoint_difference.py by detect_jumps. After all the -slices have finished processing, detect_jumps assembles the output group_dq cube -from the slices. - -.. _Anderson&Gordon2011: https://ui.adsabs.harvard.edu/abs/2011PASP..123.1237A diff --git a/docs/roman/jump/even_ramp.rst b/docs/roman/jump/even_ramp.rst deleted file mode 100644 index 8748b8ff5..000000000 --- a/docs/roman/jump/even_ramp.rst +++ /dev/null @@ -1,17 +0,0 @@ -============================= -Jump Detection for Even Ramps -============================= - -.. warning:: - - This step can only be run on evenly-spaced ramps. Using this step - will turn off the jump detection algorithm integrated into ramp fitting. - -.. toctree:: - :maxdepth: 2 - - description.rst - arguments.rst - reference_files.rst - -.. automodapi:: romancal.jump diff --git a/docs/roman/jump/index.rst b/docs/roman/jump/index.rst deleted file mode 100644 index fa1850c8f..000000000 --- a/docs/roman/jump/index.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _jump_step: - -============== -Jump Detection -============== - -.. toctree:: - :maxdepth: 3 - - uneven_ramp.rst - even_ramp.rst diff --git a/docs/roman/jump/reference_files.rst b/docs/roman/jump/reference_files.rst deleted file mode 100644 index de77cc0f2..000000000 --- a/docs/roman/jump/reference_files.rst +++ /dev/null @@ -1,15 +0,0 @@ -Reference File Types -===================== - -The ``jump`` step uses two reference files: :ref:`GAIN ` -and :ref:`READNOISE `. -The GAIN reference file is used to temporarily convert pixel values in -the ``jump`` step from units of DN to electrons. -The READNOISE reference file is used in estimating the expected noise -in each pixel. -Both are necessary for proper computation of noise estimates within the -``jump`` step. - -:ref:`GAIN ` - -:ref:`READNOISE ` diff --git a/docs/roman/jump/uneven_ramp.rst b/docs/roman/jump/uneven_ramp.rst deleted file mode 100644 index 95165f17a..000000000 --- a/docs/roman/jump/uneven_ramp.rst +++ /dev/null @@ -1,13 +0,0 @@ -=============================== -Jump Detection for Uneven Ramps -=============================== - -.. note:: - - This version of jump detection will work on both unevenly-spaced and - evenly-spaced ramps. So there is no need to worry about which type of - of ramp you have. - -Description ------------ -This "step" is actually integrated into the :ref:`unevenly-spaced ramp fitting algorithm`. diff --git a/docs/roman/outlier_detection/main.rst b/docs/roman/outlier_detection/main.rst index 765c07378..62126e4a3 100644 --- a/docs/roman/outlier_detection/main.rst +++ b/docs/roman/outlier_detection/main.rst @@ -4,11 +4,13 @@ Description :Classes: `romancal.outlier_detection.OutlierDetectionStep` :Aliases: outlier_detection, outlier_detection_scaled, outlier_detection_stack -Processing multiple datasets together allows for the identification of bad pixels -or cosmic-rays that remain in each of the input images, many times at levels which -were not detectable by the :ref:`jump ` step. The ``outlier_detection`` step -implements the following algorithm to identify and flag any remaining cosmic-rays or -other artifacts left over from previous calibrations: +Processing multiple datasets together allows for the identification of +bad pixels or cosmic-rays that remain in each of the input images, +many times at levels which were not detectable by jump detection in +:ref:`ramp_fitting ` step. The +``outlier_detection`` step implements the following algorithm to +identify and flag any remaining cosmic-rays or other artifacts left +over from previous calibrations: - build a stack of input data diff --git a/docs/roman/package_index.rst b/docs/roman/package_index.rst index bc03ed8ce..34a31b3fd 100644 --- a/docs/roman/package_index.rst +++ b/docs/roman/package_index.rst @@ -7,7 +7,6 @@ dq_init/index.rst flatfield/index.rst flux/index.rst - jump/index.rst linearity/index.rst outlier_detection/index.rst pipeline/index.rst diff --git a/docs/roman/pipeline/exposure_pipeline.rst b/docs/roman/pipeline/exposure_pipeline.rst index dda8ee7a9..30f81d863 100644 --- a/docs/roman/pipeline/exposure_pipeline.rst +++ b/docs/roman/pipeline/exposure_pipeline.rst @@ -43,11 +43,11 @@ The ``exposure`` pipeline has an optional argument:: --use_ramp_jump_detection boolean default=True -When set to ``True``, the pipeline will perform :ref:`jump ` detection as a part of the ramp +When set to ``True``, the pipeline will perform jump detection as a part of the ramp fitting step. The data at this stage of the pipeline are still in the form of the original 3D ramps ( ngroups x ncols x nrows ) and have had all of the detector-level correction steps applied to it, up to but not including the detection and flagging of -Cosmic-Ray (CR) hits within each ramp (integration). For this case the :ref:`jump ` +Cosmic-Ray (CR) hits within each ramp (integration). For this case the jump detection module in :ref:`ramp_fitting ` will update the dq array with the CR hits (jumps) that are identified in the step. @@ -62,7 +62,7 @@ Inputs :File suffix: _uncal The input to the ``ExposurePipeline`` can be a single raw exposure, -e.g. "r0008308002010007027_06311_0019_WFI01_uncal.asdf", which contains the +e.g. "r0008308002010007027_0019_wfi01_uncal.asdf", which contains the original raw data from all of the detector readouts in the exposure ( ngroups x ncols x nrows ). The raw data may also be input using an association file. diff --git a/docs/roman/pipeline/mosaic_pipeline.rst b/docs/roman/pipeline/mosaic_pipeline.rst index 6e53179d0..262373a33 100644 --- a/docs/roman/pipeline/mosaic_pipeline.rst +++ b/docs/roman/pipeline/mosaic_pipeline.rst @@ -54,7 +54,7 @@ An association of 2D calibrated image data :File suffix: _cal The input to the ``MosaicPipeline`` is a group of calibrated exposures, -e.g. "r0008308002010007027_06311_0019_WFI01_cal.asdf", which contains the +e.g. "r0008308002010007027_0019_wfi01_cal.asdf", which contains the calibrated data for the the exposures. The most convenient way to pass the list of exposures to be processed with the mosaic level pipeline is to use an association. Instructions on how to create an input association an be found at :ref:`asn-from-list`. diff --git a/docs/roman/pipeline_naming_conventions.rst b/docs/roman/pipeline_naming_conventions.rst index 27aba40bf..0540a7af1 100644 --- a/docs/roman/pipeline_naming_conventions.rst +++ b/docs/roman/pipeline_naming_conventions.rst @@ -47,10 +47,10 @@ name. If the input file name already has a known suffix, that suffix will be replaced. For example: :: - $ strun romancal.dq_init.DQInitStep r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun romancal.dq_init.DQInitStep r0008308002010007027_0019_wfi01_uncal.asdf produces an output file named -``r0008308002010007027_06311_0019_WFI01_dq_init.asdf``. +``r0008308002010007027_0019_wfi01_dq_init.asdf``. .. _pipeline_step_suffix_definitions: diff --git a/docs/roman/pipeline_parameters.rst b/docs/roman/pipeline_parameters.rst index 3531e4fca..b2ff0a694 100644 --- a/docs/roman/pipeline_parameters.rst +++ b/docs/roman/pipeline_parameters.rst @@ -17,7 +17,7 @@ To set a parameter, simply specify it on the command line. For example, to have ``strun`` command would be as follows: :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf --save_calibrated_ramp=true + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf --save_calibrated_ramp=true To specify parameter values for an individual step when running a pipeline use the syntax ``--steps..=value``. @@ -25,7 +25,7 @@ For example, to override the default selection of a dark current reference file from CRDS when running a pipeline: :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf --steps.dark_current.override_dark='my_dark.asdf' Universal Parameters @@ -46,7 +46,7 @@ have all output from ``roman_elp``, including any saved intermediate steps, appear in the sub-directory ``calibrated``, use :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf --output_dir=calibrated ``output_dir`` can be specified at the step level, overriding what was @@ -54,7 +54,7 @@ specified for the pipeline. From the example above, to change the name and location of the ``dark_current`` step, use the following :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf --output_dir=calibrated --steps.dark_current.output_file='dark_sub.asdf' --steps.dark_current.output_dir='dark_calibrated' diff --git a/docs/roman/pipeline_run.rst b/docs/roman/pipeline_run.rst index 62f266756..1fb1edba0 100644 --- a/docs/roman/pipeline_run.rst +++ b/docs/roman/pipeline_run.rst @@ -21,7 +21,7 @@ For example, the exposure level pipeline is implemented by the class run this pipeline is: :: - $ strun romancal.pipeline.ExposurePipeline r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun romancal.pipeline.ExposurePipeline r0008308002010007027_0019_wfi01_uncal.asdf Pipeline classes also have a **pipeline name**, or **alias**, that can be used @@ -30,7 +30,7 @@ instead of the full class specification. For example, can be run as :: - $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf The mosaic level pipeline can be run in a similar manner and is implemented using the class :ref:`romancal.pipeline.MosaicPipeline `. @@ -71,9 +71,9 @@ From the Python Prompt ------------------------------ You can execute a pipeline or a step from within python by using the -``call`` method of the class. +``run`` method. -The ``call`` method creates a new instance of the class and runs the pipeline or +The ``run`` method creates a new instance of the class and runs the pipeline or step. Optional parameter settings can be specified by via keyword arguments or supplying a parameter file. Some examples are shown below. @@ -105,7 +105,5 @@ For the mosaic level pipeline and steps, result = skymatch('r0000101001001001001_asn.json') -For more information, see :ref:`Execute via call()` - For details on the different ways to run a pipeline step, see the :ref:`Configuring a Step` page. diff --git a/docs/roman/pipeline_static_preview.rst b/docs/roman/pipeline_static_preview.rst index a572a3a61..42feaeb89 100644 --- a/docs/roman/pipeline_static_preview.rst +++ b/docs/roman/pipeline_static_preview.rst @@ -67,16 +67,16 @@ Examples .. code-block:: shell - roman_static_preview preview r0000501001001001001_01101_0001_WFI01_cal.asdf r0000501001001001001_01101_0001_WFI01_cal.png 400 400 + roman_static_preview preview r0000501001001001001_0001_wfi01_cal.asdf r0000501001001001001_0001_wfi01_cal.png 400 400 -.. image:: ../images/r0000501001001001001_01101_0001_WFI01_cal.png +.. image:: ../images/r0000501001001001001_0001_wfi01_cal.png :alt: preview of Roman imagery, with compass rose showing orientation .. code-block:: shell - roman_static_preview thumbnail r0000501001001001001_01101_0001_WFI01_cal.asdf r0000501001001001001_01101_0001_WFI01_cal_thumb.png + roman_static_preview thumbnail r0000501001001001001_0001_wfi01_cal.asdf r0000501001001001001_0001_wfi01_cal_thumb.png -.. image:: ../images/r0000501001001001001_01101_0001_WFI01_cal_thumb.png +.. image:: ../images/r0000501001001001001_0001_wfi01_cal_thumb.png :alt: thumbnail of Roman imagery using ``stpreview`` directly diff --git a/docs/roman/pipeline_steps.rst b/docs/roman/pipeline_steps.rst index 93e1e7185..f10d94f05 100644 --- a/docs/roman/pipeline_steps.rst +++ b/docs/roman/pipeline_steps.rst @@ -8,7 +8,6 @@ Pipeline Steps refpix/index.rst linearity/index.rst dark_current/index.rst - jump/index.rst ramp_fitting/index.rst assign_wcs/index.rst flatfield/index.rst diff --git a/docs/roman/references_general/references_general.rst b/docs/roman/references_general/references_general.rst index 44593fded..db828eaf2 100644 --- a/docs/roman/references_general/references_general.rst +++ b/docs/roman/references_general/references_general.rst @@ -51,10 +51,6 @@ documentation on each reference file. +---------------------------------------------+--------------------------------------------------+ | :ref:`flatfield ` | :ref:`FLAT ` | +---------------------------------------------+--------------------------------------------------+ -| :ref:`jump_detection ` | :ref:`GAIN ` | -+ +--------------------------------------------------+ -| | :ref:`READNOISE ` | -+---------------------------------------------+--------------------------------------------------+ | :ref:`linearity ` | :ref:`LINEARITY ` | +---------------------------------------------+--------------------------------------------------+ | :ref:`photom ` | :ref:`PHOTOM ` | @@ -76,20 +72,12 @@ documentation on each reference file. +--------------------------------------------------+---------------------------------------------+ | :ref:`FLAT ` | :ref:`flatfield ` | +--------------------------------------------------+---------------------------------------------+ -| :ref:`GAIN ` | :ref:`jump_detection ` | -+ +---------------------------------------------+ -| | :ref:`ramp_fitting ` | -+--------------------------------------------------+---------------------------------------------+ | :ref:`LINEARITY ` | :ref:`linearity ` | +--------------------------------------------------+---------------------------------------------+ | :ref:`MASK ` | :ref:`dq_init ` | +--------------------------------------------------+---------------------------------------------+ | :ref:`PHOTOM ` | :ref:`photom ` | +--------------------------------------------------+---------------------------------------------+ -| :ref:`READNOISE ` | :ref:`jump_detection ` | -+ +---------------------------------------------+ -| | :ref:`ramp_fitting ` | -+--------------------------------------------------+---------------------------------------------+ | :ref:`SATURATION ` | :ref:`saturation ` | +--------------------------------------------------+---------------------------------------------+ diff --git a/docs/roman/stpipe/call_via_call.rst b/docs/roman/stpipe/call_via_call.rst deleted file mode 100644 index 9d2697492..000000000 --- a/docs/roman/stpipe/call_via_call.rst +++ /dev/null @@ -1,72 +0,0 @@ -.. _call_examples: - -Executing a pipeline or pipeline step via call() -================================================ - -The ``call`` method will create an instance and run a pipeline or pipeline step -in a single call. - -:: - - from romancal.pipeline import ExposurePipeline - result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') - - from romancal.linearity import LinearityStep - result = LinearityStep.call('r0000101001001001001_01101_0001_WFI01_dqinit.asdf') - - -To set custom parameter values when using the ``call`` method, set the -parameters in the pipeline or parameter file and then supply the file using the -``config_file`` keyword: :: - - # Calling a pipeline - result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf', config_file='exp_pars.asdf')) - - # Calling a step - result = LinearityStep.call('r0000101001001001001_01101_0001_WFI01_dqinit.asdf', config_file = 'linearity_pars.asdf') - - -When running a pipeline, parameter values can also be supplied in the call to ``call`` itself by using a nested dictionary of step and -parameter names: - -:: - - result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf', config_file='exp_pars.asdf', steps={"jump":{"rejection_threshold": 200}}) - -When running a single step with ``call``, parameter values can be supplied more simply: - -:: - - result = JumpStep.call("r0000101001001001001_01101_0001_WFI01_dqinit.asdf", rejection_threshold=200) - -Running steps and pipelines with ``call`` also allows for the specification of a logging -configuration file using the keyword ``logcfg``: - -:: - - result = ExposurePipeline.call("r0000101001001001001_01101_0001_WFI01_dqinit.asdf", - config_file="exp_pars.asdf", - logcfg="my-logging-config.cfg") - - -Where are the results? ----------------------- - -A fundamental difference between running steps and pipelines in Python as -opposed to from the command line using ``strun`` is whether files are created or -not. When using ``strun``, results are automatically saved to files because that -is the only way to access the results. - -However, when running within a Python interpreter or script, the presumption is -that results will be used within the script. As such, results are not -automatically saved to files. It is left to the user to decide when to save. - -If one wishes for results to be saved by a particular ``call``, use the -parameter ``save_results=True``:: - - result = JumpStep.call("r0000101001001001001_01101_0001_WFI01_dqinit.asdf", - rejection_threshold=200, save_results=True) - -If one wishes to specify a different file name, rather than a system-generated -one, set :ref:`output_file` and/or -:ref:`output_dir`. diff --git a/docs/roman/stpipe/call_via_run.rst b/docs/roman/stpipe/call_via_run.rst index 2f83b12d6..ad17bbcb9 100644 --- a/docs/roman/stpipe/call_via_run.rst +++ b/docs/roman/stpipe/call_via_run.rst @@ -15,27 +15,26 @@ file, the parameter file will be ignored. # Manually set any desired non-default parameter values pipe.assign_wcs.skip = True - pipe.jump.rejection_threshold = 5 pipe.ramp_fit.override_gain = 'my_gain_file.asdf' pipe.save_result = True pipe.output_dir = '/my/data/pipeline_outputs' # Run the pipeline - result = pipe.run('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + result = pipe.run('r0000101001001001001_0001_wfi01_uncal.asdf') To run a single step: :: - from romancal.jump import JumpStep + from romancal.ramp_fitting import RampFitStep # Instantiate the step - step = JumpStep() + step = RampFitStep() # Set parameter values - step.rejection_threshold = 5 + step.override_gain = 'my_gain_file.asdf' step.save_results = True step.output_dir = '/my/data/jump_data' # Execute using the run method - result = step.run('r0000101001001001001_01101_0001_WFI01_linearity.asdf') + result = step.run('r0000101001001001001_0001_wfi01_linearity.asdf') diff --git a/docs/roman/stpipe/config_asdf.rst b/docs/roman/stpipe/config_asdf.rst index 5a526e9ba..02b5cbd61 100644 --- a/docs/roman/stpipe/config_asdf.rst +++ b/docs/roman/stpipe/config_asdf.rst @@ -16,13 +16,13 @@ class, Step class, or already existing .asdf or .cfg file, and run that step using the ``--save-parameters`` option. For example, to get the parameters for the ``ExposurePipeline`` pipeline, do the following: :: - $ strun --save-parameters=exp_pars.asdf roman_elp r0000101001001001001_01101_0001_WFI01_uncal.asdf + $ strun --save-parameters=exp_pars.asdf roman_elp r0000101001001001001_0001_wfi01_uncal.asdf Once created and modified as necessary, the file can now be used by ``strun`` to run the step/pipeline with the desired parameters: :: - $ strun exp_pars.asdf r0000101001001001001_01101_0001_WFI01_uncal.asdf + $ strun exp_pars.asdf r0000101001001001001_0001_wfi01_uncal.asdf The remaining sections will describe the file format and contents. @@ -30,26 +30,29 @@ File Contents ------------- To describe the contents of an ASDF file, the configuration for the step -``roman_elp`` will be used as the example: +``roman_elp`` will be used as the example which only shows some of the +entries for clarity: .. code-block:: - + #ASDF 1.0.0 #ASDF_STANDARD 1.5.0 %YAML 1.1 %TAG ! tag:stsci.edu:asdf/ --- !core/asdf-1.1.0 asdf_library: !core/software-1.0.0 {author: The ASDF Developers, homepage: 'http://github.com/asdf-format/asdf', - name: asdf, version: 2.13.0} + name: asdf, version: 3.3.0} history: - extensions: - - !core/extension_metadata-1.0.0 - extension_class: asdf.extension.BuiltinExtension - software: !core/software-1.0.0 {name: asdf, version: 2.13.0} + extensions: + - !core/extension_metadata-1.0.0 + extension_class: asdf.extension._manifest.ManifestExtension + extension_uri: asdf://asdf-format.org/core/extensions/core-1.5.0 + manifest_software: !core/software-1.0.0 {name: asdf_standard, version: 1.1.1} + software: !core/software-1.0.0 {name: asdf, version: 3.3.0} class: romancal.pipeline.exposure_pipeline.ExposurePipeline meta: author: - date: '2022-09-15T13:59:54' + date: '2024-11-20T19:41:22' description: Parameters for calibration step romancal.pipeline.exposure_pipeline.ExposurePipeline instrument: {name: } origin: @@ -59,7 +62,7 @@ To describe the contents of an ASDF file, the configuration for the step useafter: name: ExposurePipeline parameters: - input_dir: '' + input_dir: rsim output_dir: null output_ext: .asdf output_file: null @@ -71,33 +74,37 @@ To describe the contents of an ASDF file, the configuration for the step save_results: true search_output_file: true skip: false - suffix: null + suffix: cal steps: - - class: romancal.jump.jump_step.JumpStep - name: jump + - class: romancal.source_catalog.source_catalog_step.SourceCatalogStep + name: source_catalog parameters: - flag_4_neighbors: true - four_group_rejection_threshold: 190.0 - input_dir: '' - max_jump_to_flag_neighbors: 1000.0 - maximum_cores: none - min_jump_to_flag_neighbors: 10.0 - output_dir: null - output_ext: .asdf - output_file: null - output_use_index: true - output_use_model: false - post_hooks: [] - pre_hooks: [] - rejection_threshold: 180.0 - save_results: false - search_output_file: true - skip: false - suffix: null - three_group_rejection_threshold: 185.0 - ... - -Required Components + aperture_ee1: 30 + aperture_ee2: 50 + aperture_ee3: 70 + bkg_boxsize: 1000 + ci1_star_threshold: 2.0 + ci2_star_threshold: 1.8 + deblend: false + fit_psf: true + input_dir: rsim + kernel_fwhm: 2.0 + npixels: 25 + output_dir: null + output_ext: .asdf + output_file: null + output_use_index: true + output_use_model: false + post_hooks: [] + pre_hooks: [] + save_results: false + search_output_file: true + skip: false + snr_threshold: 3.0 + suffix: cat + ... + +REQUIRED Components ~~~~~~~~~~~~~~~~~~~ Preamble @@ -146,11 +153,8 @@ For example, the ``parameters`` block above could also have been formatted as: .. code-block:: - parameters: {flag_4_neighbors: true, four_group_rejection_threshold: 190.0, - max_jump_to_flag_neighbors: 1000.0, maximum_cores: none, - min_jump_to_flag_neighbors: 10.0, output_dir: null, output_ext: .asdf, - output_file: null, output_use_index: true, output_use_model: false, - rejection_threshold: 180.0, three_group_rejection_threshold: 185.0} + parameters:{ aperture_ee1: 30, aperture_ee2: 50, aperture_ee3: 70, + bkg_boxsize: 1000, ci1_star_threshold: 2.0, ci2_star_threshold: 1.8} Optional Components ~~~~~~~~~~~~~~~~~~~ @@ -173,14 +177,14 @@ step is executed. Remember that parameter values can come from numerous sources. Refer to :ref:`Parameter Precedence` for a full listing of how parameters can be set. -From the ``JumpStep`` example, if all that needed to change is the -``rejection_threshold`` parameter with a setting of ``80.0``, +From the ``SourceCatalogStep`` example, if all that needed to change is the +``aperture_ee1`` parameter with a setting of ``30.0``, the ``parameters`` block need only contain the following: .. code-block:: parameters: - rejection_threshold: 80.0 + aperture_ee1: 30.0 Pipeline Configuration @@ -190,8 +194,8 @@ Pipelines are essentially steps that refer to sub-steps. As in the original cfg format, parameters for sub-steps can also be specified. All sub-step parameters appear in a key called `steps`. Sub-step parameters are specified by using the sub-step name as the key, then underneath and indented, the parameters to change -for that sub-step. For example, to define the ``rejection_threshold`` of the -``JumpStep`` step in a ``ExposurePipeline`` parameter file, the parameter +for that sub-step. For example, to define the ``aperture_ee1`` of the +``SourceCatalogStep`` step in a ``ExposurePipeline`` parameter file, the parameter block would look as follows: .. code-block:: @@ -199,9 +203,9 @@ block would look as follows: class: romancal.pipeline.exposure_pipeline.ExposurePipeline parameters: {} steps: - - class: romancal.jump.jump_step.JumpStep + - class: romancal.source_catalog.source_catalog_step.SourceCatalogStep parameters: - rejection_threshold: 80.0 + aperture_ee1: 30 As with step parameter files, not all sub-steps need to be specified. If left unspecified, the sub-steps will be run with their default parameter sets. For @@ -236,14 +240,14 @@ There are a number of ways to create an ASDF parameter file. From the command line utility ``strun``, the option ``--save-parameters`` can be used. Within a Python script, the method ``Step.export_config(filename: str)`` can be -used. For example, to create a parameter file for ``JumpStep``, use the +used. For example, to create a parameter file for ``SourceCatalogStep``, use the following: .. doctest-skip:: - >>> from romancal.jump import JumpStep - >>> step = JumpStep() - >>> step.export_config('jump_step.asdf') + >>> from romancal..source_catalog import SourceCatalogStep + >>> step = SourceCatalogStep() + >>> step.export_config('source_catalog_step.asdf') diff --git a/docs/roman/stpipe/index.rst b/docs/roman/stpipe/index.rst index afba87a90..951cf1ac5 100644 --- a/docs/roman/stpipe/index.rst +++ b/docs/roman/stpipe/index.rst @@ -13,7 +13,6 @@ For Users user_pipeline.rst user_logging.rst config_asdf.rst - call_via_call.rst call_via_run.rst parameter_files.rst diff --git a/docs/roman/stpipe/parameter_files.rst b/docs/roman/stpipe/parameter_files.rst index 50cc5bcfe..359de9622 100644 --- a/docs/roman/stpipe/parameter_files.rst +++ b/docs/roman/stpipe/parameter_files.rst @@ -20,14 +20,14 @@ For example, to save the parameters used for a run of the ``ExposurePipeline`` pipeline, use: :: -$ strun roman_elp r0000101001001001001_01101_0001_WFI01_uncal.asdf --save-parameters my_exppars.asdf +$ strun roman_elp r0000101001001001001_0001_wfi01_uncal.asdf --save-parameters my_exppars.asdf Once saved, the file can be edited, removing parameters that should be left at their default values, and setting the remaining parameters to the desired values. Once modified, the new parameter file can be used: :: -$ strun my_exppars2.asdf r0000101001001001001_01101_0001_WFI01_uncal.asdf +$ strun my_exppars2.asdf r0000101001001001001_0001_wfi01_uncal.asdf Note that the parameter values will reflect whatever was set on the command-line, or through a specified local parameter file. In short, the diff --git a/docs/roman/stpipe/user_pipeline.rst b/docs/roman/stpipe/user_pipeline.rst index fddd785c1..c6da9e3a9 100644 --- a/docs/roman/stpipe/user_pipeline.rst +++ b/docs/roman/stpipe/user_pipeline.rst @@ -105,23 +105,25 @@ parameter file that contains: .. code-block:: yaml - - class: romancal.jump.jump_step.JumpStep - name: jump - parameters: - flag_4_neighbors: true - four_group_rejection_threshold: 190.0 - input_dir: '' - max_jump_to_flag_neighbors: 1000.0 - maximum_cores: none - min_jump_to_flag_neighbors: 10.0 + - class: romancal.source_catalog.source_catalog_step.SourceCatalogStep + name: SourceCatalogStep + parameters: + aperture_ee1: 30 + aperture_ee2: 50 + aperture_ee3: 70 + bkg_boxsize: 1000 + ci1_star_threshold: 2.0 + ci2_star_threshold: 1.8 + deblend: false + fit_psf: true is equivalent to: .. code-block:: yaml steps: - - class: romancal.jump.jump_step.JumpStep - name: jump + - class: romancal.source_catalog.source_catalog_step.SourceCatalogStep + name: SourceCatalogStep parameters: config_file = myconfig.asdf @@ -129,11 +131,11 @@ with the file ``myconfig.asdf.`` in the same directory: .. code-block:: yaml - class: romancal.jump.jump_step.JumpStep - name: jump + class: romancal.source_catalog.source_catalog_step.SourceCatalogStep + name: SourceCatalogStep parameters: - flag_4_neighbors: true - four_group_rejection_threshold: 190.0 + aperture_ee1: 30 + aperture_ee2: 50 If both a ``config_file`` and additional parameters are specified, the ``config_file`` is loaded, and then the local parameters override @@ -154,31 +156,29 @@ equivalent of the parameter file above: .. code-block:: python - from stpipe.pipeline import Image2Pipeline + from romancal.pipeline import ExposurePipeline steps = { - 'jump':{'rejection_threshold': 180., - 'three_group_rejection_threshold': 190., - 'four_group_rejection_threshold':195. - } + 'source_catalog':{ 'aperture_ee1': 3, 'aperture_ee2': 5, 'aperture_ee3': 7} + } pipe = ExposurePipeline(steps=steps) Running a Pipeline ================== -From the commandline --------------------- +From the command line +--------------------- -The same ``strun`` script used to run Steps from the commandline can +The same ``strun`` script used to run Steps from the command line can also run Pipelines. The only wrinkle is that any parameters overridden from the -commandline use dot notation to specify the parameter name. For -example, to override the ``rejection_threshold`` value on the ``jump`` +command line use dot notation to specify the parameter name. For +example, to override the ``aperture_ee1`` value on the ``source_catalog`` step in the example above, one can do:: - > strun romancal.pipeline.ExposurePipeline --steps.jump.rejection_threshold=180. + > strun romancal.pipeline.ExposurePipeline --steps.source_catalog.aperture_ee1=3 From Python ----------- diff --git a/docs/roman/stpipe/user_step.rst b/docs/roman/stpipe/user_step.rst index b50878603..707524510 100644 --- a/docs/roman/stpipe/user_step.rst +++ b/docs/roman/stpipe/user_step.rst @@ -172,51 +172,16 @@ for ``strun``. The call signature is:: For example, given the following command-line:: - $ strun romancal.pipeline.ExposurePipeline r0000101001001001001_01101_0001_WFI01_uncal.asdf \ - --steps.jump.override_gain=roman_wfi_gain_0033.asdf + $ strun romancal.pipeline.ExposurePipeline r0000101001001001001_0001_wfi01_uncal.asdf \ + --steps.ramp_fit.override_gain=roman_wfi_gain_0033.asdf the equivalent `from_cmdline` call would be:: from romancal.pipeline import ExposurePipeline - ExposurePipeline.from_cmdline([' r0000101001001001001_01101_0001_WFI01_uncal.asdf', - 'steps.jump.override_gain', 'roman_wfi_gain_0033.asdf']) + ExposurePipeline.from_cmdline([' r0000101001001001001_0001_wfi01_uncal.asdf', + 'steps.ramp_fit.override_gain', 'roman_wfi_gain_0033.asdf']) -call() -`````` - -Class method `Step.call` is the slightly more programmatic, and preferred, -method of executing a step or pipeline. When using ``call``, one gets the full -configuration initialization that -one gets with the ``strun`` command or ``Step.from_cmdline`` method. The call -signature is:: - - Step.call(input, logcfg=None, **parameters) - -The positional argument ``input`` is the data to be operated on, usually a -string representing a file path or a :ref:`DataModel`. The optional -keyword argument ``config_file`` is used to specify a local parameter file. The -optional keyword argument ``logcfg`` is used to specify a logging configuration file. -Finally, the remaining optional keyword arguments are the parameters that the -particular step accepts. The method returns the result of the step. A basic -example is:: - - from romancal.jump import JumpStep - output = JumpStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') - -makes a new instance of `JumpStep` and executes using the specified exposure -file. `JumpStep` has a parameter ``rejection_threshold``. To use a different -value than the default, the statement would be:: - - output = JumpStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf', - rejection_threshold=42.0) - -If one wishes to use a :ref:`parameter file`, specify the path -to it using the ``config_file`` argument:: - - output = JumpStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf', - config_file='my_jumpstep_config.asdf') - run() ````` @@ -233,11 +198,11 @@ example is:: `input` in this case can be a asdf file containing the appropriate data, or the output of a previously run step/pipeline, which is an instance of a particular :ref:`datamodel`. -Unlike the ``call`` class method, there is no parameter initialization that -occurs, either by a local parameter file or from a CRDS-retrieved parameter -reference file. Parameters can be set individually on the instance, as is shown -above. Parameters can also be specified as keyword arguments when instantiating -the step. The previous example could be re-written as:: +There is no parameter initialization that occurs, either by a local +parameter file or from a CRDS-retrieved parameter reference +file. Parameters can be set individually on the instance, as is shown +above. Parameters can also be specified as keyword arguments when +instantiating the step. The previous example could be re-written as:: from romancal.flatfield import FlatFieldStep diff --git a/romancal/lib/suffix.py b/romancal/lib/suffix.py index 0eef48b74..94f9d65e2 100644 --- a/romancal/lib/suffix.py +++ b/romancal/lib/suffix.py @@ -51,11 +51,12 @@ "dqinit", "assign_wcs", "linearity", - "jump", "rampfit", "saturation", "dark_current", "darkcurrent", + "sourcedetection", + "sourcedetectionstep", "outlier_detection", "skymatch", "refpix", @@ -73,7 +74,6 @@ _calculated_suffixes = { "saturationstep", "darkcurrentstep", - "jumpstep", "rampfit", "dark_current", "assignwcsstep", @@ -89,7 +89,8 @@ "dq_init", "linearitystep", "dark_current", - "jump", + "sourcedetection", + "sourcedetectionstep", "tweakregstep", "outlierdetectionstep", "skymatchstep", diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index 55d5acac2..1cd3891a8 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -127,12 +127,55 @@ def process(self, input): result.meta.cal_step.photom = "SKIPPED" result.meta.cal_step.source_catalog = "SKIPPED" - if any_saturated: - # the input association contains a fully saturated model - # where source_catalog can't be run which means we - # also can't run tweakreg. - result.meta.cal_step.tweakreg = "SKIPPED" - lib.shelve(result, model_index) + # Test for fully saturated data + if is_fully_saturated(result): + # Return fully saturated image file (stopping pipeline) + log.info("All pixels are saturated. Returning a zeroed-out image.") + + # if is_fully_saturated(result): + # Set all subsequent steps to skipped + for step_str in [ + "assign_wcs", + "flat_field", + "photom", + "source_detection", + "dark", + "refpix", + "linearity", + "ramp_fit", + "tweakreg", + ]: + result.meta.cal_step[step_str] = "SKIPPED" + + results.append(result) + return results + + result = self.refpix.run(result) + result = self.linearity.run(result) + result = self.dark_current.run(result) + result = self.rampfit.run(result) + result = self.assign_wcs.run(result) + + if result.meta.exposure.type == "WFI_IMAGE": + result = self.flatfield.run(result) + result = self.photom.run(result) + result = self.source_catalog.run(result) + tweakreg_input.append(result) + log.info( + f"Number of models to tweakreg: {len(tweakreg_input), n_members}" + ) + else: + log.info("Flat Field step is being SKIPPED") + log.info("Photom step is being SKIPPED") + log.info("Source Detection step is being SKIPPED") + log.info("Tweakreg step is being SKIPPED") + result.meta.cal_step.flat_field = "SKIPPED" + result.meta.cal_step.photom = "SKIPPED" + result.meta.cal_step.source_detection = "SKIPPED" + result.meta.cal_step.tweakreg = "SKIPPED" + + self.output_use_model = True + results.append(result) # Now that all the exposures are collated, run tweakreg # Note: this does not cover the case where the asn mixes imaging and spectral diff --git a/romancal/step.py b/romancal/step.py index 34692c33e..58df99475 100644 --- a/romancal/step.py +++ b/romancal/step.py @@ -8,7 +8,6 @@ from .dq_init.dq_init_step import DQInitStep from .flatfield.flat_field_step import FlatFieldStep from .flux import FluxStep -from .jump.jump_step import JumpStep from .linearity.linearity_step import LinearityStep from .multiband_catalog.multiband_catalog_step import MultibandCatalogStep from .outlier_detection.outlier_detection_step import OutlierDetectionStep @@ -27,7 +26,6 @@ "DQInitStep", "FlatFieldStep", "FluxStep", - "JumpStep", "LinearityStep", "OutlierDetectionStep", "PhotomStep", diff --git a/romancal/stpipe/integration.py b/romancal/stpipe/integration.py index e90f9921b..1251bda7c 100644 --- a/romancal/stpipe/integration.py +++ b/romancal/stpipe/integration.py @@ -27,7 +27,6 @@ def get_steps(): ("romancal.step.DQInitStep", "dq_init", False), ("romancal.step.FlatFieldStep", "flat_field", False), ("romancal.step.FluxStep", "flux", False), - ("romancal.step.JumpStep", "jump", False), ("romancal.step.LinearityStep", "linearity", False), ("romancal.step.PhotomStep", "photom", False), ("romancal.step.RampFitStep", "ramp_fit", False), From 211dd88728b5f5243460cf5f31760df2d5c98512 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Thu, 21 Nov 2024 15:59:56 -0500 Subject: [PATCH 08/19] rcal-954 Update git ignore for emacs backup files --- docs/roman/datamodels/#datamodels_asdf.rst# | 313 -------------------- docs/roman/datamodels/.#datamodels_asdf.rst | 1 - 2 files changed, 314 deletions(-) delete mode 100644 docs/roman/datamodels/#datamodels_asdf.rst# delete mode 120000 docs/roman/datamodels/.#datamodels_asdf.rst diff --git a/docs/roman/datamodels/#datamodels_asdf.rst# b/docs/roman/datamodels/#datamodels_asdf.rst# deleted file mode 100644 index 03cc2a481..000000000 --- a/docs/roman/datamodels/#datamodels_asdf.rst# +++ /dev/null @@ -1,313 +0,0 @@ -.. _datamodels_asdf: - - -Working with Roman datamodels and ASDF files -============================================ - -Please refer to `Roman Documentation `_ -for more details about `roman_datamodels`. - -This section assumes that you are familiar with the ASDF standard format. -If that's not the case, a good starting point would be to go over the `ASDF Documentation `_. - -If you have installed the roman calibration pipeline you should also have access -to the standalone tool asdfinfo which allows access to `ASDF `_ (and roman) files -from the terminal prompt,:: - - asdftool info r0000101001001001001_0001_wfi16_cal.asdf - root (AsdfObject) - ├─asdf_library (Software) - │ ├─author (str): The ASDF Developers - ... - -also useful is:: - - asdftool help - usage: asdftool [-h] [--verbose] {help,explode,implode,extract,defragment,diff,edit,remove-hdu,info,extensions,tags,to_yaml} ... - - Commandline utilities for managing ASDF files. - - optional arguments: - -h, --help show this help message and exit - --verbose, -v Increase verbosity - - subcommands: - - {help,explode,implode,extract,defragment,diff,edit,remove-hdu,info,extensions,tags,to_yaml} - help Display usage information - explode Explode a ASDF file. - implode Implode a ASDF file. - extract Extract ASDF extensions in ASDF-in-FITS files into pure ASDF files - defragment Defragment an ASDF file.. - diff Report differences between two ASDF files - remove-hdu Remove ASDF extension from ASDF-in-FITS file - info Print a rendering of an ASDF tree. - extensions Show information about installed extensions - tags List currently available tags - to_yaml Convert as ASDF file to pure YAML. - - -which gives a list of possible actions one of the more useful can be:: - - asdftool edit file.asdf - -Which will open the file in an editor you have set via the EDITOR environment variable. -A more complete description of the options can be found in the -`asdftool `_ -documentation. - -To access the files via a python session, - -.. code-block:: python - - import roman_datamodels as rdm - import asdf - with rdm.open('r0000101001001001001_0001_wfi16_cal.asdf') as model: - - - with asdf.open('r0000101001001001001_0001_wfi16_cal.asdf', memmap=False) as model: - - -Once the files are loaded you can access various attributes. Below is a table -showing how to access various properties using the roman_datamodels and the -asdf.open methods, - -+--------------------------------------+---------------------------------------------------------------+ -| Roman Datamodels | ASDF | -+--------------------------------------+---------------------------------------------------------------+ -| .. code-block:: python | .. code-block:: python | -| | | -| rdm_a.meta | asdf_a.tree['roman']['meta'] | -| rdm_a.meta.aperture | asdf_a.tree['roman']['meta']['aperture'] | -| rdm_a.meta.aperture.position_angle | asdf_a.tree['roman']['meta']['aperture']['position_angle'] | -| 120 | 120 | -+--------------------------------------+---------------------------------------------------------------+ - -You can also update or modify the metadata in Roman datamodels - -.. code-block:: python - - rdm_a.meta.aperture.position_angle = 120.21 - rdm_a.meta.aperture.position_angle - 120.21 - -The ASDF equivalent is - -.. code-block:: python - - asdf_a.tree['roman']['meta']['aperture']['position_angle'] = 120.05 - asdf_a.tree['roman']['meta']['aperture']['position_angle'] - 120.05 - -.. HINT:: - - If you trigger an error, - "ValueError: assignment destination is read-only" - make sure the asdf file was opened with memmap=False, or - with mode='rw' - -You can also access and modify the data arrays - -.. code-block:: python - :caption: Roman Datamodels - - rdm_a.data - - - rdm_a.data[10,11] - 0.0 - - rdm_a.data[10,11] = 122.1 - rdm_a.data[10,11] - 122.1 - -or by modifying the ASDF tree, - -.. code-block:: python - :caption: ASDF - - asdf_a.tree['roman']['data'] - - - asdf_a.tree['roman']['data'][10,11] - 0.0 - - asdf_a.tree['roman']['data'][10,11] = 3.14159 - asdf_a.tree['roman']['data'][10,11] - 3.14159 - -Using the info method ---------------------- - -You can examine a roman data model using the info and search methods provided -from the asdf package. The info function will print a representation of the -asdf tree. - -.. code:: python - - >>> from roman_datamodels import datamodels as rdm # doctest: +SKIP - >>> d_uncal = rdm.open('r0000101001001001001 _0001_wfi01_uncal.asdf') # doctest: +SKIP - >>> d_uncal.info() # doctest: +SKIP - root (AsdfObject) - ├─asdf_library (Software) - │ ├─author (str): The ASDF Developers - │ ├─homepage (str): http://github.com/asdf-format/asdf - │ ├─name (str): asdf - │ └─version (str): 2.8.1 - ├─history (dict) - │ └─extensions (list) - │ ├─[0] (ExtensionMetadata) ... - │ ├─[1] (ExtensionMetadata) ... - │ └─[2] (ExtensionMetadata) ... - └─roman (WfiScienceRaw) - ├─meta (dict) - │ ├─aperture (Aperture) ... - │ ├─cal_step (L2CalStep) ... - │ ├─calibration_software_version (str): 0.4.3.dev89+gca5771d - │ ├─coordinates (Coordinates) ... - │ ├─crds_context_used (str): roman_0020.pmap - │ ├─crds_software_version (str): 11.5.0 - │ ├─ephemeris (Ephemeris) ... - │ ├─exposure (Exposure) ... - │ └─17 not shown - └─data (NDArrayType): shape=(8, 4096, 4096), dtype=uint16 - Some nodes not shown. - -The info command also gives you control over the number of lines displayed -by passing the argument ``max_rows``. As an integer, ``max_rows`` -will be interpreted as an overall limit on the number of displayed lines. -If ``max_rows`` is a tuple, then each member limits lines per node at the -depth corresponding to its tuple index. -For example, to show all top-level nodes and 5 of each's children: - -.. code:: python - - >>> d_uncal.info(max_rows=(None,5)) # doctest: +SKIP - root (AsdfObject) - ├─asdf_library (Software) - │ ├─author (str): The ASDF Developers - │ ├─homepage (str): http://github.com/asdf-format/asdf - │ ├─name (str): asdf - │ └─version (str): 2.8.1 - ├─history (dict) - │ └─extensions (list) ... - └─roman (WfiScienceRaw) - ├─meta (dict) ... - └─data (NDArrayType): shape=(8, 4096, 4096), dtype=uint16 - Some nodes not shown. - -Or you can use the asdf.info method to view the contents of the tree - -.. code:: python - - >> asdf.info(d_uncal) - -Will print the same information as the above `d_uncal.info` command but also -gives you enhanced capabilities. For instance you can display the first three -lines for each of the meta entries, - -.. code:: python - - >>> asdf.info(d_uncal.meta, max_rows=(None, 3)) # doctest: +SKIP - root (DNode) - ├─aperture (Aperture) - │ ├─name (str): WFI_CEN - │ └─position_angle (int): 120 - ├─cal_step (L2CalStep) - │ ├─assign_wcs (str): INCOMPLETE - │ ├─flat_field (str): INCOMPLETE - │ └─6 not shown - ├─calibration_software_version (str): 0.4.3.dev89+gca5771d - ├─coordinates (Coordinates) - │ └─reference_frame (str): ICRS - ├─crds_context_used (str): roman_0020.pmap - ├─crds_software_version (str): 11.5.0 - ├─ephemeris (Ephemeris) - │ ├─earth_angle (float): 3.3161255787892263 - │ ├─moon_angle (float): 3.3196162372932148 - │ └─10 not shown - ... - -or you can concentrate on a given attribute. To list all the attributes -in `cal_step` without listing the values, - -.. code:: python - - >>> asdf.info(d_uncal.meta.cal_step,max_rows=(None, 3),show_values=False) # doctest: +SKIP - root (L2CalStep) - ├─assign_wcs (str) - ├─flat_field (str) - ├─dark (str) - ├─dq_init (str) - ├─jump (str) - ├─linearity (str) - ├─ramp_fit (str) - └─saturation (str) - -More information on the info method can be found in the ASDF documentation at -`rendering the ASDF trees. `_ - -Using the search method ------------------------ - -You can also use the search method to find attributes, - -.. code:: python - - >>> d_uncal.search('cal_step') # doctest: +SKIP - root (AsdfObject) - └─roman (WfiScienceRaw) - └─meta (dict) - └─cal_step (L2CalStep) - -or a a general search for all attributes with cal in the name - -.. code:: python - - >>> d_uncal.search('cal') # doctest: +SKIP - root (AsdfObject) - └─roman (WfiScienceRaw) - └─meta (dict) - ├─cal_step (L2CalStep) - ├─calibration_software_version (str): 0.4.3.dev89+gca5771d - ├─instrument (WfiMode) - │ └─optical_element (str): F158 - └─velocity_aberration (VelocityAberration) - └─scale_factor (float): 0.9999723133902021 - -This will do a regular expression search for `cal` in the attribute name. More -information on using regular expressions in the search method can be found -in the ASDF documentation linked below. - -To search only within the meta tree, - -.. code:: python - - >>> d_uncal.search('cal_')['roman']['meta'] # doctest: +SKIP - meta (dict) - ├─cal_step (L2CalStep) - └─instrument (WfiMode) - └─optical_element (str): F158 - -You can also use the search method to find attributes by type in the asdf tree. -For instance, you can find all integers, floats, or booleans by using the type -keyword, - -.. code:: python - - >>> d_uncal.search(type=bool) # doctest: +SKIP - root (AsdfObject) - └─roman (WfiScienceRaw) - └─meta (dict) - ├─exposure (Exposure) - │ └─data_problem (bool): False - └─visit (Visit) - ├─internal_target (bool): False - └─target_of_opportunity (bool): False - - >>> d_uncal.search(type=bool, value=True) # doctest: +SKIP - No results found. - -More information and options for the search method can be found in the -ASDF documentation -`here. `_ diff --git a/docs/roman/datamodels/.#datamodels_asdf.rst b/docs/roman/datamodels/.#datamodels_asdf.rst deleted file mode 120000 index 8cb62536e..000000000 --- a/docs/roman/datamodels/.#datamodels_asdf.rst +++ /dev/null @@ -1 +0,0 @@ -ddavis@rugila.fios-router.home.stsci.edu.12833 \ No newline at end of file From 1a1fc66e78aeb538b0fe4399cc0dca6d71525737 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 10 Dec 2024 09:18:12 -0500 Subject: [PATCH 09/19] rcal-954 Resolve merge conflicts --- romancal/regtest/test_wfi_image_pipeline.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/romancal/regtest/test_wfi_image_pipeline.py b/romancal/regtest/test_wfi_image_pipeline.py index 0fd85a297..73d6e9a7b 100644 --- a/romancal/regtest/test_wfi_image_pipeline.py +++ b/romancal/regtest/test_wfi_image_pipeline.py @@ -265,7 +265,6 @@ def test_all_saturated_against_truth(run_all_saturated, ignore_asdf_paths): ) assert diff.identical, diff.report() - @pytest.mark.parametrize( "step_name, status", [ @@ -301,6 +300,16 @@ def test_all_saturated_zeroed(all_saturated_model, array_name): For an all saturated input the output model should contain 0s for data and err arrays. """ np.testing.assert_array_equal(getattr(all_saturated_model, array_name), 0) + # Ensure step completion is as expected + with rdm.open(rtdata.output) as model: + assert model.meta.cal_step.dq_init == "COMPLETE" + assert model.meta.cal_step.saturation == "COMPLETE" + assert model.meta.cal_step.linearity == "SKIPPED" + assert model.meta.cal_step.dark == "SKIPPED" + assert model.meta.cal_step.ramp_fit == "SKIPPED" + assert model.meta.cal_step.assign_wcs == "SKIPPED" + assert model.meta.cal_step.flat_field == "SKIPPED" + assert model.meta.cal_step.photom == "SKIPPED" def test_pipeline_suffix(rtdata, ignore_asdf_paths): From 419d74cdd10179f7455285726f41a615f852c5aa Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Fri, 22 Nov 2024 15:21:47 -0500 Subject: [PATCH 10/19] rcal-954 updates for run --- docs/roman/pipeline_run.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/roman/pipeline_run.rst b/docs/roman/pipeline_run.rst index 1fb1edba0..e6c486b02 100644 --- a/docs/roman/pipeline_run.rst +++ b/docs/roman/pipeline_run.rst @@ -70,11 +70,8 @@ Exit Status From the Python Prompt ------------------------------ -You can execute a pipeline or a step from within python by using the -``run`` method. - -The ``run`` method creates a new instance of the class and runs the pipeline or -step. Optional parameter settings can be specified by via keyword arguments or +You can execute a pipeline or a step from within python. +Optional parameter settings can be specified by via keyword arguments or supplying a parameter file. Some examples are shown below. For the exposure pipeline and steps, From 080b5deefc17f072b777295afde7c48e3a96366e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 18:59:49 +0000 Subject: [PATCH 11/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/roman/stpipe/config_asdf.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/roman/stpipe/config_asdf.rst b/docs/roman/stpipe/config_asdf.rst index 02b5cbd61..ea96396e0 100644 --- a/docs/roman/stpipe/config_asdf.rst +++ b/docs/roman/stpipe/config_asdf.rst @@ -31,10 +31,10 @@ File Contents To describe the contents of an ASDF file, the configuration for the step ``roman_elp`` will be used as the example which only shows some of the -entries for clarity: +entries for clarity: .. code-block:: - + #ASDF 1.0.0 #ASDF_STANDARD 1.5.0 %YAML 1.1 From 97835713e72a88cdbabc65251ecebf5e3ea9b5b6 Mon Sep 17 00:00:00 2001 From: D Davis <49163225+ddavis-stsci@users.noreply.github.com> Date: Wed, 4 Dec 2024 08:14:32 -0500 Subject: [PATCH 12/19] Update 1534.general.rst --- changes/1534.general.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/1534.general.rst b/changes/1534.general.rst index 1a8fc3458..e336983a2 100644 --- a/changes/1534.general.rst +++ b/changes/1534.general.rst @@ -1 +1 @@ -Remove the jump code for the depricated jump detection for even ramps and update the documentation +Remove the jump code for the deprecated jump detection for even ramps and update the documentation From ec87f6ee996a614f80dfbc4e3c4b993010c7e0b6 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 10 Dec 2024 13:22:09 -0500 Subject: [PATCH 13/19] rcal-954 restore --- romancal/pipeline/exposure_pipeline.py | 55 +++----------------------- 1 file changed, 6 insertions(+), 49 deletions(-) diff --git a/romancal/pipeline/exposure_pipeline.py b/romancal/pipeline/exposure_pipeline.py index 1cd3891a8..55d5acac2 100644 --- a/romancal/pipeline/exposure_pipeline.py +++ b/romancal/pipeline/exposure_pipeline.py @@ -127,55 +127,12 @@ def process(self, input): result.meta.cal_step.photom = "SKIPPED" result.meta.cal_step.source_catalog = "SKIPPED" - # Test for fully saturated data - if is_fully_saturated(result): - # Return fully saturated image file (stopping pipeline) - log.info("All pixels are saturated. Returning a zeroed-out image.") - - # if is_fully_saturated(result): - # Set all subsequent steps to skipped - for step_str in [ - "assign_wcs", - "flat_field", - "photom", - "source_detection", - "dark", - "refpix", - "linearity", - "ramp_fit", - "tweakreg", - ]: - result.meta.cal_step[step_str] = "SKIPPED" - - results.append(result) - return results - - result = self.refpix.run(result) - result = self.linearity.run(result) - result = self.dark_current.run(result) - result = self.rampfit.run(result) - result = self.assign_wcs.run(result) - - if result.meta.exposure.type == "WFI_IMAGE": - result = self.flatfield.run(result) - result = self.photom.run(result) - result = self.source_catalog.run(result) - tweakreg_input.append(result) - log.info( - f"Number of models to tweakreg: {len(tweakreg_input), n_members}" - ) - else: - log.info("Flat Field step is being SKIPPED") - log.info("Photom step is being SKIPPED") - log.info("Source Detection step is being SKIPPED") - log.info("Tweakreg step is being SKIPPED") - result.meta.cal_step.flat_field = "SKIPPED" - result.meta.cal_step.photom = "SKIPPED" - result.meta.cal_step.source_detection = "SKIPPED" - result.meta.cal_step.tweakreg = "SKIPPED" - - self.output_use_model = True - results.append(result) + if any_saturated: + # the input association contains a fully saturated model + # where source_catalog can't be run which means we + # also can't run tweakreg. + result.meta.cal_step.tweakreg = "SKIPPED" + lib.shelve(result, model_index) # Now that all the exposures are collated, run tweakreg # Note: this does not cover the case where the asn mixes imaging and spectral From 70b28f79a7fee0ae09d63a7d86aa81c9b440d163 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 10 Dec 2024 14:15:43 -0500 Subject: [PATCH 14/19] rcal-954 More review updates --- docs/roman/pipeline_run.rst | 10 ++- docs/roman/stpipe/call_via_call.rst | 72 +++++++++++++++++++++ docs/roman/stpipe/index.rst | 1 + romancal/lib/suffix.py | 4 -- romancal/regtest/test_wfi_image_pipeline.py | 11 ---- 5 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 docs/roman/stpipe/call_via_call.rst diff --git a/docs/roman/pipeline_run.rst b/docs/roman/pipeline_run.rst index e6c486b02..bea5e76e5 100644 --- a/docs/roman/pipeline_run.rst +++ b/docs/roman/pipeline_run.rst @@ -80,11 +80,11 @@ For the exposure pipeline and steps, from romancal.pipeline import ExposurePipeline elp = ExposurePipeline() - result = elp('r0000101001001001001_0001_wfi01_uncal.asdf') + result = elp.call('r0000101001001001001_0001_wfi01_uncal.asdf') from romancal.linearity import LinearityStep linearity = LinearityStep() - result = linearity('r0000101001001001001_0001_wfi01_uncal.asdf') + result = linearity.call('r0000101001001001001_0001_wfi01_uncal.asdf') One difference between the mosaic level pipeline and the exposure level pipeline is that the mosaic level pipeline is generally designed to run on multiple overlapping exposures. To achieve @@ -94,12 +94,10 @@ For the mosaic level pipeline and steps, :: from romancal.pipeline import MosaicPipeline - mosp = MosaicPipeline() - result = mosp('r0000101001001001001_asn.json') + mosp = MosaicPipeline('r0000101001001001001_asn.json') from romancal.skymatch import SkyMatchStep - skymatch = SkyMatchStep() - result = skymatch('r0000101001001001001_asn.json') + skymatch = SkyMatchStep.call('r0000101001001001001_asn.json') For details on the different ways to run a pipeline step, see diff --git a/docs/roman/stpipe/call_via_call.rst b/docs/roman/stpipe/call_via_call.rst new file mode 100644 index 000000000..9d2697492 --- /dev/null +++ b/docs/roman/stpipe/call_via_call.rst @@ -0,0 +1,72 @@ +.. _call_examples: + +Executing a pipeline or pipeline step via call() +================================================ + +The ``call`` method will create an instance and run a pipeline or pipeline step +in a single call. + +:: + + from romancal.pipeline import ExposurePipeline + result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + + from romancal.linearity import LinearityStep + result = LinearityStep.call('r0000101001001001001_01101_0001_WFI01_dqinit.asdf') + + +To set custom parameter values when using the ``call`` method, set the +parameters in the pipeline or parameter file and then supply the file using the +``config_file`` keyword: :: + + # Calling a pipeline + result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf', config_file='exp_pars.asdf')) + + # Calling a step + result = LinearityStep.call('r0000101001001001001_01101_0001_WFI01_dqinit.asdf', config_file = 'linearity_pars.asdf') + + +When running a pipeline, parameter values can also be supplied in the call to ``call`` itself by using a nested dictionary of step and +parameter names: + +:: + + result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf', config_file='exp_pars.asdf', steps={"jump":{"rejection_threshold": 200}}) + +When running a single step with ``call``, parameter values can be supplied more simply: + +:: + + result = JumpStep.call("r0000101001001001001_01101_0001_WFI01_dqinit.asdf", rejection_threshold=200) + +Running steps and pipelines with ``call`` also allows for the specification of a logging +configuration file using the keyword ``logcfg``: + +:: + + result = ExposurePipeline.call("r0000101001001001001_01101_0001_WFI01_dqinit.asdf", + config_file="exp_pars.asdf", + logcfg="my-logging-config.cfg") + + +Where are the results? +---------------------- + +A fundamental difference between running steps and pipelines in Python as +opposed to from the command line using ``strun`` is whether files are created or +not. When using ``strun``, results are automatically saved to files because that +is the only way to access the results. + +However, when running within a Python interpreter or script, the presumption is +that results will be used within the script. As such, results are not +automatically saved to files. It is left to the user to decide when to save. + +If one wishes for results to be saved by a particular ``call``, use the +parameter ``save_results=True``:: + + result = JumpStep.call("r0000101001001001001_01101_0001_WFI01_dqinit.asdf", + rejection_threshold=200, save_results=True) + +If one wishes to specify a different file name, rather than a system-generated +one, set :ref:`output_file` and/or +:ref:`output_dir`. diff --git a/docs/roman/stpipe/index.rst b/docs/roman/stpipe/index.rst index 951cf1ac5..afba87a90 100644 --- a/docs/roman/stpipe/index.rst +++ b/docs/roman/stpipe/index.rst @@ -13,6 +13,7 @@ For Users user_pipeline.rst user_logging.rst config_asdf.rst + call_via_call.rst call_via_run.rst parameter_files.rst diff --git a/romancal/lib/suffix.py b/romancal/lib/suffix.py index 94f9d65e2..9f8e9728d 100644 --- a/romancal/lib/suffix.py +++ b/romancal/lib/suffix.py @@ -55,8 +55,6 @@ "saturation", "dark_current", "darkcurrent", - "sourcedetection", - "sourcedetectionstep", "outlier_detection", "skymatch", "refpix", @@ -89,8 +87,6 @@ "dq_init", "linearitystep", "dark_current", - "sourcedetection", - "sourcedetectionstep", "tweakregstep", "outlierdetectionstep", "skymatchstep", diff --git a/romancal/regtest/test_wfi_image_pipeline.py b/romancal/regtest/test_wfi_image_pipeline.py index 73d6e9a7b..18c1def7a 100644 --- a/romancal/regtest/test_wfi_image_pipeline.py +++ b/romancal/regtest/test_wfi_image_pipeline.py @@ -300,17 +300,6 @@ def test_all_saturated_zeroed(all_saturated_model, array_name): For an all saturated input the output model should contain 0s for data and err arrays. """ np.testing.assert_array_equal(getattr(all_saturated_model, array_name), 0) - # Ensure step completion is as expected - with rdm.open(rtdata.output) as model: - assert model.meta.cal_step.dq_init == "COMPLETE" - assert model.meta.cal_step.saturation == "COMPLETE" - assert model.meta.cal_step.linearity == "SKIPPED" - assert model.meta.cal_step.dark == "SKIPPED" - assert model.meta.cal_step.ramp_fit == "SKIPPED" - assert model.meta.cal_step.assign_wcs == "SKIPPED" - assert model.meta.cal_step.flat_field == "SKIPPED" - assert model.meta.cal_step.photom == "SKIPPED" - def test_pipeline_suffix(rtdata, ignore_asdf_paths): """ From 490ec059a3b502a045f96698cbd405677ceefd8f Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 10 Dec 2024 14:58:53 -0500 Subject: [PATCH 15/19] rcal-954 user requests --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5312b3528..cdfff2a0d 100644 --- a/.gitignore +++ b/.gitignore @@ -57,7 +57,9 @@ distribute-*.tar.gz .tox .*.sw[op] *~ -#* + +# emacs save files +\#* .#* .project .pydevproject From b4ec081e87a01aeb9af2b8d6d5215c10e200269b Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 10 Dec 2024 15:15:59 -0500 Subject: [PATCH 16/19] rcal-954 user requests --- .gitignore | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index cdfff2a0d..c32d780f2 100644 --- a/.gitignore +++ b/.gitignore @@ -52,15 +52,15 @@ develop-eggs .installed.cfg distribute-*.tar.gz +# emacs save files +\#* +.#* + # Other .cache .tox .*.sw[op] *~ - -# emacs save files -\#* -.#* .project .pydevproject .settings From 8843126e9d0afcdc86857a7b2d27fce758f07585 Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 10 Dec 2024 15:37:03 -0500 Subject: [PATCH 17/19] rcal-954 user requests --- docs/roman/pipeline_run.rst | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/roman/pipeline_run.rst b/docs/roman/pipeline_run.rst index bea5e76e5..2697fdb74 100644 --- a/docs/roman/pipeline_run.rst +++ b/docs/roman/pipeline_run.rst @@ -21,7 +21,7 @@ For example, the exposure level pipeline is implemented by the class run this pipeline is: :: - $ strun romancal.pipeline.ExposurePipeline r0008308002010007027_0019_wfi01_uncal.asdf + $ strun romancal.pipeline.ExposurePipeline r0008308002010007027_06311_0019_WFI01_uncal.asdf Pipeline classes also have a **pipeline name**, or **alias**, that can be used @@ -30,7 +30,7 @@ instead of the full class specification. For example, can be run as :: - $ strun roman_elp r0008308002010007027_0019_wfi01_uncal.asdf + $ strun roman_elp r0008308002010007027_06311_0019_WFI01_uncal.asdf The mosaic level pipeline can be run in a similar manner and is implemented using the class :ref:`romancal.pipeline.MosaicPipeline `. @@ -70,8 +70,11 @@ Exit Status From the Python Prompt ------------------------------ -You can execute a pipeline or a step from within python. -Optional parameter settings can be specified by via keyword arguments or +You can execute a pipeline or a step from within python by using the +``call`` method of the class. + +The ``call`` method creates a new instance of the class and runs the pipeline or +step. Optional parameter settings can be specified by via keyword arguments or supplying a parameter file. Some examples are shown below. For the exposure pipeline and steps, @@ -79,12 +82,10 @@ For the exposure pipeline and steps, :: from romancal.pipeline import ExposurePipeline - elp = ExposurePipeline() - result = elp.call('r0000101001001001001_0001_wfi01_uncal.asdf') + result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') from romancal.linearity import LinearityStep - linearity = LinearityStep() - result = linearity.call('r0000101001001001001_0001_wfi01_uncal.asdf') + result = LinearityStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') One difference between the mosaic level pipeline and the exposure level pipeline is that the mosaic level pipeline is generally designed to run on multiple overlapping exposures. To achieve @@ -94,11 +95,13 @@ For the mosaic level pipeline and steps, :: from romancal.pipeline import MosaicPipeline - mosp = MosaicPipeline('r0000101001001001001_asn.json') + result = ExposurePipeline.call('r0000101001001001001_asn.json') from romancal.skymatch import SkyMatchStep - skymatch = SkyMatchStep.call('r0000101001001001001_asn.json') + result = SkyMatchStep.call('r0000101001001001001_asn.json') + +For more information, see :ref:`Execute via call()` For details on the different ways to run a pipeline step, see the :ref:`Configuring a Step` page. From 733ae50f0ed9e274fb0deafb42f82af7f6cfbdad Mon Sep 17 00:00:00 2001 From: Dave Davis Date: Tue, 10 Dec 2024 16:28:49 -0500 Subject: [PATCH 18/19] rcal-954 user requests --- docs/roman/pipeline_run.rst | 9 ++-- .../references_general/references_general.rst | 4 ++ docs/roman/stpipe/call_via_run.rst | 3 +- docs/roman/stpipe/config_asdf.rst | 4 +- docs/roman/stpipe/user_pipeline.rst | 2 +- docs/roman/stpipe/user_step.rst | 44 ++++++++++++++++--- 6 files changed, 53 insertions(+), 13 deletions(-) diff --git a/docs/roman/pipeline_run.rst b/docs/roman/pipeline_run.rst index 2697fdb74..295164ceb 100644 --- a/docs/roman/pipeline_run.rst +++ b/docs/roman/pipeline_run.rst @@ -21,7 +21,7 @@ For example, the exposure level pipeline is implemented by the class run this pipeline is: :: - $ strun romancal.pipeline.ExposurePipeline r0008308002010007027_06311_0019_WFI01_uncal.asdf + $ strun romancal.pipeline.ExposurePipeline r0008308002010007027_0019_wfi01_uncal.asdf Pipeline classes also have a **pipeline name**, or **alias**, that can be used @@ -71,7 +71,8 @@ From the Python Prompt ------------------------------ You can execute a pipeline or a step from within python by using the -``call`` method of the class. +``call`` method of the class. This creates a new instance of the class +and runs the pipeline or step. The ``call`` method creates a new instance of the class and runs the pipeline or step. Optional parameter settings can be specified by via keyword arguments or @@ -82,10 +83,10 @@ For the exposure pipeline and steps, :: from romancal.pipeline import ExposurePipeline - result = ExposurePipeline.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + result = ExposurePipeline.call('r0000101001001001001_0001_wfi01_uncal.asdf') from romancal.linearity import LinearityStep - result = LinearityStep.call('r0000101001001001001_01101_0001_WFI01_uncal.asdf') + result = LinearityStep.call('r0000101001001001001_0001_wfi01_uncal.asdf') One difference between the mosaic level pipeline and the exposure level pipeline is that the mosaic level pipeline is generally designed to run on multiple overlapping exposures. To achieve diff --git a/docs/roman/references_general/references_general.rst b/docs/roman/references_general/references_general.rst index db828eaf2..de4261a53 100644 --- a/docs/roman/references_general/references_general.rst +++ b/docs/roman/references_general/references_general.rst @@ -51,6 +51,8 @@ documentation on each reference file. +---------------------------------------------+--------------------------------------------------+ | :ref:`flatfield ` | :ref:`FLAT ` | +---------------------------------------------+--------------------------------------------------+ +| :ref:`GAIN ` | :ref:`ramp_fitting ` | ++---------------------------------------------+--------------------------------------------------+ | :ref:`linearity ` | :ref:`LINEARITY ` | +---------------------------------------------+--------------------------------------------------+ | :ref:`photom ` | :ref:`PHOTOM ` | @@ -78,6 +80,8 @@ documentation on each reference file. +--------------------------------------------------+---------------------------------------------+ | :ref:`PHOTOM ` | :ref:`photom ` | +--------------------------------------------------+---------------------------------------------+ +| :ref:`READNOISE ` | :ref:`ramp_fitting ` | ++--------------------------------------------------+---------------------------------------------+ | :ref:`SATURATION ` | :ref:`saturation ` | +--------------------------------------------------+---------------------------------------------+ diff --git a/docs/roman/stpipe/call_via_run.rst b/docs/roman/stpipe/call_via_run.rst index ad17bbcb9..9fad623b3 100644 --- a/docs/roman/stpipe/call_via_run.rst +++ b/docs/roman/stpipe/call_via_run.rst @@ -6,7 +6,8 @@ Executing a pipeline or pipeline step directly, or via run() When calling a pipeline or step instance directly, or using the ``run`` method, you can specify individual parameter values manually. In this case, parameter files are not used. If you use ``run`` after instantiating with a parameter -file, the parameter file will be ignored. +file (as is done when using the :ref:`call` method), +the parameter file will be ignored. :: diff --git a/docs/roman/stpipe/config_asdf.rst b/docs/roman/stpipe/config_asdf.rst index ea96396e0..923d517b9 100644 --- a/docs/roman/stpipe/config_asdf.rst +++ b/docs/roman/stpipe/config_asdf.rst @@ -178,13 +178,13 @@ Remember that parameter values can come from numerous sources. Refer to :ref:`Parameter Precedence` for a full listing of how parameters can be set. From the ``SourceCatalogStep`` example, if all that needed to change is the -``aperture_ee1`` parameter with a setting of ``30.0``, +``aperture_ee1`` parameter with a setting of ``30``, the ``parameters`` block need only contain the following: .. code-block:: parameters: - aperture_ee1: 30.0 + aperture_ee1: 30 Pipeline Configuration diff --git a/docs/roman/stpipe/user_pipeline.rst b/docs/roman/stpipe/user_pipeline.rst index c6da9e3a9..5c7dde510 100644 --- a/docs/roman/stpipe/user_pipeline.rst +++ b/docs/roman/stpipe/user_pipeline.rst @@ -186,7 +186,7 @@ From Python Once the pipeline has been configured (as above) it can be executed using run. - pipe(input_data) + pipe.run(input_data) Caching details --------------- diff --git a/docs/roman/stpipe/user_step.rst b/docs/roman/stpipe/user_step.rst index 707524510..f6212251c 100644 --- a/docs/roman/stpipe/user_step.rst +++ b/docs/roman/stpipe/user_step.rst @@ -181,6 +181,40 @@ the equivalent `from_cmdline` call would be:: ExposurePipeline.from_cmdline([' r0000101001001001001_0001_wfi01_uncal.asdf', 'steps.ramp_fit.override_gain', 'roman_wfi_gain_0033.asdf']) +call() +`````` + +Class method `Step.call` is the slightly more programmatic, and preferred, +method of executing a step or pipeline. When using ``call``, one gets the full +configuration initialization that +one gets with the ``strun`` command or ``Step.from_cmdline`` method. The call +signature is:: + + Step.call(input, logcfg=None, **parameters) + +The positional argument ``input`` is the data to be operated on, usually a +string representing a file path or a :ref:`DataModel`. The optional +keyword argument ``config_file`` is used to specify a local parameter file. The +optional keyword argument ``logcfg`` is used to specify a logging configuration file. +Finally, the remaining optional keyword arguments are the parameters that the +particular step accepts. The method returns the result of the step. A basic +example is:: + + from romancal.jump import JumpStep + output = JumpStep.call('r0000101001001001001_0001_wfi01_uncal.asdf') + +makes a new instance of `JumpStep` and executes using the specified exposure +file. `JumpStep` has a parameter ``rejection_threshold``. To use a different +value than the default, the statement would be:: + + output = JumpStep.call('r0000101001001001001_0001_wfi01_uncal.asdf', + rejection_threshold=42.0) + +If one wishes to use a :ref:`parameter file`, specify the path +to it using the ``config_file`` argument:: + + output = JumpStep.call('r0000101001001001001_0001_wfi01_uncal.asdf', + config_file='my_jumpstep_config.asdf') run() ````` @@ -198,11 +232,11 @@ example is:: `input` in this case can be a asdf file containing the appropriate data, or the output of a previously run step/pipeline, which is an instance of a particular :ref:`datamodel`. -There is no parameter initialization that occurs, either by a local -parameter file or from a CRDS-retrieved parameter reference -file. Parameters can be set individually on the instance, as is shown -above. Parameters can also be specified as keyword arguments when -instantiating the step. The previous example could be re-written as:: +Unlike the ``call`` class method, there is no parameter initialization that +occurs, either by a local parameter file or from a CRDS-retrieved parameter +reference file. Parameters can be set individually on the instance, as is shown +above. Parameters can also be specified as keyword arguments when instantiating +the step. The previous example could be re-written as:: from romancal.flatfield import FlatFieldStep From 7f75c72a99b1aa08cd480dbd637719cb30611fd1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Dec 2024 12:38:25 +0000 Subject: [PATCH 19/19] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/roman/stpipe/user_step.rst | 2 +- romancal/regtest/test_wfi_image_pipeline.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/roman/stpipe/user_step.rst b/docs/roman/stpipe/user_step.rst index f6212251c..26eaf125c 100644 --- a/docs/roman/stpipe/user_step.rst +++ b/docs/roman/stpipe/user_step.rst @@ -214,7 +214,7 @@ If one wishes to use a :ref:`parameter file`, specify the path to it using the ``config_file`` argument:: output = JumpStep.call('r0000101001001001001_0001_wfi01_uncal.asdf', - config_file='my_jumpstep_config.asdf') + config_file='my_jumpstep_config.asdf') run() ````` diff --git a/romancal/regtest/test_wfi_image_pipeline.py b/romancal/regtest/test_wfi_image_pipeline.py index 18c1def7a..0fd85a297 100644 --- a/romancal/regtest/test_wfi_image_pipeline.py +++ b/romancal/regtest/test_wfi_image_pipeline.py @@ -265,6 +265,7 @@ def test_all_saturated_against_truth(run_all_saturated, ignore_asdf_paths): ) assert diff.identical, diff.report() + @pytest.mark.parametrize( "step_name, status", [ @@ -301,6 +302,7 @@ def test_all_saturated_zeroed(all_saturated_model, array_name): """ np.testing.assert_array_equal(getattr(all_saturated_model, array_name), 0) + def test_pipeline_suffix(rtdata, ignore_asdf_paths): """ Tests passing suffix to the pipeline