From bdf477074e6f7061d3a5ae45edce893b29067db4 Mon Sep 17 00:00:00 2001 From: "Maxwell A. Millar-Blanchaer" Date: Mon, 9 Dec 2024 16:13:38 -0800 Subject: [PATCH 01/24] adjust data dq when dividing by flat = 0 --- corgidrp/l2a_to_l2b.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/corgidrp/l2a_to_l2b.py b/corgidrp/l2a_to_l2b.py index 78bfd980..412f8d83 100644 --- a/corgidrp/l2a_to_l2b.py +++ b/corgidrp/l2a_to_l2b.py @@ -1,5 +1,6 @@ # A file that holds the functions that transmogrify l2a data to l2b data import numpy as np +import copy import corgidrp.data as data from corgidrp.darks import build_synthesized_dark from corgidrp.detector import detector_areas @@ -128,6 +129,12 @@ def flat_division(input_dataset, flat_field): #Divide by the master flat flatdiv_cube = flatdiv_dataset.all_data / flat_field.data + #Find where the flat_field is 0 and set a DQ flag: + where_zero = np.where(flat_field.data == 0) + flatdiv_dq = copy.deepcopy(flatdiv_dataset.all_dq) + for i in range(len(flatdiv_dataset)): + flatdiv_dq[i].dq[where_zero] = np.bitwise_or(flatdiv_dataset[i].dq[where_zero], 4) + # propagate the error of the master flat frame if hasattr(flat_field, "err"): flatdiv_dataset.rescale_error(1/flat_field.data, "FlatField") @@ -138,7 +145,7 @@ def flat_division(input_dataset, flat_field): history_msg = "Flat calibration done using Flat field {0}".format(flat_field.filename) # update the output dataset with this new flat calibrated data and update the history - flatdiv_dataset.update_after_processing_step(history_msg,new_all_data=flatdiv_cube) + flatdiv_dataset.update_after_processing_step(history_msg,new_all_data=flatdiv_cube, new_all_dq = flatdiv_dq) return flatdiv_dataset From 7b707b2a0866e9c9751e1d238e2b2c43f49000a9 Mon Sep 17 00:00:00 2001 From: "Maxwell A. Millar-Blanchaer" Date: Mon, 9 Dec 2024 16:21:40 -0800 Subject: [PATCH 02/24] test the DQ setting --- tests/test_flat_div.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_flat_div.py b/tests/test_flat_div.py index 904362f6..48b6c80e 100644 --- a/tests/test_flat_div.py +++ b/tests/test_flat_div.py @@ -77,9 +77,25 @@ def test_flat_div(): print("Error estimated:",err_estimated) assert(err_flatdiv == pytest.approx(err_estimated, abs = 1e-2)) - print(flatdivided_dataset[0].ext_hdr) + # print(flatdivided_dataset[0].ext_hdr) corgidrp.track_individual_errors = old_err_tracking + + ### Check to make sure DQ gets set when flatfield zero. ### + + ## Injected some 0s. + pixel_list = [[110,120],[50,50], [100,100]] + for pixel in pixel_list: + flatfield.data[pixel[0],pixel[1]] = 0. + + ## Perform flat division + flatdivided_dataset_w_zeros = l2a_to_l2b.flat_division(simflat_dataset, flatfield) + + ## Check that all the pixels that were zeroed out have the DQ flag set to 4. + for pixel in pixel_list: + for i in range(len(simdata_filenames)): + assert np.bitwise_and(flatdivided_dataset_w_zeros.all_dq[i,pixel[0],pixel[1]],4) + if __name__ == "__main__": test_flat_div() \ No newline at end of file From 93b5b2dde706ebda08fdbffcecbd30d78817c826 Mon Sep 17 00:00:00 2001 From: "Maxwell A. Millar-Blanchaer" Date: Mon, 9 Dec 2024 16:30:05 -0800 Subject: [PATCH 03/24] fixing typo --- corgidrp/l2a_to_l2b.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corgidrp/l2a_to_l2b.py b/corgidrp/l2a_to_l2b.py index 412f8d83..e294ef1b 100644 --- a/corgidrp/l2a_to_l2b.py +++ b/corgidrp/l2a_to_l2b.py @@ -133,7 +133,7 @@ def flat_division(input_dataset, flat_field): where_zero = np.where(flat_field.data == 0) flatdiv_dq = copy.deepcopy(flatdiv_dataset.all_dq) for i in range(len(flatdiv_dataset)): - flatdiv_dq[i].dq[where_zero] = np.bitwise_or(flatdiv_dataset[i].dq[where_zero], 4) + flatdiv_dq[i][where_zero] = np.bitwise_or(flatdiv_dataset[i].dq[where_zero], 4) # propagate the error of the master flat frame if hasattr(flat_field, "err"): From 5440517b9b51f9b1574e7ae61a44136b38e351e2 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 11 Dec 2024 11:50:04 -0600 Subject: [PATCH 04/24] Revert "update README release notes" This reverts commit 5e7f70fde48d05d4e2d6f6a0b426660cc2b8af1c. --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index feaccab3..628538d0 100644 --- a/README.md +++ b/README.md @@ -209,9 +209,6 @@ Before creating a pull request, review the design Principles below. Use the Gith ## Change Log -**v1.1.1** - * Fix unit test that wasn't cleaning up environment properly - **v1.1** * Bug fix so that corgidrp classes can be pickled * New corgidrp.ops interface From 7485f43a8ff7790a104563827517e3099050b97c Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 11 Dec 2024 11:55:40 -0600 Subject: [PATCH 05/24] Revert "Revert "update README release notes"" This reverts commit 5440517b9b51f9b1574e7ae61a44136b38e351e2. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 628538d0..feaccab3 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,9 @@ Before creating a pull request, review the design Principles below. Use the Gith ## Change Log +**v1.1.1** + * Fix unit test that wasn't cleaning up environment properly + **v1.1** * Bug fix so that corgidrp classes can be pickled * New corgidrp.ops interface From e4e79ee54a7eb242965de14eaa21b0d864c44391 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 11 Dec 2024 15:00:03 -0600 Subject: [PATCH 06/24] v1.1.2 --- README.md | 3 +++ corgidrp/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index feaccab3..05b57f9c 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,9 @@ Before creating a pull request, review the design Principles below. Use the Gith ## Change Log +**v1.1.2** + * Flat field correction marks pixels divided by 0 as bad + **v1.1.1** * Fix unit test that wasn't cleaning up environment properly diff --git a/corgidrp/__init__.py b/corgidrp/__init__.py index f2908c7a..515571e4 100644 --- a/corgidrp/__init__.py +++ b/corgidrp/__init__.py @@ -3,7 +3,7 @@ import pathlib import configparser -__version__ = "1.1.1" +__version__ = "1.1.2" version = __version__ # temporary backwards compatability #### Create a configuration file for the corgidrp if it doesn't exist. diff --git a/setup.py b/setup.py index 9aa7d2fc..4a954c0b 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ def get_requires(): setup( name='corgidrp', - version="1.1.1", + version="1.1.2", description='(Roman Space Telescope) CORonaGraph Instrument Data Reduction Pipeline', #long_description="", #long_description_content_type="text/markdown", From d506fde3890c465789b65405ab4e428c9d20274a Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Wed, 11 Dec 2024 13:36:00 +0000 Subject: [PATCH 07/24] Fixed typo in error message --- corgidrp/combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corgidrp/combine.py b/corgidrp/combine.py index 6bc38b9e..4f8929c6 100644 --- a/corgidrp/combine.py +++ b/corgidrp/combine.py @@ -72,7 +72,7 @@ def combine_subexposures(input_dataset, num_frames_per_group=None, collapse="mea num_frames_per_group = len(input_dataset) if len(input_dataset) % num_frames_per_group != 0: - raise ValueError("Input dataset of lenght {0} cannot be grouped in sets of {1}".format(len(input_dataset, num_frames_per_group))) + raise ValueError("Input dataset of length {0} cannot be grouped in sets of {1}".format(len(input_dataset, num_frames_per_group))) if collapse.lower() not in ["mean", "median"]: raise ValueError("combine_subexposures can only collapse with mean or median") From ddcb95288088f4f7784d616e50946c9094f34dc5 Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Wed, 11 Dec 2024 13:38:12 +0000 Subject: [PATCH 08/24] Checked error propagation calculation --- corgidrp/combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corgidrp/combine.py b/corgidrp/combine.py index 4f8929c6..7c70f41f 100644 --- a/corgidrp/combine.py +++ b/corgidrp/combine.py @@ -32,7 +32,7 @@ def combine_images(data_subset, err_subset, dq_subset, collapse, num_frames_scal n_samples = np.sum(n_samples, axis=0) if collapse.lower() == "mean": data_collapse = np.nanmean(data_subset, axis=0) - err_collapse = np.sqrt(np.nanmean(err_subset**2, axis=0)) /np.sqrt(n_samples) # not sure if this is correct, but good enough for now + err_collapse = np.sqrt(np.nanmean(err_subset**2, axis=0)) /np.sqrt(n_samples) # correct assuming standard error propagation elif collapse.lower() == "median": data_collapse = np.nanmedian(data_subset, axis=0) err_collapse = np.sqrt(np.nanmean(err_subset**2, axis=0)) /np.sqrt(n_samples) * np.sqrt(np.pi/2) # inflate median error From 0efbdd7b00f1c099c1a00e762a5eb61afbf2408d Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Wed, 11 Dec 2024 14:04:20 +0000 Subject: [PATCH 09/24] Fixed misplaced brackets that were causing a TypeError --- corgidrp/combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corgidrp/combine.py b/corgidrp/combine.py index 7c70f41f..b1dc1cf2 100644 --- a/corgidrp/combine.py +++ b/corgidrp/combine.py @@ -72,7 +72,7 @@ def combine_subexposures(input_dataset, num_frames_per_group=None, collapse="mea num_frames_per_group = len(input_dataset) if len(input_dataset) % num_frames_per_group != 0: - raise ValueError("Input dataset of length {0} cannot be grouped in sets of {1}".format(len(input_dataset, num_frames_per_group))) + raise ValueError("Input dataset of length {0} cannot be grouped in sets of {1}".format(len(input_dataset), num_frames_per_group)) if collapse.lower() not in ["mean", "median"]: raise ValueError("combine_subexposures can only collapse with mean or median") From a35f80b530488a11c301e06beb6eb436ae755b9c Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Wed, 11 Dec 2024 15:08:08 +0000 Subject: [PATCH 10/24] Fixed typos in test_combine.py docstring --- tests/test_combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_combine.py b/tests/test_combine.py index e6ed9faf..20fc9b60 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -102,7 +102,7 @@ def test_mean_combine_subexposures_with_bad(): def test_median_combine_subexposures(): """ - Test median combine of subexposures. And tests defualt case wihere num_frames_per_group isn't specified. + Test median combine of subexposures. And tests default case where num_frames_per_group isn't specified. """ image1 = data.Image(img1, err=err1, dq=dq, pri_hdr = prhd, ext_hdr = exthd) From 0340f47bebd45e917425ee4eec80ea5ea725745a Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 11:43:33 +0000 Subject: [PATCH 11/24] Added test for non-divisible case --- tests/test_combine.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_combine.py b/tests/test_combine.py index 20fc9b60..8e7e3f9a 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -123,6 +123,25 @@ def test_median_combine_subexposures(): assert(np.all(combined_dataset[0].err == pytest.approx(np.sqrt(2*np.pi)))) assert(np.all(combined_dataset[0].dq == 0)) +def test_not_divisible(): + """ + Tests that function correctly fails when the length of the dataset is not divisible by num_frames_per_group. + """ + + image1 = data.Image(img1, err=err1, dq=dq, pri_hdr = prhd, ext_hdr = exthd) + image1.filename = "1.fits" + image2 = image1.copy() + image2.filename = "2.fits" + image3 = image1.copy() + image3.filename = "3.fits" + image4 = image1.copy() + image4.filename = "4.fits" + + dataset = data.Dataset([image1, image2, image3, image4]) + + with pytest.raises(ValueError): + combined_dataset = combine.combine_subexposures(dataset, 3) # Should fail as 4 % 3 != 0 + if __name__ == "__main__": test_mean_combine_subexposures() \ No newline at end of file From 370ad6974cbef3990144ebe0d451f86728a69c99 Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 11:48:17 +0000 Subject: [PATCH 12/24] Added test for invalid collapse type --- tests/test_combine.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_combine.py b/tests/test_combine.py index 8e7e3f9a..c64585ba 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -142,6 +142,24 @@ def test_not_divisible(): with pytest.raises(ValueError): combined_dataset = combine.combine_subexposures(dataset, 3) # Should fail as 4 % 3 != 0 +def test_invalid_collapse(): + """ + Tests that function correctly fails when collapse type is not valid. + """ + + image1 = data.Image(img1, err=err1, dq=dq, pri_hdr = prhd, ext_hdr = exthd) + image1.filename = "1.fits" + image2 = image1.copy() + image2.filename = "2.fits" + image3 = image1.copy() + image3.filename = "3.fits" + image4 = image1.copy() + image4.filename = "4.fits" + + dataset = data.Dataset([image1, image2, image3, image4]) + + with pytest.raises(ValueError): + combined_dataset = combine.combine_subexposures(dataset, collapse="invalid_option") if __name__ == "__main__": test_mean_combine_subexposures() \ No newline at end of file From 3d2b031cea89b1a172e28b66335cdc045f8e6815 Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 13:21:17 +0000 Subject: [PATCH 13/24] Added test for case where num_frames_scaling is False --- tests/test_combine.py | 45 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_combine.py b/tests/test_combine.py index c64585ba..0978dba3 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -61,6 +61,51 @@ def test_mean_combine_subexposures(): assert combined_dataset_2[0].ext_hdr['FILE2'] in ["2.fits", "1.fits", "3.fits", "4.fits"] assert combined_dataset_2[0].ext_hdr['FILE3'] in ["2.fits", "1.fits", "3.fits", "4.fits"] +def test_mean_combine_subexposures_without_scaling(): + """ + Test mean combine of subexposures for case where num_frames_scaling=False + """ + + image1 = data.Image(img1, err=err1, dq=dq, pri_hdr = prhd, ext_hdr = exthd) + image1.filename = "1.fits" + image2 = image1.copy() + image2.filename = "2.fits" + image3 = image1.copy() + image3.filename = "3.fits" + image4 = image1.copy() + image4.filename = "4.fits" + + dataset = data.Dataset([image1, image2, image3, image4]) + + combined_dataset = combine.combine_subexposures(dataset, 2, num_frames_scaling=False) + + assert(len(combined_dataset) == 2) + assert(np.all(combined_dataset[0].data == 1)) + assert(np.all(combined_dataset[0].err == pytest.approx(1/np.sqrt(2)))) + assert(np.all(combined_dataset[0].dq == 0)) + + assert combined_dataset[0].ext_hdr['DRPNFILE'] == 2 + assert combined_dataset[0].ext_hdr['FILE0'] in ["2.fits", "1.fits"] + assert combined_dataset[0].ext_hdr['FILE1'] in ["2.fits", "1.fits"] + + assert combined_dataset[1].ext_hdr['DRPNFILE'] == 2 + assert combined_dataset[1].ext_hdr['FILE0'] in ["3.fits", "4.fits"] + assert combined_dataset[1].ext_hdr['FILE1'] in ["3.fits", "4.fits"] + + # combine again + combined_dataset_2 = combine.combine_subexposures(combined_dataset, 2, num_frames_scaling=False) + + assert(len(combined_dataset_2) == 1) + assert(np.all(combined_dataset_2[0].data == 1)) + assert(np.all(combined_dataset_2[0].err == pytest.approx((1/np.sqrt(2))/np.sqrt(2)))) + assert(np.all(combined_dataset_2[0].dq == 0)) + + assert combined_dataset_2[0].ext_hdr['DRPNFILE'] == 4 + assert combined_dataset_2[0].ext_hdr['FILE0'] in ["2.fits", "1.fits", "3.fits", "4.fits"] + assert combined_dataset_2[0].ext_hdr['FILE1'] in ["2.fits", "1.fits", "3.fits", "4.fits"] + assert combined_dataset_2[0].ext_hdr['FILE2'] in ["2.fits", "1.fits", "3.fits", "4.fits"] + assert combined_dataset_2[0].ext_hdr['FILE3'] in ["2.fits", "1.fits", "3.fits", "4.fits"] + def test_mean_combine_subexposures_with_bad(): """ From 862fbd56defd2857066b818539ae0990031ca332 Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 14:10:50 +0000 Subject: [PATCH 14/24] Cleaned up test_mean_combine_subexposures_without_scaling --- tests/test_combine.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tests/test_combine.py b/tests/test_combine.py index 0978dba3..46279554 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -79,19 +79,12 @@ def test_mean_combine_subexposures_without_scaling(): combined_dataset = combine.combine_subexposures(dataset, 2, num_frames_scaling=False) + # Check that data and error values are not scaled up assert(len(combined_dataset) == 2) assert(np.all(combined_dataset[0].data == 1)) assert(np.all(combined_dataset[0].err == pytest.approx(1/np.sqrt(2)))) assert(np.all(combined_dataset[0].dq == 0)) - assert combined_dataset[0].ext_hdr['DRPNFILE'] == 2 - assert combined_dataset[0].ext_hdr['FILE0'] in ["2.fits", "1.fits"] - assert combined_dataset[0].ext_hdr['FILE1'] in ["2.fits", "1.fits"] - - assert combined_dataset[1].ext_hdr['DRPNFILE'] == 2 - assert combined_dataset[1].ext_hdr['FILE0'] in ["3.fits", "4.fits"] - assert combined_dataset[1].ext_hdr['FILE1'] in ["3.fits", "4.fits"] - # combine again combined_dataset_2 = combine.combine_subexposures(combined_dataset, 2, num_frames_scaling=False) @@ -100,13 +93,6 @@ def test_mean_combine_subexposures_without_scaling(): assert(np.all(combined_dataset_2[0].err == pytest.approx((1/np.sqrt(2))/np.sqrt(2)))) assert(np.all(combined_dataset_2[0].dq == 0)) - assert combined_dataset_2[0].ext_hdr['DRPNFILE'] == 4 - assert combined_dataset_2[0].ext_hdr['FILE0'] in ["2.fits", "1.fits", "3.fits", "4.fits"] - assert combined_dataset_2[0].ext_hdr['FILE1'] in ["2.fits", "1.fits", "3.fits", "4.fits"] - assert combined_dataset_2[0].ext_hdr['FILE2'] in ["2.fits", "1.fits", "3.fits", "4.fits"] - assert combined_dataset_2[0].ext_hdr['FILE3'] in ["2.fits", "1.fits", "3.fits", "4.fits"] - - def test_mean_combine_subexposures_with_bad(): """ Test mean combine of subexposures with bad pixels From 55541570ea5c29fd45cd45bee303c9da3d6a281c Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 14:41:01 +0000 Subject: [PATCH 15/24] Added dq assert statements --- tests/test_combine.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_combine.py b/tests/test_combine.py index 46279554..2f510e24 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -122,9 +122,11 @@ def test_mean_combine_subexposures_with_bad(): # the pixel with one bad pixel should have same value but higher error assert combined_dataset[0].data[0][0] == 2 assert combined_dataset[0].err[0][0][0] == pytest.approx(2) + assert combined_dataset[0].dq[0][0] == 0 # 0 because one of the two frames had a good value # compare against a pixel without any bad pixels assert combined_dataset[1].data[0][0] == 2 assert combined_dataset[1].err[0][0][0] == pytest.approx(np.sqrt(2)) + assert combined_dataset[1].dq[0][0] == 0 # the pixel with two bad pixels should be nan assert np.isnan(combined_dataset[0].data[0][1]) From 51c231eb016271f15959431dbdb9510f4e76c603 Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 14:55:40 +0000 Subject: [PATCH 16/24] Fixed typo --- tests/test_mean_combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mean_combine.py b/tests/test_mean_combine.py index 84dec301..da1932c1 100644 --- a/tests/test_mean_combine.py +++ b/tests/test_mean_combine.py @@ -29,7 +29,7 @@ def test_mean_im(): - """Verify method calculates mean image and erorr term.""" + """Verify method calculates mean image and error term.""" tol = 1e-13 check_combined_im = np.mean(check_ims, axis=0) From 9597d0f3b25586dcd360c89e066a14ed2ec3f5d9 Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 15:34:44 +0000 Subject: [PATCH 17/24] Fixed typos in readme.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 05b57f9c..f72a3b9f 100644 --- a/README.md +++ b/README.md @@ -95,11 +95,11 @@ def example_step(dataset, calib_data, tuneable_arg=1, another_arg="test"): Inside the function can be nearly anything you want, but the function signature and start/end of the function should follow a few rules. - * Each function should include a docstring that descibes what the function is doing, what the inputs (including units if appropriate) are and what the outputs (also with units). The dosctrings should be [goggle style docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). + * Each function should include a docstring that describes what the function is doing, what the inputs (including units if appropriate) are and what the outputs (also with units). The dosctrings should be [google style docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). * The input dataset should always be the first input * Additional arguments and keywords exist only if you need them--many relevant parameters might already by in Dataset headers. A pipeline step can only have a single argument (the input dataset) if needed. * All additional function arguments/keywords should only consist of the following types: int, float, str, or a class defined in corgidrp.Data. - * (Long explaination for the curious: The reason for this is that pipeline steps can be written out as text files. Int/float/str are easily represented succinctly by textfiles. All classes in corgidrp.Data can be created simply by passing in a filepath. Therefore, all pipeline steps have easily recordable arguments for easy reproducibility.) + * (Long explanation for the curious: The reason for this is that pipeline steps can be written out as text files. Int/float/str are easily represented succinctly by textfiles. All classes in corgidrp.Data can be created simply by passing in a filepath. Therefore, all pipeline steps have easily recordable arguments for easy reproducibility.) * The first line of the function generally should be creating a copy of the dataset (which will be the output dataset). This way, the output dataset is not the same instance as the input dataset. This will make it easier to ensure reproducibility. * The function should always end with updating the header and (typically) the data of the output dataset. The history of running this pipeline step should be recorded in the header. @@ -133,7 +133,7 @@ End-to-end testing refers to processing data as one would when we get the real d - if you need to create mock L1 data, please do it in the script as well. - See the existing tests in `tests/e2e_tests/` for how to structure this script. You should only need to write a single script. 4. Test that the script runs successfully on your local machine and produces the expected output. Debug as necessary. When appropriate, test your results against those obtained from the II&T/TVAC pipeline using the same input data. - 5. Determine how resource intensive your recipe is. There are many ways to do this, but Linux users can run `/usr/bin/time -v python your_e2e_test.py` and Mac userse can run `/usr/bin/time -l -h -p python `. Record elapsed (wall clock) time, the percent of CPU this job got (only if parallelization was used), and total memory used (labelled "Maximum resident set size"). + 5. Determine how resource intensive your recipe is. There are many ways to do this, but Linux users can run `/usr/bin/time -v python your_e2e_test.py` and Mac users can run `/usr/bin/time -l -h -p python `. Record elapsed (wall clock) time, the percent of CPU this job got (only if parallelization was used), and total memory used (labelled "Maximum resident set size"). 6. Document your recipe on the "Corgi-DRP Implementation Document" on Confluence (see the big table in Section 2.0). You should fill out an entire row with your recipe. Under addition notes, note if your recipe took significant run time (> 1 minute) and significant memory (> 1 GB). 7. PR! From 7c9e82de7c9c51c9672b8876a209086a8abb8c7d Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 15:38:06 +0000 Subject: [PATCH 18/24] Added np.copy for dq_subset in combine.py --- corgidrp/combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corgidrp/combine.py b/corgidrp/combine.py index b1dc1cf2..e48a50e6 100644 --- a/corgidrp/combine.py +++ b/corgidrp/combine.py @@ -82,7 +82,7 @@ def combine_subexposures(input_dataset, num_frames_per_group=None, collapse="mea for i in range(num_groups): data_subset = np.copy(input_dataset.all_data[num_frames_per_group*i:num_frames_per_group*(i+1)]) err_subset = np.copy(input_dataset.all_err[num_frames_per_group*i:num_frames_per_group*(i+1)]) - dq_subset = input_dataset.all_dq[num_frames_per_group*i:num_frames_per_group*(i+1)] + dq_subset = np.copy(input_dataset.all_dq[num_frames_per_group*i:num_frames_per_group*(i+1)]) data_collapse, err_collapse, dq_collapse = combine_images(data_subset, err_subset, dq_subset, collapse=collapse, num_frames_scaling=num_frames_scaling) From d8dffbaac310c6fc6a27d43aa7ee1bb65379f13b Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 16:58:02 +0000 Subject: [PATCH 19/24] Added test for median combination with multiple bad pixels --- tests/test_combine.py | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/test_combine.py b/tests/test_combine.py index 2f510e24..2de7de10 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -156,6 +156,63 @@ def test_median_combine_subexposures(): assert(np.all(combined_dataset[0].err == pytest.approx(np.sqrt(2*np.pi)))) assert(np.all(combined_dataset[0].dq == 0)) +def test_median_combine_subexposures_with_bad(): + """ + Test median combine of subexposures with bad pixels + """ + # use copies since we are going to modify their values + image1 = data.Image(np.copy(img1), err=np.copy(err1), dq=np.copy(dq), + pri_hdr = prhd, ext_hdr = exthd) + image1.filename = "1.fits" + image2 = image1.copy() + image2.filename = "2.fits" + image3 = image1.copy() + image3.filename = "3.fits" + image4 = image1.copy() + image4.filename = "4.fits" + + # (0,0) has one bad frame + image1.dq[0][0] = 1 + # (0,1) has both pixels bad + image1.dq[0][1] = 1 + image2.dq[0][1] = 1 + + dataset = data.Dataset([image1, image2, image3, image4]) + + combined_dataset = combine.combine_subexposures(dataset, 2, collapse="median") + + assert(len(combined_dataset) == 2) + # the pixel with one bad pixel should have same value but higher error. In both cases the error should be inflated by np.sqrt(np.pi/2) compared to mean error. + assert combined_dataset[0].data[0][0] == 2 + assert combined_dataset[0].err[0][0][0] == pytest.approx(2 * np.sqrt(np.pi/2)) + assert combined_dataset[0].dq[0][0] == 0 # 0 because one of the two frames had a good value + # compare against a pixel without any bad pixels + assert combined_dataset[1].data[0][0] == 2 + assert combined_dataset[1].err[0][0][0] == pytest.approx(np.sqrt(2) * np.sqrt(np.pi/2)) + assert combined_dataset[1].dq[0][0] == 0 + + # the pixel with two bad pixels should be nan + assert np.isnan(combined_dataset[0].data[0][1]) + assert np.isnan(combined_dataset[0].err[0][0][1]) + assert combined_dataset[0].dq[0][1] == 1 + + # combine again + combined_dataset_2 = combine.combine_subexposures(combined_dataset, 2, collapse="median") + + assert(len(combined_dataset_2) == 1) + assert(np.all(combined_dataset_2[0].data == 4)) + + # error for pixel with no bad pixels in original data (i.e. most pixels in data) + assert combined_dataset_2[0].err[0][5][0] == pytest.approx(np.pi) + + # error for pixel with one bad pixel in original data + assert combined_dataset_2[0].err[0][0][0] == pytest.approx(0.5 * np.pi * np.sqrt(6)) + + # error for pixel with two bad pixel in original data + assert combined_dataset_2[0].err[0][0][1] == pytest.approx(np.pi * np.sqrt(2)) + + assert(np.all(combined_dataset_2[0].dq == 0)) + def test_not_divisible(): """ Tests that function correctly fails when the length of the dataset is not divisible by num_frames_per_group. From 7d9622a14786ae25ebbbc9c4c12c85d78aa281fd Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 17:01:28 +0000 Subject: [PATCH 20/24] Updated docstrings and fixed typos --- tests/test_combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_combine.py b/tests/test_combine.py index 2de7de10..187c7726 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -158,7 +158,7 @@ def test_median_combine_subexposures(): def test_median_combine_subexposures_with_bad(): """ - Test median combine of subexposures with bad pixels + Test median combine of subexposures with bad pixels over multiple combinations """ # use copies since we are going to modify their values image1 = data.Image(np.copy(img1), err=np.copy(err1), dq=np.copy(dq), From 56d0bd869ce333d59c5e4081742971618ae2c7aa Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 17:02:42 +0000 Subject: [PATCH 21/24] Clarified test comments --- tests/test_combine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_combine.py b/tests/test_combine.py index 187c7726..0034b2f6 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -205,10 +205,10 @@ def test_median_combine_subexposures_with_bad(): # error for pixel with no bad pixels in original data (i.e. most pixels in data) assert combined_dataset_2[0].err[0][5][0] == pytest.approx(np.pi) - # error for pixel with one bad pixel in original data + # error for pixel with one bad pixel in original data (i.e. no nans after first combination) assert combined_dataset_2[0].err[0][0][0] == pytest.approx(0.5 * np.pi * np.sqrt(6)) - # error for pixel with two bad pixel in original data + # error for pixel with two bad pixels in original data (i.e. 1 nan after first combination) assert combined_dataset_2[0].err[0][0][1] == pytest.approx(np.pi * np.sqrt(2)) assert(np.all(combined_dataset_2[0].dq == 0)) From 10a1031a1d66d1c8a16f0b624cea8e4632a89206 Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 19:02:17 +0000 Subject: [PATCH 22/24] Added test to confirm combining different values and nans works as expected --- tests/test_combine.py | 53 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/tests/test_combine.py b/tests/test_combine.py index 0034b2f6..1768785c 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -213,6 +213,57 @@ def test_median_combine_subexposures_with_bad(): assert(np.all(combined_dataset_2[0].dq == 0)) +def test_combine_different_values(): + """ + Test whether the function correctly combines different values. + """ + + image1 = data.Image(img1, err=err1, dq=dq, pri_hdr = prhd, ext_hdr = exthd) + image1.filename = "1.fits" + image2 = image1.copy() + image2.filename = "2.fits" + image3 = image1.copy() + image3.filename = "3.fits" + image4 = image1.copy() + image4.filename = "4.fits" + + # (0,0) has different values in each frame. Some are bad pixels. + image1.data[0][0] = 5 + image2.data[0][0] = 6 + image3.data[0][0] = 9 + image4.data[0][0] = 19 + + image2.dq[0][0] = 1 + image4.dq[0][0] = 1 + + # (0,1) is a bad pixel in every frame + image1.dq[0][1] = 1 + image2.dq[0][1] = 1 + image3.dq[0][1] = 1 + image4.dq[0][1] = 1 + + dataset = data.Dataset([image1, image2, image3, image4]) + + combined_dataset = combine.combine_subexposures(dataset, collapse="median") + + assert(len(combined_dataset) == 1) + + # Most pixels had good values of 1 in all frames + assert combined_dataset[0].data[0][2] == 4 + assert combined_dataset[0].err[0][0][2] == pytest.approx(2*np.sqrt(np.pi/2)) + + # (0,0) has a different median value calculated ignoring nans + assert combined_dataset[0].data[0][0] == 7 * 4 # median value scaled by number of images + assert combined_dataset[0].err[0][0][0] == pytest.approx(2 * np.sqrt(np.pi)) + + # (0,1) is a nan + assert np.isnan(combined_dataset[0].data[0][1]) + assert np.isnan(combined_dataset[0].err[0][0][1]) + + # the updated bad pixel map only contains one bad pixel (i.e. the pixel for which there were no good values) + assert combined_dataset[0].dq[0][0] == 0 + assert combined_dataset[0].dq[0][1] == 1 + def test_not_divisible(): """ Tests that function correctly fails when the length of the dataset is not divisible by num_frames_per_group. @@ -252,4 +303,4 @@ def test_invalid_collapse(): combined_dataset = combine.combine_subexposures(dataset, collapse="invalid_option") if __name__ == "__main__": - test_mean_combine_subexposures() \ No newline at end of file + test_combine_different_values() \ No newline at end of file From 66cf2bd1f70d559beaf7eec91dc1857a8e663256 Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 19:11:20 +0000 Subject: [PATCH 23/24] Ensured that data were copied before modification --- tests/test_combine.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_combine.py b/tests/test_combine.py index 1768785c..306b7131 100644 --- a/tests/test_combine.py +++ b/tests/test_combine.py @@ -218,7 +218,9 @@ def test_combine_different_values(): Test whether the function correctly combines different values. """ - image1 = data.Image(img1, err=err1, dq=dq, pri_hdr = prhd, ext_hdr = exthd) + # use copies since we are going to modify their values + image1 = data.Image(np.copy(img1), err=np.copy(err1), dq=np.copy(dq), + pri_hdr = prhd, ext_hdr = exthd) image1.filename = "1.fits" image2 = image1.copy() image2.filename = "2.fits" @@ -303,4 +305,4 @@ def test_invalid_collapse(): combined_dataset = combine.combine_subexposures(dataset, collapse="invalid_option") if __name__ == "__main__": - test_combine_different_values() \ No newline at end of file + test_mean_combine_subexposures() \ No newline at end of file From 2bee6436fb3fca15fd1cff60a4d00ae91ba2bc5e Mon Sep 17 00:00:00 2001 From: bensutlieff Date: Tue, 17 Dec 2024 23:17:26 +0000 Subject: [PATCH 24/24] Fixed typo in combine.py --- corgidrp/combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/corgidrp/combine.py b/corgidrp/combine.py index e48a50e6..3235db97 100644 --- a/corgidrp/combine.py +++ b/corgidrp/combine.py @@ -91,7 +91,7 @@ def combine_subexposures(input_dataset, num_frames_per_group=None, collapse="mea pri_hdr = input_dataset[num_frames_per_group*i].pri_hdr.copy() ext_hdr = input_dataset[num_frames_per_group*i].ext_hdr.copy() err_hdr = input_dataset[num_frames_per_group*i].err_hdr.copy() - dq_hdr = input_dataset[num_frames_per_group*i].err_hdr.copy() + dq_hdr = input_dataset[num_frames_per_group*i].dq_hdr.copy() hdulist = input_dataset[num_frames_per_group*i].hdu_list.copy() new_image = data.Image(data_collapse, pri_hdr=pri_hdr, ext_hdr=ext_hdr, err=err_collapse, dq=dq_collapse, err_hdr=err_hdr, dq_hdr=dq_hdr, input_hdulist=hdulist)