From 05cc7494dffa4c7db635d911aae4efa22fed7288 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Nov 2023 10:18:05 -0500 Subject: [PATCH 1/7] Copy over dqflags from romancal Current as of https://github.com/spacetelescope/romancal/commit/92785353bd27e60a113fb623d595b1a55c4d197d --- src/roman_datamodels/dqflags.py | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/roman_datamodels/dqflags.py diff --git a/src/roman_datamodels/dqflags.py b/src/roman_datamodels/dqflags.py new file mode 100644 index 00000000..78f84da9 --- /dev/null +++ b/src/roman_datamodels/dqflags.py @@ -0,0 +1,68 @@ +""" Roman Data Quality Flags + +The definitions are documented in the Roman RTD: + +[NOTE: Documentation not yet implemented. Fix this URL when completed.] + +https://roman-cal-pipeline.readthedocs.io/en/latest/roman/references_general/references_general.html#data-quality-flags + + +Implementation +------------- + +The flags are implemented as "bit flags": Each flag is assigned a bit position +in a byte, or multi-byte word, of memory. If that bit is set, the flag assigned +to that bit is interpreted as being set or active. + +The data structure that stores bit flags is just the standard Python `int`, +which provides 32 bits. Bits of an integer are most easily referred to using +the formula `2**bit_number` where `bit_number` is the 0-index bit of interest. +""" +# for this file drop E241 (multiple spaces after :, for readability) +# flake8: noqa: E241 +# Pixel-specific flags +pixel = { + "GOOD": 0, # No bits set, all is good + "DO_NOT_USE": 2**0, # Bad pixel. Do not use. + "SATURATED": 2**1, # Pixel saturated during exposure + "JUMP_DET": 2**2, # Jump detected during exposure + "DROPOUT": 2**3, # Data lost in transmission + "GW_AFFECTED_DATA": 2**4, # Data affected by the GW read window + "PERSISTENCE": 2**5, # High persistence (was RESERVED_2) + "AD_FLOOR": 2**6, # Below A/D floor (0 DN, was RESERVED_3) + "RESERVED_4": 2**7, # + "UNRELIABLE_ERROR": 2**8, # Uncertainty exceeds quoted error + "NON_SCIENCE": 2**9, # Pixel not on science portion of detector + "DEAD": 2**10, # Dead pixel + "HOT": 2**11, # Hot pixel + "WARM": 2**12, # Warm pixel + "LOW_QE": 2**13, # Low quantum efficiency + "TELEGRAPH": 2**15, # Telegraph pixel + "NONLINEAR": 2**16, # Pixel highly nonlinear + "BAD_REF_PIXEL": 2**17, # Reference pixel cannot be used + "NO_FLAT_FIELD": 2**18, # Flat field cannot be measured + "NO_GAIN_VALUE": 2**19, # Gain cannot be measured + "NO_LIN_CORR": 2**20, # Linearity correction not available + "NO_SAT_CHECK": 2**21, # Saturation check not available + "UNRELIABLE_BIAS": 2**22, # Bias variance large + "UNRELIABLE_DARK": 2**23, # Dark variance large + "UNRELIABLE_SLOPE": 2**24, # Slope variance large (i.e., noisy pixel) + "UNRELIABLE_FLAT": 2**25, # Flat variance large + "RESERVED_5": 2**26, # + "RESERVED_6": 2**27, # + "UNRELIABLE_RESET": 2**28, # Sensitive to reset anomaly + "RESERVED_7": 2**29, # + "OTHER_BAD_PIXEL": 2**30, # A catch-all flag + "REFERENCE_PIXEL": 2**31, # Pixel is a reference pixel +} + +# Group-specific flags. Once groups are combined, these flags +# are equivalent to the pixel-specific flags. +group = { + "GOOD": pixel["GOOD"], + "DO_NOT_USE": pixel["DO_NOT_USE"], + "SATURATED": pixel["SATURATED"], + "JUMP_DET": pixel["JUMP_DET"], + "DROPOUT": pixel["DROPOUT"], + "AD_FLOOR": pixel["AD_FLOOR"], +} From aba28c7729ca36a6058652911eba637b48690722 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Nov 2023 10:40:44 -0500 Subject: [PATCH 2/7] Change from dictionary to `IntEnum` This makes the flags immutable and enables `.` access. This still allows dictionary like access e.g. pixel["GOOD"] --- src/roman_datamodels/dqflags.py | 99 +++++++++++++++++---------------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/src/roman_datamodels/dqflags.py b/src/roman_datamodels/dqflags.py index 78f84da9..5410c7ff 100644 --- a/src/roman_datamodels/dqflags.py +++ b/src/roman_datamodels/dqflags.py @@ -18,51 +18,56 @@ which provides 32 bits. Bits of an integer are most easily referred to using the formula `2**bit_number` where `bit_number` is the 0-index bit of interest. """ -# for this file drop E241 (multiple spaces after :, for readability) -# flake8: noqa: E241 -# Pixel-specific flags -pixel = { - "GOOD": 0, # No bits set, all is good - "DO_NOT_USE": 2**0, # Bad pixel. Do not use. - "SATURATED": 2**1, # Pixel saturated during exposure - "JUMP_DET": 2**2, # Jump detected during exposure - "DROPOUT": 2**3, # Data lost in transmission - "GW_AFFECTED_DATA": 2**4, # Data affected by the GW read window - "PERSISTENCE": 2**5, # High persistence (was RESERVED_2) - "AD_FLOOR": 2**6, # Below A/D floor (0 DN, was RESERVED_3) - "RESERVED_4": 2**7, # - "UNRELIABLE_ERROR": 2**8, # Uncertainty exceeds quoted error - "NON_SCIENCE": 2**9, # Pixel not on science portion of detector - "DEAD": 2**10, # Dead pixel - "HOT": 2**11, # Hot pixel - "WARM": 2**12, # Warm pixel - "LOW_QE": 2**13, # Low quantum efficiency - "TELEGRAPH": 2**15, # Telegraph pixel - "NONLINEAR": 2**16, # Pixel highly nonlinear - "BAD_REF_PIXEL": 2**17, # Reference pixel cannot be used - "NO_FLAT_FIELD": 2**18, # Flat field cannot be measured - "NO_GAIN_VALUE": 2**19, # Gain cannot be measured - "NO_LIN_CORR": 2**20, # Linearity correction not available - "NO_SAT_CHECK": 2**21, # Saturation check not available - "UNRELIABLE_BIAS": 2**22, # Bias variance large - "UNRELIABLE_DARK": 2**23, # Dark variance large - "UNRELIABLE_SLOPE": 2**24, # Slope variance large (i.e., noisy pixel) - "UNRELIABLE_FLAT": 2**25, # Flat variance large - "RESERVED_5": 2**26, # - "RESERVED_6": 2**27, # - "UNRELIABLE_RESET": 2**28, # Sensitive to reset anomaly - "RESERVED_7": 2**29, # - "OTHER_BAD_PIXEL": 2**30, # A catch-all flag - "REFERENCE_PIXEL": 2**31, # Pixel is a reference pixel -} +from enum import IntEnum -# Group-specific flags. Once groups are combined, these flags -# are equivalent to the pixel-specific flags. -group = { - "GOOD": pixel["GOOD"], - "DO_NOT_USE": pixel["DO_NOT_USE"], - "SATURATED": pixel["SATURATED"], - "JUMP_DET": pixel["JUMP_DET"], - "DROPOUT": pixel["DROPOUT"], - "AD_FLOOR": pixel["AD_FLOOR"], -} + +# fmt: off +class pixel(IntEnum): + """Pixel-specific data quality flags""" + + GOOD = 0 # No bits set, all is good + DO_NOT_USE = 2**0 # Bad pixel. Do not use. + SATURATED = 2**1 # Pixel saturated during exposure + JUMP_DET = 2**2 # Jump detected during exposure + DROPOUT = 2**3 # Data lost in transmission + GW_AFFECTED_DATA = 2**4 # Data affected by the GW read window + PERSISTENCE = 2**5 # High persistence (was RESERVED_2) + AD_FLOOR = 2**6 # Below A/D floor (0 DN, was RESERVED_3) + RESERVED_4 = 2**7 # + UNRELIABLE_ERROR = 2**8 # Uncertainty exceeds quoted error + NON_SCIENCE = 2**9 # Pixel not on science portion of detector + DEAD = 2**10 # Dead pixel + HOT = 2**11 # Hot pixel + WARM = 2**12 # Warm pixel + LOW_QE = 2**13 # Low quantum efficiency + TELEGRAPH = 2**15 # Telegraph pixel + NONLINEAR = 2**16 # Pixel highly nonlinear + BAD_REF_PIXEL = 2**17 # Reference pixel cannot be used + NO_FLAT_FIELD = 2**18 # Flat field cannot be measured + NO_GAIN_VALUE = 2**19 # Gain cannot be measured + NO_LIN_CORR = 2**20 # Linearity correction not available + NO_SAT_CHECK = 2**21 # Saturation check not available + UNRELIABLE_BIAS = 2**22 # Bias variance large + UNRELIABLE_DARK = 2**23 # Dark variance large + UNRELIABLE_SLOPE = 2**24 # Slope variance large (i.e., noisy pixel) + UNRELIABLE_FLAT = 2**25 # Flat variance large + RESERVED_5 = 2**26 # + RESERVED_6 = 2**27 # + UNRELIABLE_RESET = 2**28 # Sensitive to reset anomaly + RESERVED_7 = 2**29 # + OTHER_BAD_PIXEL = 2**30 # A catch-all flag + REFERENCE_PIXEL = 2**31 # Pixel is a reference pixel + + +class group(IntEnum): + """Group-specific data quality flags + Once groups are combined, these flags are equivalent to the pixel-specific flags. + """ + GOOD = pixel.GOOD + DO_NOT_USE = pixel.DO_NOT_USE + SATURATED = pixel.SATURATED + JUMP_DET = pixel.JUMP_DET + DROPOUT = pixel.DROPOUT + AD_FLOOR = pixel.AD_FLOOR + +# fmt: on From 2f72305da2ca2e1e10e9ce8e2fbe5b29be3c0484 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Nov 2023 11:18:43 -0500 Subject: [PATCH 3/7] Add tests for dq flags. --- src/roman_datamodels/dqflags.py | 4 +- tests/test_dqflags.py | 68 +++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/test_dqflags.py diff --git a/src/roman_datamodels/dqflags.py b/src/roman_datamodels/dqflags.py index 5410c7ff..9d446995 100644 --- a/src/roman_datamodels/dqflags.py +++ b/src/roman_datamodels/dqflags.py @@ -18,10 +18,11 @@ which provides 32 bits. Bits of an integer are most easily referred to using the formula `2**bit_number` where `bit_number` is the 0-index bit of interest. """ -from enum import IntEnum +from enum import IntEnum, unique # fmt: off +@unique class pixel(IntEnum): """Pixel-specific data quality flags""" @@ -59,6 +60,7 @@ class pixel(IntEnum): REFERENCE_PIXEL = 2**31 # Pixel is a reference pixel +@unique class group(IntEnum): """Group-specific data quality flags Once groups are combined, these flags are equivalent to the pixel-specific flags. diff --git a/tests/test_dqflags.py b/tests/test_dqflags.py new file mode 100644 index 00000000..21b1a440 --- /dev/null +++ b/tests/test_dqflags.py @@ -0,0 +1,68 @@ +from math import log10 + +import pytest + +from roman_datamodels import dqflags + + +def _is_power_of_two(x): + return (log10(x) / log10(2)) % 1 == 0 + + +def test_pixel_uniqueness(): + """ + Test that there are no duplicate names in dqflags.pixel + + Note: The @unique decorator should ensure that no flag names have the + same value as another in the enum raising an error at first import + of this module. However, this test is just a sanity check on this. + """ + + assert len(dqflags.pixel) == len(dqflags.pixel.__members__) + + +@pytest.mark.parametrize("flag", dqflags.pixel) +def test_pixel_flags(flag): + """Test that each pixel flag follows the defined rules""" + # Test that the pixel flags are dqflags.pixel instances + assert isinstance(flag, dqflags.pixel) + + # Test that the pixel flags are ints + assert isinstance(flag, int) + + # Test that the pixel flags are dict accessible + assert dqflags.pixel[flag.name] is flag + + # Test that the pixel flag is a power of 2 + if flag.name == "GOOD": + # GOOD is the only non-power-of-two flag (it is 0) + assert flag.value == 0 + else: + assert _is_power_of_two(flag.value) + + +def test_group_uniqueness(): + """ + Test that there are no duplicate names in dqflags.group + + Note: The @unique decorator should ensure that no flag names have the + same value as another in the enum raising an error at first import + of this module. However, this test is just a sanity check on this. + """ + assert len(dqflags.group) == len(dqflags.group.__members__) + + +@pytest.mark.parametrize("flag", dqflags.group) +def test_group_flags(flag): + """Test that each group flag follows the defined rules""" + # Test that the group flags are dqflags.group instances + assert isinstance(flag, dqflags.group) + + # Test that the group flags are ints + assert isinstance(flag, int) + + # Test that the group flags are dict accessible + assert dqflags.group[flag.name] is flag + + # Test that each group flag matches a pixel flag of the same name + assert dqflags.pixel[flag.name] == flag From 13a179bc7cec68e9fc3b8382e49ee7845b38251b Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Thu, 16 Nov 2023 12:07:50 -0500 Subject: [PATCH 4/7] Update changes --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index bbe3b6a1..82816b54 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -8,6 +8,7 @@ - Allow assignment to or creation of node attributes using dot notation of object instances with validation. [#284] +- Move ``dqflags`` from ``romancal`` to ``roman_datamodels``. [#293] - Bugfix for ``model.meta.filename`` not matching the filename of the file on disk. [#295] From 57494d97d1e5252d212fae8c744b78bc81b71819 Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 24 Nov 2023 12:07:33 -0500 Subject: [PATCH 5/7] Add tests to confirm that we can set the flags directly and validate without issues --- tests/test_dqflags.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_dqflags.py b/tests/test_dqflags.py index 21b1a440..0b366543 100644 --- a/tests/test_dqflags.py +++ b/tests/test_dqflags.py @@ -2,7 +2,9 @@ import pytest +from roman_datamodels import datamodels as rdm from roman_datamodels import dqflags +from roman_datamodels.maker_utils import mk_datamodel def _is_power_of_two(x): @@ -41,6 +43,23 @@ def test_pixel_flags(flag): assert _is_power_of_two(flag.value) +@pytest.mark.parametrize("flag", dqflags.pixel) +def test_write_pixel_flags(tmp_path, flag): + filename = tmp_path / "test_dq.asdf" + + ramp = mk_datamodel(rdm.RampModel, shape=(2, 8, 8)) + + # Set all pixels to the flag value + ramp.pixeldq[...] = flag + + # Check that we can write the model to disk (i.e. the flag validates) + ramp.save(filename) + + # Check that we can read the model back in and the flag is preserved + with rdm.open(filename) as dm: + assert (dm.pixeldq == flag).all() + + def test_group_uniqueness(): """ Test that there are no duplicate names in dqflags.group @@ -66,3 +85,20 @@ def test_group_flags(flag): # Test that each group flag matches a pixel flag of the same name assert dqflags.pixel[flag.name] == flag + + +@pytest.mark.parametrize("flag", dqflags.group) +def test_write_group_flags(tmp_path, flag): + filename = tmp_path / "test_dq.asdf" + + ramp = mk_datamodel(rdm.RampModel, shape=(2, 8, 8)) + + # Set all pixels to the flag value + ramp.groupdq[...] = flag + + # Check that we can write the model to disk (i.e. the flag validates) + ramp.save(filename) + + # Check that we can read the model back in and the flag is preserved + with rdm.open(filename) as dm: + assert (dm.groupdq == flag).all() From c0442f78d2ae1539aab3bc18a7baf14ff6f6fafe Mon Sep 17 00:00:00 2001 From: William Jamieson Date: Fri, 24 Nov 2023 15:26:30 -0500 Subject: [PATCH 6/7] Update reserved dq flag for outlier detection These are changes made by spacetelescope/romancal#981 --- src/roman_datamodels/dqflags.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/roman_datamodels/dqflags.py b/src/roman_datamodels/dqflags.py index 9d446995..6cf8be2a 100644 --- a/src/roman_datamodels/dqflags.py +++ b/src/roman_datamodels/dqflags.py @@ -26,17 +26,17 @@ class pixel(IntEnum): """Pixel-specific data quality flags""" - GOOD = 0 # No bits set, all is good - DO_NOT_USE = 2**0 # Bad pixel. Do not use. - SATURATED = 2**1 # Pixel saturated during exposure - JUMP_DET = 2**2 # Jump detected during exposure - DROPOUT = 2**3 # Data lost in transmission - GW_AFFECTED_DATA = 2**4 # Data affected by the GW read window - PERSISTENCE = 2**5 # High persistence (was RESERVED_2) - AD_FLOOR = 2**6 # Below A/D floor (0 DN, was RESERVED_3) - RESERVED_4 = 2**7 # - UNRELIABLE_ERROR = 2**8 # Uncertainty exceeds quoted error - NON_SCIENCE = 2**9 # Pixel not on science portion of detector + GOOD = 0 # No bits set, all is good + DO_NOT_USE = 2**0 # Bad pixel. Do not use. + SATURATED = 2**1 # Pixel saturated during exposure + JUMP_DET = 2**2 # Jump detected during exposure + DROPOUT = 2**3 # Data lost in transmission + GW_AFFECTED_DATA = 2**4 # Data affected by the GW read window + PERSISTENCE = 2**5 # High persistence (was RESERVED_2) + AD_FLOOR = 2**6 # Below A/D floor (0 DN, was RESERVED_3) + OUTLIER = 2**7 # Flagged by outlier detection (was RESERVED_4) + UNRELIABLE_ERROR = 2**8 # Uncertainty exceeds quoted error + NON_SCIENCE = 2**9 # Pixel not on science portion of detector DEAD = 2**10 # Dead pixel HOT = 2**11 # Hot pixel WARM = 2**12 # Warm pixel From 74d8aa888a1b69aef3ff64c564673a31a934c038 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 17:01:06 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/roman_datamodels/dqflags.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/roman_datamodels/dqflags.py b/src/roman_datamodels/dqflags.py index 6cf8be2a..a0db85f6 100644 --- a/src/roman_datamodels/dqflags.py +++ b/src/roman_datamodels/dqflags.py @@ -18,6 +18,7 @@ which provides 32 bits. Bits of an integer are most easily referred to using the formula `2**bit_number` where `bit_number` is the 0-index bit of interest. """ + from enum import IntEnum, unique