tre: update infrastructure and BANDSB implementation

CHANGELOG.md

@@ -35,6 +35,7 @@ release points are not being annotated in GitHub.
 - SIDD file reading in `sarpy/consistency/sidd_consistency.py`
 - Application of adjustable parameter offsets in RIC frames during projection
 - Overflow bug in `ComplexFormatFunction` magnitude/phase -> real/imag
+- Fix BANDSB implementation to parse correctly
 
 ## [1.3.58] - 2023-08-07
 ### Added

sarpy/io/general/nitf_elements/tres/tre_elements.py

@@ -6,6 +6,7 @@
 __author__ = "Thomas McCullough"
 
 import logging
+import struct
 from collections import OrderedDict
 from typing import Union, List
 
@@ -34,6 +35,14 @@ def _parse_type(typ_string, leng, value, start):
         return byt.decode('utf-8').strip()
     elif typ_string == 'd':
         return int(byt)
+    elif typ_string == 'f':
+        if byt == b'-'*leng:
+            # hyphen-minus filled, no value present
+            return None
+        return float(byt)
+    elif typ_string == 'F754':
+        # IEEE-754 float (32 bits)
+        return struct.unpack('>f', byt)[0]
     elif typ_string == 'b':
         return byt
     else:
@@ -45,6 +54,9 @@ def _create_format(typ_string, leng):
         return '{0:' + '{0:d}'.format(leng) + 's}'
     elif typ_string == 'd':
         return '{0:0' + '{0:d}'.format(leng) + 'd}'
+    elif typ_string == 'f':
+        # TODO: need to test this
+        return '{0:0' + '{0:d}'.format(leng) + 'f}'
     else:
         return ValueError('Unknown typ_string {}'.format(typ_string))
 
@@ -74,7 +86,7 @@ def add_field(self, attribute, typ_string, leng, value):
         attribute : str
             The new field/attribute name for out object instance.
         typ_string : str
-            One of 's' (string attribute), 'd' (integer attribute), or 'b' raw/bytes attribute
+            One of 's' (string), 'd' (integer), 'f' (floating point), 'F754' (32 bit float) or 'b' raw/bytes attribute
         leng : int
             The length in bytes of the representation of this attribute
         value : bytes
@@ -152,7 +164,7 @@ def _attribute_to_bytes(self, attribute):
             return val.to_bytes()
         elif isinstance(val, bytes):
             return val
-        elif isinstance(val, (int, str)):
+        elif isinstance(val, (int, str, float)):
             return self._field_format[attribute].format(val).encode('utf-8')
         else:
             raise TypeError('Got unhandled type {}'.format(type(val)))
@@ -169,7 +181,7 @@ def to_dict(self):
         out = OrderedDict()
         for fld in self._field_ordering:
             val = getattr(self, fld)
-            if val is None or isinstance(val, (bytes, str, int)):
+            if val is None or isinstance(val, (bytes, str, int, float)):
                 out[fld] = val
             elif isinstance(val, TREElement):
                 out[fld] = val.to_dict()

sarpy/io/general/nitf_elements/tres/unclass/BANDSB.py

@@ -19,46 +19,46 @@ def __init__(self, value, EXISTENCE_MASK):
         if EXISTENCE_MASK & 0x02000000:
             self.add_field('FOCAL_LEN', 'd', 5, value)
         if EXISTENCE_MASK & 0x01000000:
-            self.add_field('CWAVE', 'd', 7, value)
+            self.add_field('CWAVE', 'f', 7, value)
         if EXISTENCE_MASK & 0x00800000:
-            self.add_field('FWHM', 'd', 7, value)
+            self.add_field('FWHM', 'f', 7, value)
         if EXISTENCE_MASK & 0x00400000:
-            self.add_field('FWHM_UNC', 'd', 7, value)
+            self.add_field('FWHM_UNC', 'f', 7, value)
         if EXISTENCE_MASK & 0x00200000:
-            self.add_field('NOM_WAVE', 'd', 7, value)
+            self.add_field('NOM_WAVE', 'f', 7, value)
         if EXISTENCE_MASK & 0x00100000:
-            self.add_field('NOM_WAVE_UNC', 'd', 7, value)
+            self.add_field('NOM_WAVE_UNC', 'f', 7, value)
         if EXISTENCE_MASK & 0x00080000:
-            self.add_field('LBOUND', 'd', 7, value)
-            self.add_field('UBOUND', 'd', 7, value)
+            self.add_field('LBOUND', 'f', 7, value)
+            self.add_field('UBOUND', 'f', 7, value)
         if EXISTENCE_MASK & 0x00040000:
-            self.add_field('SCALE_FACTOR', 'b', 4, value)
-            self.add_field('ADDITIVE_FACTOR', 'b', 4, value)
+            self.add_field('SCALE_FACTOR', 'b', 4, value) # TODO: should be ieee-754 float
+            self.add_field('ADDITIVE_FACTOR', 'b', 4, value) # TODO: should be ieee-754 float
         if EXISTENCE_MASK & 0x00020000:
             self.add_field('START_TIME', 's', 16, value)
         if EXISTENCE_MASK & 0x00010000:
-            self.add_field('INT_TIME', 'd', 6, value)
+            self.add_field('INT_TIME', 'f', 6, value)
         if EXISTENCE_MASK & 0x00008000:
-            self.add_field('CALDRK', 'd', 6, value)
-            self.add_field('CALIBRATION_SENSITIVITY', 'd', 5, value)
+            self.add_field('CALDRK', 'f', 6, value)
+            self.add_field('CALIBRATION_SENSITIVITY', 'f', 5, value)
         if EXISTENCE_MASK & 0x00004000:
-            self.add_field('ROW_GSD', 'd', 7, value)
+            self.add_field('ROW_GSD', 'f', 7, value)
             if EXISTENCE_MASK & 0x00002000:
-                self.add_field('ROW_GSD_UNC', 'd', 7, value)
+                self.add_field('ROW_GSD_UNC', 'f', 7, value)
             self.add_field('ROW_GSD_UNIT', 's', 1, value)
-            self.add_field('COL_GSD', 'd', 7, value)
+            self.add_field('COL_GSD', 'f', 7, value)
             if EXISTENCE_MASK & 0x00002000:
-                self.add_field('COL_GSD_UNC', 'd', 7, value)
+                self.add_field('COL_GSD_UNC', 'f', 7, value)
             self.add_field('COL_GSD_UNIT', 's', 1, value)
         if EXISTENCE_MASK & 0x00001000:
-            self.add_field('BKNOISE', 'd', 5, value)
-            self.add_field('SCNNOISE', 'd', 5, value)
+            self.add_field('BKNOISE', 'f', 5, value)
+            self.add_field('SCNNOISE', 'f', 5, value)
         if EXISTENCE_MASK & 0x00000800:
-            self.add_field('SPT_RESP_FUNCTION_ROW', 'd', 7, value)
+            self.add_field('SPT_RESP_FUNCTION_ROW', 'f', 7, value)
             if EXISTENCE_MASK & 0x00000400:
-                self.add_field('SPT_RESP_UNC_ROW', 'd', 7, value)
-            self.add_field('SPT_RESP_UNIT_ROW', 's', 1, value)
-            self.add_field('SPT_RESP_FUNCTION_COL', 'd', 7, value)
+                self.add_field('SPT_RESP_UNC_ROW', 'f', 7, value)
+            self.add_field('SPT_RESP_UNIT_ROW', 'f', 1, value)
+            self.add_field('SPT_RESP_FUNCTION_COL', 'f', 7, value)
             if EXISTENCE_MASK & 0x00000400:
                 self.add_field('SPT_RESP_UNC_COL', 'd', 7, value)
             self.add_field('SPT_RESP_UNIT_COL', 's', 1, value)
@@ -110,29 +110,29 @@ def __init__(self, value):
         self.add_field('COUNT', 'd', 5, value)
         self.add_field('RADIOMETRIC_QUANTITY', 's', 24, value)
         self.add_field('RADIOMETRIC_QUANTITY_UNIT', 's', 1, value)
-        self.add_field('SCALE_FACTOR', 'b', 4, value)
-        self.add_field('ADDITIVE_FACTOR', 'b', 4, value)
-        self.add_field('ROW_GSD', 'd', 7, value)
+        self.add_field('SCALE_FACTOR', 'F754', 4, value)
+        self.add_field('ADDITIVE_FACTOR', 'F754', 4, value)
+        self.add_field('ROW_GSD', 'f', 7, value)
         self.add_field('ROW_GSD_UNIT', 's', 1, value)
-        self.add_field('COL_GSD', 'd', 7, value)
+        self.add_field('COL_GSD', 'f', 7, value)
         self.add_field('COL_GSD_UNIT', 's', 1, value)
-        self.add_field('SPT_RESP_ROW', 'd', 7, value)
+        self.add_field('SPT_RESP_ROW', 'f', 7, value)
         self.add_field('SPT_RESP_UNIT_ROW', 's', 1, value)
-        self.add_field('SPT_RESP_COL', 'd', 7, value)
+        self.add_field('SPT_RESP_COL', 'f', 7, value)
         self.add_field('SPT_RESP_UNIT_COL', 's', 1, value)
         self.add_field('DATA_FLD_1', 'b', 48, value)
         self.add_field('EXISTENCE_MASK', 'b', 4, value)
-        if self.EXISTENCE_MASK & 0x80000000:
+        if int.from_bytes(self.EXISTENCE_MASK) & 0x80000000:
             self.add_field('RADIOMETRIC_ADJUSTMENT_SURFACE', 's', 24, value)
-            self.add_field('ATMOSPHERIC_ADJUSTMENT_ALTITUDE', 'b', 4, value)
-        if self.EXISTENCE_MASK & 0x40000000:
+            self.add_field('ATMOSPHERIC_ADJUSTMENT_ALTITUDE', 'F754', 4, value)
+        if int.from_bytes(self.EXISTENCE_MASK) & 0x40000000:
             self.add_field('DIAMETER', 'd', 7, value)
-        if self.EXISTENCE_MASK & 0x20000000:
+        if int.from_bytes(self.EXISTENCE_MASK) & 0x20000000:
             self.add_field('DATA_FLD_2', 'b', 32, value)
-        if self.EXISTENCE_MASK & 0x01F80000:
+        if int.from_bytes(self.EXISTENCE_MASK) & 0x01F80000:
             self.add_field('WAVE_LENGTH_UNIT', 's', 1, value)
-        self.add_loop('PARAMETERs', self.COUNT, PARAMETER, value, self.EXISTENCE_MASK)
-        if self.EXISTENCE_MASK & 0x00000001:
+        self.add_loop('PARAMETERs', self.COUNT, PARAMETER, value, int.from_bytes(self.EXISTENCE_MASK))
+        if int.from_bytes(self.EXISTENCE_MASK) & 0x00000001:
             self.add_field('NUM_AUX_B', 'd', 2, value)
             self.add_field('NUM_AUX_C', 'd', 2, value)
             self.add_loop('AUX_Bs', self.NUM_AUX_B, AUX_B, value)

tests/data/example_tre_extractor.py

@@ -4,7 +4,7 @@
 
 # From https://nsgreg.nga.mil/doc/view?i=5516
 nitf_with_tres = '07APR2005_Hyperion_331406N0442000E_SWIR172_1p2B_L1R-BIP.ntf'
-cetags = [b'MATESA']
+cetags = [b'MATESA', b'BANDSB']
 
 for cetag in cetags:
     with open(nitf_with_tres, 'r+b') as nitffile, mmap.mmap(nitffile.fileno(), 0) as mm:

tests/io/general/test_bandsb_tre.py

@@ -0,0 +1,35 @@
+from sarpy.io.general.nitf_elements.tres.registration import find_tre
+import math
+import unittest
+
+def test_bandsb(tests_path):
+    example = find_tre('BANDSB').from_bytes((tests_path / 'data/example_bandsb_tre.bin').read_bytes(), 0)
+    print(example.DATA.to_dict().keys())
+    assert example.DATA.COUNT == 172
+    assert example.DATA.RADIOMETRIC_QUANTITY == 'RADIANCE'
+    assert example.DATA.RADIOMETRIC_QUANTITY_UNIT == 'S'
+    assert example.DATA.SCALE_FACTOR == 80.0
+    assert example.DATA.ADDITIVE_FACTOR == 0.0
+    assert example.DATA.ROW_GSD == 30.49
+    assert example.DATA.ROW_GSD_UNIT == 'M'
+    assert example.DATA.COL_GSD == 29.96
+    assert example.DATA.COL_GSD_UNIT == 'M'
+    assert example.DATA.SPT_RESP_ROW == None
+    assert example.DATA.SPT_RESP_UNIT_ROW == 'M'
+    assert example.DATA.SPT_RESP_COL == None
+    assert example.DATA.SPT_RESP_UNIT_COL == 'M'
+    assert example.DATA.RADIOMETRIC_ADJUSTMENT_SURFACE == 'DETECTOR'
+    assert math.isnan(example.DATA.ATMOSPHERIC_ADJUSTMENT_ALTITUDE)
+    assert len(example.DATA.PARAMETERs) == 172
+    band0 = example.DATA.PARAMETERs[0]
+    assert band0.BAD_BAND == 0
+    assert band0.CWAVE == 0.85192
+    assert band0.FWHM == 0.01105
+    band127 = example.DATA.PARAMETERs[127]
+    assert band127.BAD_BAND == 1
+    assert band127.CWAVE == 2.13324
+    assert band127.FWHM == 0.01073
+    band171 = example.DATA.PARAMETERs[171]
+    assert band171.BAD_BAND == 0
+    assert band171.CWAVE == 2.57708
+    assert band171.FWHM == 0.01041