update for allow for dynamic size of images

python/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "trex-imager-readfile"
-version = "1.6.4"
+version = "1.6.5"
 description = "Read functions for TREx ASI raw image files"
 readme = "README.md"
 homepage = "https://github.com/ucalgary-aurora/trex-imager-readfile"

python/tests/test_suite/test_version.py

@@ -2,4 +2,4 @@
 
 
 def test_version():
-    assert __version__ == "1.6.4"
+    assert __version__ == "1.6.5"

python/trex_imager_readfile/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.6.4"
+__version__ = "1.6.5"
 
 # core functions for easy use
 from .blueline import read as read_blueline

python/trex_imager_readfile/blueline.py

@@ -6,7 +6,8 @@
 from functools import partial
 
 # globals
-__BLUELINE_IMAGE_SIZE_BYTES = 270 * 320 * 2
+__BLUELINE_EXPECTED_HEIGHT = 270
+__BLUELINE_EXPECTED_WIDTH = 320
 __BLUELINE_DT = np.dtype("uint16")
 __BLUELINE_DT = __BLUELINE_DT.newbyteorder('>')  # force big endian byte ordering
 
@@ -61,13 +62,16 @@ def __blueline_readfile_worker(file, first_frame=False, no_metadata=False, quiet
         return images, metadata_dict_list, problematic, file, error_message
 
     # read the file
+    prev_line = None
+    line = None
     while True:
         # break out depending on first_frame param
         if (first_frame is True and is_first is False):
             break
 
         # read a line
         try:
+            prev_line = line
             line = unzipped.readline()
         except Exception as e:
             if (quiet is False):
@@ -132,19 +136,26 @@ def __blueline_readfile_worker(file, first_frame=False, no_metadata=False, quiet
                     metadata_dict = {}
         elif line == b'65535\n':
             # there are 2 lines between "exposure plus read out" and the image
-            # data, the first is b'270 320\n' and the second is b'65535\n'
+            # data, the first is the image dimensions and the second is the max
+            # value
             #
+            # check the previous line to get the dimensions of the image
+            prev_line_split = prev_line.decode("ascii").strip().split()
+            image_width = int(prev_line_split[0])
+            image_height = int(prev_line_split[1])
+            bytes_to_read = image_width * image_height * 2  # 16-bit image depth
+
             # read image
             try:
                 # read the image size in bytes from the file
-                image_bytes = unzipped.read(__BLUELINE_IMAGE_SIZE_BYTES)
+                image_bytes = unzipped.read(bytes_to_read)
 
                 # format bytes into numpy array of unsigned shorts (2byte numbers, 0-65536),
                 # effectively an array of pixel values
                 image_np = np.frombuffer(image_bytes, dtype=__BLUELINE_DT)
 
-                # change 1d numpy array into 270x320 matrix with correctly located pixels
-                image_matrix = np.reshape(image_np, (270, 320, 1))
+                # change 1d numpy array into matrix with correctly located pixels
+                image_matrix = np.reshape(image_np, (image_height, image_width, 1))
             except Exception as e:
                 if (quiet is False):
                     print("Failed reading image data frame: %s" % (str(e)))
@@ -238,13 +249,17 @@ def read(file_list, workers=1, first_frame=False, no_metadata=False, quiet=False
 
     # derive number of frames to prepare for
     total_num_frames = 0
+    image_height = __BLUELINE_EXPECTED_HEIGHT
+    image_width = __BLUELINE_EXPECTED_WIDTH
     for i in range(0, len(data)):
         if (data[i][2] is True):
             continue
         total_num_frames += data[i][0].shape[2]
+        image_height = data[i][0].shape[0]
+        image_width = data[i][0].shape[1]
 
     # pre-allocate array sizes
-    images = np.empty([270, 320, total_num_frames], dtype=__BLUELINE_DT)
+    images = np.empty([image_height, image_width, total_num_frames], dtype=__BLUELINE_DT)
     metadata_dict_list = [{}] * total_num_frames
     problematic_file_list = []
 

python/trex_imager_readfile/nir.py

@@ -6,7 +6,8 @@
 from functools import partial
 
 # globals
-__NIR_IMAGE_SIZE_BYTES = 256 * 256 * 2
+__NIR_EXPECTED_HEIGHT = 256
+__NIR_EXPECTED_WIDTH = 256
 __NIR_DT = np.dtype("uint16")
 __NIR_DT = __NIR_DT.newbyteorder('>')  # force big endian byte ordering
 
@@ -131,32 +132,26 @@ def __nir_readfile_worker(file, first_frame=False, no_metadata=False, quiet=Fals
                     metadata_dict = {}
         elif line == b'65535\n':
             # there are 2 lines between "exposure plus read out" and the image
-            # data, the first is b'256 256\n' and the second is b'65535\n'
+            # data, the first is the image dimensions and the second is the max
+            # value
             #
-            # check the previous line to make sure the image size is what we want
-            if (prev_line is not None and prev_line.strip().decode("ascii") != "256 256"):
-                # invalid image size
-                problematic = True
-                metadata_dict_list = []
-                images = np.array([])
-                error_message = "invalid image size: %s" % (prev_line.decode("ascii").strip())
-                try:
-                    unzipped.close()
-                except Exception:
-                    pass
-                return images, metadata_dict_list, problematic, file, error_message
+            # check the previous line to get the dimensions of the image
+            prev_line_split = prev_line.decode("ascii").strip().split()
+            image_width = int(prev_line_split[0])
+            image_height = int(prev_line_split[1])
+            bytes_to_read = image_width * image_height * 2  # 16-bit image depth
 
             # read image
             try:
                 # read the image size in bytes from the file
-                image_bytes = unzipped.read(__NIR_IMAGE_SIZE_BYTES)
+                image_bytes = unzipped.read(bytes_to_read)
 
                 # format bytes into numpy array of unsigned shorts (2byte numbers, 0-65536),
                 # effectively an array of pixel values
                 image_np = np.frombuffer(image_bytes, dtype=__NIR_DT)
 
-                # change 1d numpy array into 256x256 matrix with correctly located pixels
-                image_matrix = np.reshape(image_np, (256, 256, 1))
+                # change 1d numpy array into matrix with correctly located pixels
+                image_matrix = np.reshape(image_np, (image_height, image_width, 1))
             except Exception as e:
                 if (quiet is False):
                     print("Failed reading image data frame: %s" % (str(e)))
@@ -250,13 +245,17 @@ def read(file_list, workers=1, first_frame=False, no_metadata=False, quiet=False
 
     # derive number of frames to prepare for
     total_num_frames = 0
+    image_height = __NIR_EXPECTED_HEIGHT
+    image_width = __NIR_EXPECTED_WIDTH
     for i in range(0, len(data)):
         if (data[i][2] is True):
             continue
         total_num_frames += data[i][0].shape[2]
+        image_height = data[i][0].shape[0]
+        image_width = data[i][0].shape[1]
 
     # pre-allocate array sizes
-    images = np.empty([256, 256, total_num_frames], dtype=__NIR_DT)
+    images = np.empty([image_height, image_width, total_num_frames], dtype=__NIR_DT)
     metadata_dict_list = [{}] * total_num_frames
     problematic_file_list = []
 

python/trex_imager_readfile/rgb.py

@@ -13,7 +13,8 @@
 from multiprocessing import Pool
 
 # static globals
-__RGB_PGM_IMAGE_SIZE_BYTES = 480 * 553 * 2
+__RGB_PGM_EXPECTED_HEIGHT = 480
+__RGB_PGM_EXPECTED_WIDTH = 553
 __RGB_PGM_DT = np.dtype("uint16")
 __RGB_PGM_DT = __RGB_PGM_DT.newbyteorder('>')  # force big endian byte ordering
 __RGB_PNG_DT = np.dtype("uint8")
@@ -263,8 +264,8 @@ def __rgb_readfile_worker_pgm(file_obj):
     device_uid = ""
     problematic = False
     error_message = ""
-    image_width = 553
-    image_height = 480
+    image_width = __RGB_PGM_EXPECTED_WIDTH
+    image_height = __RGB_PGM_EXPECTED_HEIGHT
     image_channels = 1
     image_dtype = np.dtype("uint16")
 
@@ -303,13 +304,16 @@ def __rgb_readfile_worker_pgm(file_obj):
             image_width, image_height, image_channels, image_dtype
 
     # read the file
+    prev_line = None
+    line = None
     while True:
         # break out depending on first_frame param
         if (file_obj["first_frame"] is True and is_first is False):
             break
 
         # read a line
         try:
+            prev_line = line
             line = unzipped.readline()
         except Exception as e:
             if (file_obj["quiet"] is False):
@@ -374,21 +378,28 @@ def __rgb_readfile_worker_pgm(file_obj):
                     metadata_dict = {}
         elif line == b'65535\n':
             # there are 2 lines between "exposure plus read out" and the image
-            # data, the first is b'480 553\n' and the second is b'65535\n'
+            # data, the first is the image dimensions and the second is the max
+            # value
             #
+            # check the previous line to get the dimensions of the image
+            prev_line_split = prev_line.decode("ascii").strip().split()
+            image_width = int(prev_line_split[0])
+            image_height = int(prev_line_split[1])
+            bytes_to_read = image_width * image_height * 2  # 16-bit image depth
+
             # read image
             try:
                 # read the image size in bytes from the file
-                image_bytes = unzipped.read(__RGB_PGM_IMAGE_SIZE_BYTES)
+                image_bytes = unzipped.read(bytes_to_read)
 
                 # format bytes into numpy array of unsigned shorts (2byte numbers, 0-65536),
                 # effectively an array of pixel values
                 #
                 # NOTE: this is set to a different dtype that what we return on purpose.
                 image_np = np.frombuffer(image_bytes, dtype=__RGB_PGM_DT)
 
-                # change 1d numpy array into 480x553 matrix with correctly located pixels
-                image_matrix = np.reshape(image_np, (480, 553, 1))
+                # change 1d numpy array into matrix with correctly located pixels
+                image_matrix = np.reshape(image_np, (image_height, image_width, 1))
             except Exception as e:
                 if (file_obj["quiet"] is False):
                     print("Failed reading image data frame: %s" % (str(e)))

python/trex_imager_readfile/spectrograph.py

@@ -6,7 +6,8 @@
 from functools import partial
 
 # globals
-__SPECTROGRAPH_IMAGE_SIZE_BYTES = 1024 * 256 * 2
+__SPECTROGRAPH_EXPECTED_HEIGHT = 1024
+__SPECTROGRAPH_EXPECTED_WIDTH = 256
 __SPECTROGRAPH_DT = np.dtype("uint16")
 __SPECTROGRAPH_DT = __SPECTROGRAPH_DT.newbyteorder('>')  # force big endian byte ordering
 
@@ -61,13 +62,16 @@ def __spectrograph_readfile_worker(file, first_frame=False, no_metadata=False, q
         return images, metadata_dict_list, problematic, file, error_message
 
     # read the file
+    prev_line = None
+    line = None
     while True:
         # break out depending on first_frame param
         if (first_frame is True and is_first is False):
             break
 
         # read a line
         try:
+            prev_line = line
             line = unzipped.readline()
         except Exception as e:
             if (quiet is False):
@@ -132,19 +136,26 @@ def __spectrograph_readfile_worker(file, first_frame=False, no_metadata=False, q
                     metadata_dict = {}
         elif line == b'65535\n':
             # there are 2 lines between "exposure plus read out" and the image
-            # data, the first is b'1024 256\n' and the second is b'65535\n'
+            # data, the first is the image dimensions and the second is the max
+            # value
             #
+            # check the previous line to get the dimensions of the image
+            prev_line_split = prev_line.decode("ascii").strip().split()
+            image_width = int(prev_line_split[0])
+            image_height = int(prev_line_split[1])
+            bytes_to_read = image_width * image_height * 2  # 16-bit image depth
+
             # read image
             try:
                 # read the image size in bytes from the file
-                image_bytes = unzipped.read(__SPECTROGRAPH_IMAGE_SIZE_BYTES)
+                image_bytes = unzipped.read(bytes_to_read)
 
                 # format bytes into numpy array of unsigned shorts (2byte numbers, 0-65536),
                 # effectively an array of pixel values
                 image_np = np.frombuffer(image_bytes, dtype=__SPECTROGRAPH_DT)
 
-                # change 1d numpy array into 1024x256 matrix with correctly located pixels
-                image_matrix = np.reshape(image_np, (1024, 256, 1))
+                # change 1d numpy array into matrix with correctly located pixels
+                image_matrix = np.reshape(image_np, (image_height, image_width, 1))
             except Exception as e:
                 if (quiet is False):
                     print("Failed reading image data frame: %s" % (str(e)))
@@ -238,13 +249,17 @@ def read(file_list, workers=1, first_frame=False, no_metadata=False, quiet=False
 
     # derive number of frames to prepare for
     total_num_frames = 0
+    image_width = __SPECTROGRAPH_EXPECTED_WIDTH
+    image_height = __SPECTROGRAPH_EXPECTED_HEIGHT
     for i in range(0, len(data)):
         if (data[i][2] is True):
             continue
         total_num_frames += data[i][0].shape[2]
+        image_height = data[i][0].shape[0]
+        image_width = data[i][0].shape[1]
 
     # pre-allocate array sizes
-    images = np.empty([1024, 256, total_num_frames], dtype=__SPECTROGRAPH_DT)
+    images = np.empty([image_height, image_width, total_num_frames], dtype=__SPECTROGRAPH_DT)
     metadata_dict_list = [{}] * total_num_frames
     problematic_file_list = []