From 5ff6ac64a63ed7e75aafd05a3a5eaa0b0cac9032 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Mon, 11 Sep 2023 16:16:05 -0500 Subject: [PATCH 01/25] Add basic img to dicom --- examples/demo_img_extraction.py | 11 +++ oct_converter/dicom/dicom.py | 26 ++++-- oct_converter/dicom/img_meta.py | 135 ++++++++++++++++++++++++++++++++ oct_converter/readers/img.py | 60 +++++++++++++- 4 files changed, 224 insertions(+), 8 deletions(-) create mode 100644 oct_converter/dicom/img_meta.py diff --git a/examples/demo_img_extraction.py b/examples/demo_img_extraction.py index 0f4ef08..e0b9033 100644 --- a/examples/demo_img_extraction.py +++ b/examples/demo_img_extraction.py @@ -1,3 +1,4 @@ +from oct_converter.dicom import create_dicom_from_oct from oct_converter.readers import IMG filepath = "../sample_files/file.img" @@ -7,3 +8,13 @@ ) # returns an OCT volume with additional metadata if available oct_volume.peek() # plots a montage of the volume oct_volume.save("img_testing.avi") # save volume + +# create a DICOM from .img +dcm = create_dicom_from_oct(filepath) +# Output dir can be specified, otherwise will +# default to current working directory. +# Output filename can be specified, otherwise +# will default to the input filename. +# Additionally, rows, columns, and interlaced can +# be specified to more accurately create an image. +dcm = create_dicom_from_oct(filepath, interlaced=True) diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index 58965bb..e7ebb3d 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -13,8 +13,9 @@ from oct_converter.dicom.fda_meta import fda_dicom_metadata from oct_converter.dicom.fds_meta import fds_dicom_metadata +from oct_converter.dicom.img_meta import img_dicom_metadata from oct_converter.dicom.metadata import DicomMetadata -from oct_converter.readers import FDA, FDS +from oct_converter.readers import FDA, FDS, IMG # Deterministic implentation UID based on package name and version version = metadata.version("oct_converter") @@ -242,14 +243,18 @@ def write_opt_dicom( def create_dicom_from_oct( - input_file: str, output_dir: str = None, output_filename: str = None + input_file: str, + output_dir: str = None, + output_filename: str = None, + rows: int = 1024, + cols: int = 512, + interlaced: bool = False, ) -> Path: """Creates a DICOM file with the data parsed from the input file. Args: - input_file: File with OCT data (Currently only Topcon - files supported) + input_file: File with OCT data, .fda/.fds/.img output_dir: Output directory, will be created if not currently exists. Default None places file in current working directory. @@ -257,6 +262,9 @@ def create_dicom_from_oct( `filename.dcm`. Default None saves the file under the input filename (if input_file = `test.fds`, output_filename = `test.dcm`) + rows: If .img file, allows for manually setting rows + cols: If .img file, allows for manually setting cols + interlaced: If .img file, allows for setting interlaced Returns: Path to DICOM file @@ -270,13 +278,17 @@ def create_dicom_from_oct( fda = FDA(input_file) oct = fda.read_oct_volume() meta = fda_dicom_metadata(oct) - elif file_suffix in ["e2e", "img", "oct"]: + elif file_suffix == "img": + img = IMG(input_file) + oct = img.read_oct_volume(rows, cols, interlaced) + meta = img_dicom_metadata(oct) + elif file_suffix in ["e2e", "oct"]: raise NotImplementedError( - f"DICOM conversion for {file_suffix} is not yet supported. Currently supported filetypes are .fds, .fda." + f"DICOM conversion for {file_suffix} is not yet supported. Currently supported filetypes are .fds, .fda, .img." ) else: raise TypeError( - f"DICOM conversion for {file_suffix} is not supported. Currently supported filetypes are .fds, .fda." + f"DICOM conversion for {file_suffix} is not supported. Currently supported filetypes are .fds, .fda, .img." ) if output_dir: diff --git a/oct_converter/dicom/img_meta.py b/oct_converter/dicom/img_meta.py new file mode 100644 index 0000000..e0cf366 --- /dev/null +++ b/oct_converter/dicom/img_meta.py @@ -0,0 +1,135 @@ +from datetime import datetime + +from oct_converter.dicom.metadata import ( + DicomMetadata, + ImageGeometry, + ManufacturerMeta, + OCTDetectorType, + OCTImageParams, + OPTAcquisitionDevice, + OPTAnatomyStructure, + PatientMeta, + SeriesMeta, +) +from oct_converter.image_types import OCTVolumeWithMetaData + + +def img_patient_meta(patient_id: str = "") -> PatientMeta: + """Creates PatientMeta populated with id if known from filename + + Args: + patient_id: Patient ID, parsed from filename + Returns: + PatientMeta: Empty (or mostly empty) PatientMeta + """ + patient = PatientMeta() + + patient.first_name = "" + patient.last_name = "" + patient.patient_id = patient_id + patient.patient_sex = "" + patient.patient_dob = None + + return patient + + +def img_series_meta( + acquisition_date: datetime | None = None, + laterality: str = "", +) -> SeriesMeta: + """Creates SeriesMeta popualted with aquisition_date and + laterality if known + + Args: + acquisition_date: Date of acquisition, parsed from filename + laterality: R or L, parsed from filename OD or OS + Returns: + SeriesMeta: Mostly empty SeriesMeta + """ + series = SeriesMeta() + + series.study_id = "" + series.series_id = 0 + series.laterality = laterality + series.acquisition_date = acquisition_date + series.opt_anatomy = OPTAnatomyStructure.Retina + + return series + + +def img_manu_meta() -> ManufacturerMeta: + """Because img file contains no manufacture data, + creates ManufacturerMeta with only manufacturer. + + Args: + None + Returns: + ManufacturerMeta: Mostly empty ManufacturerMeta + """ + manufacture = ManufacturerMeta() + + manufacture.manufacturer = "Zeiss" + manufacture.manufacturer_model = "" + manufacture.device_serial = "" + manufacture.software_version = "" + + return manufacture + + +def img_image_geom() -> ImageGeometry: + """Creates ImageGeometry with generic values + + Args: + None + Returns: + ImageGeometry: Geometry data populated with generic values + """ + image_geom = ImageGeometry() + image_geom.pixel_spacing = [0.002, 0.002] + image_geom.slice_thickness = 0.004 + image_geom.image_orientation = [1, 0, 0, 0, 1, 0] + + return image_geom + + +def img_image_params() -> OCTImageParams: + """Creates OCTImageParams specific to Zeiss + + Args: + None + Returns: + OCTImageParams: Image params populated with img defaults + """ + image_params = OCTImageParams() + image_params.opt_acquisition_device = OPTAcquisitionDevice.OCTScanner + image_params.DetectorType = OCTDetectorType.CCD + image_params.IlluminationWaveLength = 830 + image_params.IlluminationPower = 800 + image_params.IlluminationBandwidth = 50 + image_params.DepthSpatialResolution = 6 + image_params.MaximumDepthDistortion = 0.5 + image_params.AlongscanSpatialResolution = [] + image_params.MaximumAlongscanDistortion = [] + image_params.AcrossscanSpatialResolution = [] + image_params.MaximumAcrossscanDistortion = [] + + return image_params + + +def img_dicom_metadata(oct: OCTVolumeWithMetaData) -> DicomMetadata: + """Creates DicomMetadata and populates each module + + Args: + oct: OCTVolumeWithMetaData created by img reader + Returns: + DicomMetadata: Populated DicomMetadata created with Zeiss defaults + and information extracted from filename if able + """ + meta = DicomMetadata + meta.patient_info = img_patient_meta(oct.patient_id) + meta.series_info = img_series_meta(oct.acquistion_date, oct.laterality) + meta.manufacturer_info = img_manu_meta() + meta.image_geometry = img_image_geom() + meta.oct_image_params = img_image_params() + + return meta diff --git a/oct_converter/readers/img.py b/oct_converter/readers/img.py index 2cb52e7..568d967 100644 --- a/oct_converter/readers/img.py +++ b/oct_converter/readers/img.py @@ -1,5 +1,8 @@ from __future__ import annotations +import os +import re +from datetime import datetime from pathlib import Path import numpy as np @@ -46,7 +49,62 @@ def read_oct_volume( interlaced = np.rot90(interlaced, axes=(0, 1)) volume = interlaced + meta = self.get_metadata_from_filename() + lat_map = {"OD": "R", "OS": "L", None: ""} + oct_volume = OCTVolumeWithMetaData( - [volume[:, :, i] for i in range(volume.shape[2])] + [volume[:, :, i] for i in range(volume.shape[2])], + patient_id=meta.get("patient_id"), + acquisition_date=meta.get("acq"), + laterality=lat_map[meta.get("lat", None)], + metadata=meta, ) return oct_volume + + def get_metadata_from_filename(self) -> dict: + """Attempts to find metadata within the filename + + Returns: + meta: dict of information extracted from filename + """ + filename = os.path.basename(self.filepath) + meta = {} + meta["patient_id"] = ( + re.search(r"^P\d+", filename).group(0) + if re.search(r"^P\d+", filename) + else None + ) + acq = ( + list( + re.search( + r"(?P\d{1,2})-(?P\d{1,2})-(?P\d{4})_(?P\d{1,2})-(?P\d{1,2})-(?P\d{1,2})", + filename, + ).groups() + ) + if re.search( + r"(?P\d{1,2})-(?P\d{1,2})-(?P\d{4})_(?P\d{1,2})-(?P\d{1,2})-(?P\d{1,2})", + filename, + ) + else None + ) + if acq: + meta["acq"] = datetime( + year=int(acq[2]), + month=int(acq[0]), + day=int(acq[1]), + hour=int(acq[3]), + minute=int(acq[4]), + second=int(acq[5]), + ) + meta["lat"] = ( + re.search(r"O[D|S]", filename).group(0) + if re.search(r"O[D|S]", filename) + else None + ) + meta["sn"] = ( + re.search(r"sn\d+", filename).group(0) + if re.search(r"sn\d*", filename) + else None + ) + + return meta From 85a7f77740ca86534f1e194c2c78063f8ce76240 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Fri, 15 Sep 2023 15:31:37 -0500 Subject: [PATCH 02/25] Fix spelling of acquisition --- oct_converter/dicom/img_meta.py | 2 +- oct_converter/image_types/oct.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/oct_converter/dicom/img_meta.py b/oct_converter/dicom/img_meta.py index e0cf366..0e5c7ef 100644 --- a/oct_converter/dicom/img_meta.py +++ b/oct_converter/dicom/img_meta.py @@ -127,7 +127,7 @@ def img_dicom_metadata(oct: OCTVolumeWithMetaData) -> DicomMetadata: """ meta = DicomMetadata meta.patient_info = img_patient_meta(oct.patient_id) - meta.series_info = img_series_meta(oct.acquistion_date, oct.laterality) + meta.series_info = img_series_meta(oct.acquisition_date, oct.laterality) meta.manufacturer_info = img_manu_meta() meta.image_geometry = img_image_geom() meta.oct_image_params = img_image_params() diff --git a/oct_converter/image_types/oct.py b/oct_converter/image_types/oct.py index 773f2fb..7411b3d 100644 --- a/oct_converter/image_types/oct.py +++ b/oct_converter/image_types/oct.py @@ -68,7 +68,7 @@ def __init__( # volume data self.volume_id = volume_id - self.acquistion_date = acquisition_date + self.acquisition_date = acquisition_date self.laterality = laterality self.num_slices = len(self.volume) self.contours = contours From fa563bfa01c54c723686e2283a7e4a6606a4b5f9 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Fri, 15 Sep 2023 15:34:28 -0500 Subject: [PATCH 03/25] Add base optovue oct to dicom converter --- oct_converter/dicom/dicom.py | 14 +++- oct_converter/dicom/poct_meta.py | 129 +++++++++++++++++++++++++++++++ oct_converter/readers/poct.py | 66 +++++++++++++++- 3 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 oct_converter/dicom/poct_meta.py diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index e7ebb3d..d745b18 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -15,7 +15,8 @@ from oct_converter.dicom.fds_meta import fds_dicom_metadata from oct_converter.dicom.img_meta import img_dicom_metadata from oct_converter.dicom.metadata import DicomMetadata -from oct_converter.readers import FDA, FDS, IMG +from oct_converter.dicom.poct_meta import poct_dicom_metadata +from oct_converter.readers import FDA, FDS, IMG, POCT # Deterministic implentation UID based on package name and version version = metadata.version("oct_converter") @@ -282,13 +283,18 @@ def create_dicom_from_oct( img = IMG(input_file) oct = img.read_oct_volume(rows, cols, interlaced) meta = img_dicom_metadata(oct) - elif file_suffix in ["e2e", "oct"]: + elif file_suffix == "oct": + # May need to adjust this to double check that this is Optovue + poct = POCT(input_file) + oct = poct.read_oct_volume()[0] + meta = poct_dicom_metadata(oct) + elif file_suffix == "e2e": raise NotImplementedError( - f"DICOM conversion for {file_suffix} is not yet supported. Currently supported filetypes are .fds, .fda, .img." + f"DICOM conversion for {file_suffix} is not yet supported. Currently supported filetypes are .fds, .fda, .img, .OCT." ) else: raise TypeError( - f"DICOM conversion for {file_suffix} is not supported. Currently supported filetypes are .fds, .fda, .img." + f"DICOM conversion for {file_suffix} is not supported. Currently supported filetypes are .fds, .fda, .img, .OCT." ) if output_dir: diff --git a/oct_converter/dicom/poct_meta.py b/oct_converter/dicom/poct_meta.py new file mode 100644 index 0000000..84e2648 --- /dev/null +++ b/oct_converter/dicom/poct_meta.py @@ -0,0 +1,129 @@ +from oct_converter.dicom.metadata import ( + DicomMetadata, + ImageGeometry, + ManufacturerMeta, + OCTDetectorType, + OCTImageParams, + OPTAcquisitionDevice, + OPTAnatomyStructure, + PatientMeta, + SeriesMeta, +) +from oct_converter.image_types import OCTVolumeWithMetaData + + +def poct_patient_meta() -> PatientMeta: + """Creates empty PatientMeta + + Args: + None + Returns: + PatientMeta: Patient metadata populated by poct_metadata + """ + + patient = PatientMeta() + + patient.first_name = "" + patient.last_name = "" + patient.patient_id = "" # Might be in filename + patient.patient_sex = "" + patient.patient_dob = "" + + return patient + + +def poct_series_meta(poct: OCTVolumeWithMetaData) -> SeriesMeta: + """Creates SeriesMeta from Optovue OCT metadata + + Args: + poct: OCTVolumeWithMetaData with laterality and acquisition_date attributes + Returns: + SeriesMeta: Series metadata populated with laterality and acquisition_date + """ + # Can probably save this in the OCT obj instead of in the metadata dict + series = SeriesMeta() + + series.study_id = "" + series.series_id = 0 + series.laterality = poct.laterality + series.acquisition_date = poct.acquisition_date + series.opt_anatomy = OPTAnatomyStructure.Retina + + return series + + +def poct_manu_meta() -> ManufacturerMeta: + """Creates base ManufacturerMeta for Optovue + + Args: + None + Returns: + ManufacturerMeta: base Optovue manufacture metadata + """ + + manufacture = ManufacturerMeta() + + manufacture.manufacturer = "Optovue" + manufacture.manufacturer_model = "" + manufacture.device_serial = "" + manufacture.software_version = "" + + return manufacture + + +def poct_image_geom(pixel_spacing: list) -> ImageGeometry: + """Creates ImageGeometry from Optovue OCT metadata + + Args: + pixel_spacing: Pixel spacing calculated in the poct reader + Returns: + ImageGeometry: Geometry data populated by pixel_spacing + """ + image_geom = ImageGeometry() + image_geom.pixel_spacing = pixel_spacing + image_geom.slice_thickness = 0.02 # Placeholder value + image_geom.image_orientation = [1, 0, 0, 0, 1, 0] + + return image_geom + + +def poct_image_params() -> OCTImageParams: + """Creates OCTImageParams specific to Optovue + + Args: + None + Returns: + OCTImageParams: Image params populated with Optovue defaults + """ + image_params = OCTImageParams() + image_params.opt_acquisition_device = OPTAcquisitionDevice.OCTScanner + image_params.DetectorType = OCTDetectorType.CCD + image_params.IlluminationWaveLength = 880 + image_params.IlluminationPower = 1200 + image_params.IlluminationBandwidth = 50 + image_params.DepthSpatialResolution = 7 + image_params.MaximumDepthDistortion = 0.5 + image_params.AlongscanSpatialResolution = [] + image_params.MaximumAlongscanDistortion = [] + image_params.AcrossscanSpatialResolution = [] + image_params.MaximumAcrossscanDistortion = [] + + return image_params + + +def poct_dicom_metadata(poct: OCTVolumeWithMetaData) -> DicomMetadata: + """Creates DicomMetadata and populates each module + + Args: + oct: OCTVolumeWithMetaData created by the poct reader + Returns: + DicomMetadata: Populated DicomMetadata created with OCT metadata + """ + meta = DicomMetadata + meta.patient_info = poct_patient_meta() + meta.series_info = poct_series_meta(poct) + meta.manufacturer_info = poct_manu_meta() + meta.image_geometry = poct_image_geom(poct.pixel_spacing) + meta.oct_image_params = poct_image_params() + + return meta diff --git a/oct_converter/readers/poct.py b/oct_converter/readers/poct.py index 642fcce..402da63 100644 --- a/oct_converter/readers/poct.py +++ b/oct_converter/readers/poct.py @@ -1,5 +1,7 @@ from __future__ import annotations +import re +from datetime import datetime from pathlib import Path import numpy as np @@ -27,6 +29,7 @@ def __init__(self, filepath: str | Path) -> None: def _read_filespec(self) -> None: scan_info = [] + file_info = {} with open(self.filespec, "r", encoding="iso-8859-1") as f: lines = f.readlines() for i, line in enumerate(lines): @@ -40,7 +43,58 @@ def _read_filespec(self) -> None: scan_info.append( {"height": height, "length": scan_length, "number": scan_number} ) + if "Eye Scanned" in line: + if "OD" in line: + file_info["laterality"] = "R" + elif "OS" in line: + file_info["laterality"] = "L" + if "Video Height" in line: + file_info["video_height"] = line.split("=")[-1] + if "Video Width" in line: + file_info["video_width"] = line.split("=")[-1] + if "BitCount" in line: + file_info["bitcount"] = line.split("=")[-1] + if "Physical video width" in line: + file_info["physical_width"] = line.split("=")[-1] + if "Physical video Height" in line: + file_info["physical_height"] = line.split("=")[-1] + + if "video_height" in file_info and "physical_height" in file_info: + file_info["scale_y"] = float(file_info["physical_height"].split()[0]) / abs( + float(file_info["video_height"]) + ) + if "video_width" in file_info and "physical_width" in file_info: + file_info["scale_x"] = float(file_info["physical_width"].split()[0]) / abs( + float(file_info["video_width"]) + ) + + # Also, from the filename, grab acquisition date + # And maybe patient ID? + acq = ( + list( + re.search( + r"(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})_(?P\d{1,2}).(?P\d{1,2}).(?P\d{1,2})", + str(self.filespec), + ).groups() + ) + if re.search( + r"(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})_(?P\d{1,2}).(?P\d{1,2}).(?P\d{1,2})", + str(self.filespec), + ) + else None + ) + if acq: + file_info["acquisition_date"] = datetime( + year=int(acq[0]), + month=int(acq[1]), + day=int(acq[2]), + hour=int(acq[3]), + minute=int(acq[4]), + second=int(acq[5]), + ) + self.scan_info = scan_info + self.file_info = file_info def read_oct_volume(self) -> list[OCTVolumeWithMetaData]: """Reads OCT data. @@ -63,5 +117,15 @@ def read_oct_volume(self) -> list[OCTVolumeWithMetaData]: slice = data[i * num_pixels_slice : (i + 1) * num_pixels_slice] slice = np.rot90(slice.reshape(volume["length"], volume["height"])) all_slices.append(slice) - all_volumes.append(OCTVolumeWithMetaData(all_slices)) + all_volumes.append( + OCTVolumeWithMetaData( + all_slices, + acquisition_date=self.file_info.get("acquisition_date", None), + laterality=self.file_info.get("laterality", ""), + pixel_spacing=[ + self.file_info.get("scale_x", 0.015), + self.file_info.get("scale_y", 0.015), + ], + ) + ) return all_volumes From 57344ac50bc28c56b656a315fcabb2d2174128d7 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Wed, 20 Sep 2023 11:43:38 -0500 Subject: [PATCH 04/25] Add base e2e to dicom --- oct_converter/dicom/dicom.py | 74 +++++++++++++++--- oct_converter/dicom/e2e_meta.py | 132 ++++++++++++++++++++++++++++++++ oct_converter/readers/e2e.py | 51 ++++++++++-- 3 files changed, 240 insertions(+), 17 deletions(-) create mode 100644 oct_converter/dicom/e2e_meta.py diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index d745b18..ae67a65 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -11,12 +11,13 @@ generate_uid, ) +from oct_converter.dicom.e2e_meta import e2e_dicom_metadata from oct_converter.dicom.fda_meta import fda_dicom_metadata from oct_converter.dicom.fds_meta import fds_dicom_metadata from oct_converter.dicom.img_meta import img_dicom_metadata from oct_converter.dicom.metadata import DicomMetadata from oct_converter.dicom.poct_meta import poct_dicom_metadata -from oct_converter.readers import FDA, FDS, IMG, POCT +from oct_converter.readers import E2E, FDA, FDS, IMG, POCT # Deterministic implentation UID based on package name and version version = metadata.version("oct_converter") @@ -219,6 +220,8 @@ def write_opt_dicom( per_frame = [] pixel_data_bytes = list() + # Normalize + frames = normalize_volume(frames) # Convert to a 3d volume pixel_data = np.array(frames).astype(np.uint16) ds.Rows = pixel_data.shape[1] @@ -271,6 +274,7 @@ def create_dicom_from_oct( Path to DICOM file """ file_suffix = input_file.split(".")[-1].lower() + vol_dict = None if file_suffix == "fds": fds = FDS(input_file) oct = fds.read_oct_volume() @@ -286,15 +290,31 @@ def create_dicom_from_oct( elif file_suffix == "oct": # May need to adjust this to double check that this is Optovue poct = POCT(input_file) - oct = poct.read_oct_volume()[0] - meta = poct_dicom_metadata(oct) + octs = poct.read_oct_volume() + if len(octs) == 1: + oct = octs[0] + meta = poct_dicom_metadata(oct) + else: + vol_dict = dict() + for count, oct in enumerate(octs): + meta = poct_dicom_metadata(oct) + vol_dict[count] = (oct, meta) + meta = poct_dicom_metadata(oct) elif file_suffix == "e2e": - raise NotImplementedError( - f"DICOM conversion for {file_suffix} is not yet supported. Currently supported filetypes are .fds, .fda, .img, .OCT." - ) + e2e = E2E(input_file) + octs = e2e.read_oct_volume() + if len(octs) == 1: + oct = octs[0] + meta = e2e_dicom_metadata(oct) + else: + vol_dict = dict() + for count, oct in enumerate(octs): + meta = e2e_dicom_metadata(oct) + vol_dict[count] = (oct, meta) else: raise TypeError( - f"DICOM conversion for {file_suffix} is not supported. Currently supported filetypes are .fds, .fda, .img, .OCT." + f"DICOM conversion for {file_suffix} is not supported. " + "Currently supported filetypes are .fds, .fda, .img, .OCT." ) if output_dir: @@ -303,11 +323,41 @@ def create_dicom_from_oct( else: output_dir = Path.cwd() - if not output_filename: - output_filename = Path(input_file).stem + ".dcm" + if vol_dict: + files = [] + for vol_num in vol_dict.keys(): + if not output_filename: + filename = f"{Path(input_file).stem}_{str(vol_num)}.dcm" + else: + # Needs refining. Filename might include additional .'s + # or other problems. + filename_parts = output_filename.split(".") + filename = f"{filename_parts[0]}_{str(vol_num)}.{filename_parts[-1]}" + filepath = Path(output_dir, filename) + oct, meta = vol_dict[vol_num] + file = write_opt_dicom(meta, oct.volume, filepath) + files.append(file) + return files + else: + if not output_filename: + output_filename = f"{Path(input_file).stem}.dcm" + filepath = Path(output_dir, output_filename) + file = write_opt_dicom(meta, oct.volume, filepath) + return file - filepath = Path(output_dir, output_filename) - file = write_opt_dicom(meta, oct.volume, filepath) +def normalize_volume(vol: list[np.ndarray]) -> list[np.ndarray]: + """Normalizes pixel intensities within a range of 0-100. - return file + Args: + vol: List of frames + Returns: + Normalized list of frames + """ + arr = np.array(vol) + norm_vol = [] + diff_arr = arr.max() - arr.min() + for i in arr: + temp = ((i - arr.min()) / diff_arr) * 100 + norm_vol.append(temp) + return norm_vol diff --git a/oct_converter/dicom/e2e_meta.py b/oct_converter/dicom/e2e_meta.py new file mode 100644 index 0000000..908ca46 --- /dev/null +++ b/oct_converter/dicom/e2e_meta.py @@ -0,0 +1,132 @@ +from oct_converter.dicom.metadata import ( + DicomMetadata, + ImageGeometry, + ManufacturerMeta, + OCTDetectorType, + OCTImageParams, + OPTAcquisitionDevice, + OPTAnatomyStructure, + PatientMeta, + SeriesMeta, +) +from oct_converter.image_types import OCTVolumeWithMetaData + + +def e2e_patient_meta(oct: OCTVolumeWithMetaData) -> PatientMeta: + """Creates PatientMeta from e2e info stored in OCTVolumeWithMetaData + + Args: + oct: OCTVolumeWithMetaData obj with patient info attributes + Returns: + PatientMeta: Patient metadata populated by oct + """ + patient = PatientMeta() + + patient.first_name = oct.first_name + patient.last_name = oct.surname + # E2E has conflicting patient_ids, between the one in the + # patient chunk and the chunk and subdirectory headers. + # The patient chunk data is used here. + patient.patient_id = oct.patient_id + patient.patient_sex = oct.sex + patient.patient_dob = oct.DOB + + return patient + + +def e2e_series_meta(oct: OCTVolumeWithMetaData) -> SeriesMeta: + """Creates SeriesMeta from e2e info stored in OCTVolumeWithMetaData + + Args: + oct: OCTVolumeWithMetaData obj with series info attributes + Returns: + SeriesMeta: Series metadata populated by oct + """ + patient_id, study_id, series_id = oct.volume_id.split("_") + series = SeriesMeta() + + series.study_id = study_id + series.series_id = series_id + series.laterality = oct.laterality + series.acquisition_date = oct.acquisition_date + series.opt_anatomy = OPTAnatomyStructure.Retina + + return series + + +def e2e_manu_meta() -> ManufacturerMeta: + """Creates ManufacturerMeta with Heidelberg defaults. + + Args: + None + Returns: + ManufacturerMeta: Manufacture metadata module + """ + manufacture = ManufacturerMeta() + + manufacture.manufacturer = "Heidelberg" + manufacture.manufacturer_model = "" + manufacture.device_serial = "" + manufacture.software_version = "" + + return manufacture + + +def e2e_image_geom(pixel_spacing: list) -> ImageGeometry: + """Creates ImageGeometry from E2E metadata + + Args: + pixel_spacing: Pixel spacing identified by E2E reader + Returns: + ImageGeometry: Geometry data populated by pixel_spacing + """ + image_geom = ImageGeometry() + image_geom.pixel_spacing = [pixel_spacing[0], pixel_spacing[1]] + # ScaleX, ScaleY + # ScaleY seems to be found, and also constant. + image_geom.slice_thickness = pixel_spacing[2] + image_geom.image_orientation = [1, 0, 0, 0, 1, 0] + + return image_geom + + +def e2e_image_params() -> OCTImageParams: + """Creates OCTImageParams specific to E2E + + Args: + None + Returns: + OCTImageParams: Image params populated with E2E defaults + """ + image_params = OCTImageParams() + image_params.opt_acquisition_device = OPTAcquisitionDevice.OCTScanner + image_params.DetectorType = OCTDetectorType.CCD + image_params.IlluminationWaveLength = 880 + image_params.IlluminationPower = 1200 + image_params.IlluminationBandwidth = 50 + image_params.DepthSpatialResolution = 7 + image_params.MaximumDepthDistortion = 0.5 + image_params.AlongscanSpatialResolution = "" + image_params.MaximumAlongscanDistortion = "" + image_params.AcrossscanSpatialResolution = "" + image_params.MaximumAcrossscanDistortion = "" + + return image_params + + +def e2e_dicom_metadata(oct: OCTVolumeWithMetaData) -> DicomMetadata: + """Creates DicomMetadata and populates each module + + Args: + oct: OCTVolumeWithMetaData created by the E2E reader + Returns: + DicomMetadata: Populated DicomMetadata created with OCT metadata + """ + meta = DicomMetadata + meta.patient_info = e2e_patient_meta(oct) + meta.series_info = e2e_series_meta(oct) + meta.manufacturer_info = e2e_manu_meta() + meta.image_geometry = e2e_image_geom(oct.pixel_spacing) + meta.oct_image_params = e2e_image_params() + + return meta diff --git a/oct_converter/readers/e2e.py b/oct_converter/readers/e2e.py index 5e33cfd..03564e3 100644 --- a/oct_converter/readers/e2e.py +++ b/oct_converter/readers/e2e.py @@ -2,7 +2,7 @@ import warnings from collections import defaultdict -from datetime import datetime, timedelta +from datetime import date, datetime, timedelta from itertools import chain from pathlib import Path @@ -33,6 +33,8 @@ def __init__(self, filepath: str | Path) -> None: self.first_name = None self.surname = None self.acquisition_date = None + self.birthdate = None + self.pixel_spacing = None # get initial directory structure with open(self.filepath, "rb") as f: @@ -129,8 +131,8 @@ def _make_lut(): self.sex = patient_data.sex self.first_name = patient_data.first_name self.surname = patient_data.surname - # this gives the birthdate as a Julian date, needs converting to calendar date - self.birthdate = (patient_data.birthdate / 64) - 14558805 + julian_birthdate = (patient_data.birthdate / 64) - 14558805 + self.birthdate = self.julian_to_ymd(julian_birthdate) self.patient_id = patient_data.patient_id except Exception: pass @@ -146,6 +148,9 @@ def _make_lut(): ) if self.acquisition_date is None: self.acquisition_date = acquisition_datetime.date() + if self.pixel_spacing is None: + # 0.004's are placeholders + self.pixel_spacing = [0.004, bscan_metadata.scaley, 0.004] elif chunk.type == 11: # laterality data raw = f.read(20) @@ -254,6 +259,9 @@ def _make_lut(): for slice_id, contour in contour_values.items(): (contour_data[volume_id][contour_name][slice_id]) = contour + # Read metadata to attach to OCTVolumeWithMetaData + metadata = self.read_all_metadata() + oct_volumes = [] for key, volume in chain( volume_array_dict.items(), volume_array_dict_additional.items() @@ -269,10 +277,13 @@ def _make_lut(): first_name=self.first_name, surname=self.surname, sex=self.sex, + patient_dob=self.birthdate, acquisition_date=self.acquisition_date, volume_id=key, laterality=laterality_dict.get(key), contours=contour_data.get(key), + pixel_spacing=self.pixel_spacing, + metadata=metadata, ) ) @@ -315,8 +326,8 @@ def read_fundus_image(self) -> list[FundusImageWithMetaData]: self.sex = patient_data.sex self.first_name = patient_data.first_name self.surname = patient_data.surname - # this gives the birthdate as a Julian date, needs converting to calendar date - self.birthdate = (patient_data.birthdate / 64) - 14558805 + julian_birthdate = (patient_data.birthdate / 64) - 14558805 + self.birthdate = self.julian_to_ymd(julian_birthdate) self.patient_id = patient_data.patient_id except Exception: pass @@ -513,3 +524,33 @@ def vol_intensity_transform(self, data: np.array) -> np.array: data[selection_0] = 0 data = np.clip(data, 0, 1) return data + + def julian_to_ymd(self, J): + """Converts Julian Day to Gregorian YMD. + + see https://en.wikipedia.org/wiki/Julian_day + with thanks to https://github.com/seanredmond/juliandate + """ + y = 4716 + j = 1401 + m = 2 + n = 12 + r = 4 + p = 1461 + v = 3 + u = 5 + s = 153 + w = 2 + B = 274277 + C = -38 + + f = J + j + int(((int((4 * J + B) / 146097)) * 3) / 4) + C + e = r * f + v + g = int((e % p) / r) + h = u * g + w + + D = int((h % s) / u) + 1 + M = ((int(h / s) + m) % n) + 1 + Y = int(e / p) - y + int((n + m - M) / n) + + return date(Y, M, D) From 63c72d8d164d73b67892bb6537a7ede103be07fb Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 26 Sep 2023 16:03:22 -0500 Subject: [PATCH 05/25] Small updates --- oct_converter/dicom/dicom.py | 12 ++++++++---- oct_converter/readers/binary_structs/e2e_binary.py | 11 ++++++++--- oct_converter/readers/e2e.py | 8 ++++---- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index ae67a65..fc7d2c1 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -258,7 +258,7 @@ def create_dicom_from_oct( the input file. Args: - input_file: File with OCT data, .fda/.fds/.img + input_file: File with OCT data, .fda/.fds/.img/.e2e/.OCT output_dir: Output directory, will be created if not currently exists. Default None places file in current working directory. @@ -299,11 +299,15 @@ def create_dicom_from_oct( for count, oct in enumerate(octs): meta = poct_dicom_metadata(oct) vol_dict[count] = (oct, meta) - meta = poct_dicom_metadata(oct) elif file_suffix == "e2e": e2e = E2E(input_file) octs = e2e.read_oct_volume() - if len(octs) == 1: + if len(octs) == 0: + raise ValueError( + "No OCT volumes found in e2e input file. Fundus images are " + "not currently supported for DICOM conversion." + ) + elif len(octs) == 1: oct = octs[0] meta = e2e_dicom_metadata(oct) else: @@ -314,7 +318,7 @@ def create_dicom_from_oct( else: raise TypeError( f"DICOM conversion for {file_suffix} is not supported. " - "Currently supported filetypes are .fds, .fda, .img, .OCT." + "Currently supported filetypes are .e2e, .fds, .fda, .img, .OCT." ) if output_dir: diff --git a/oct_converter/readers/binary_structs/e2e_binary.py b/oct_converter/readers/binary_structs/e2e_binary.py index f166dec..fb0e48f 100644 --- a/oct_converter/readers/binary_structs/e2e_binary.py +++ b/oct_converter/readers/binary_structs/e2e_binary.py @@ -32,7 +32,9 @@ "start" / Int32un, "size" / Int32un, "unknown" / Int32un, - "patient_id" / Int32un, + # Patient DB ID is set by the software + # and is not necessarily equal to patient_id_structure's patient_id + "patient_db_id" / Int32un, "study_id" / Int32un, "series_id" / Int32un, "slice_id" / Int32sn, @@ -48,7 +50,9 @@ "pos" / Int32un, "size" / Int32un, "unknown3" / Int32un, - "patient_id" / Int32un, + # Patient DB ID is set by the software + # and is not necessarily equal to patient_id_structure's patient_id + "patient_db_id" / Int32un, "study_id" / Int32un, "series_id" / Int32un, "slice_id" / Int32sn, @@ -66,7 +70,8 @@ ) patient_id_structure = Struct( "first_name" / PaddedString(31, "ascii"), - "surname" / PaddedString(66, "ascii"), + "surname" / PaddedString(51, "ascii"), + "title" / PaddedString(15, "ascii"), "birthdate" / Int32un, "sex" / PaddedString(1, "ascii"), "patient_id" / PaddedString(25, "ascii"), diff --git a/oct_converter/readers/e2e.py b/oct_converter/readers/e2e.py index 03564e3..a5cbdf5 100644 --- a/oct_converter/readers/e2e.py +++ b/oct_converter/readers/e2e.py @@ -94,7 +94,7 @@ def _make_lut(): raw = f.read(44) chunk = e2e_binary.sub_directory_structure.parse(raw) volume_string = "{}_{}_{}".format( - chunk.patient_id, chunk.study_id, chunk.series_id + chunk.patient_db_id, chunk.study_id, chunk.series_id ) if volume_string not in volume_dict.keys(): volume_dict[volume_string] = chunk.slice_id / 2 @@ -169,7 +169,7 @@ def _make_lut(): if contour_data.width > 0: volume_string = "{}_{}_{}".format( - chunk.patient_id, chunk.study_id, chunk.series_id + chunk.patient_db_id, chunk.study_id, chunk.series_id ) slice_id = int(chunk.slice_id / 2) - 1 contour_name = f"contour{contour_data.id}" @@ -205,7 +205,7 @@ def _make_lut(): break raw_volume = np.fromfile(f, dtype=np.uint16, count=count) volume_string = "{}_{}_{}".format( - chunk.patient_id, chunk.study_id, chunk.series_id + chunk.patient_db_id, chunk.study_id, chunk.series_id ) try: image = LUT[raw_volume].reshape( @@ -355,7 +355,7 @@ def read_fundus_image(self) -> list[FundusImageWithMetaData]: image_data.height, image_data.width ) image_string = "{}_{}_{}".format( - chunk.patient_id, chunk.study_id, chunk.series_id + chunk.patient_db_id, chunk.study_id, chunk.series_id ) image_array_dict[image_string] = image # here assumes laterality stored in chunk before the image itself From ea2401e344eff66ce0173a646e3927a5f580ab3c Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 26 Sep 2023 16:04:39 -0500 Subject: [PATCH 06/25] Add CZM to DCM --- oct_converter/readers/czm_dcm_unscrambler.py | 148 +++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 oct_converter/readers/czm_dcm_unscrambler.py diff --git a/oct_converter/readers/czm_dcm_unscrambler.py b/oct_converter/readers/czm_dcm_unscrambler.py new file mode 100644 index 0000000..9062db9 --- /dev/null +++ b/oct_converter/readers/czm_dcm_unscrambler.py @@ -0,0 +1,148 @@ +"""With many, many thanks to scaramallion, https://github.com/pydicom/pydicom/discussions/1618""" +import math + +from pydicom import dcmread +from pydicom.dataelem import validate_value, DataElement +from pydicom.dataset import validate_file_meta +from pydicom.encaps import generate_pixel_data_frame, encapsulate +from pydicom.uid import JPEG2000Lossless, ExplicitVRLittleEndian + + +def unscramble_czm(frame: bytes) -> bytearray: + """Return an unscrambled image frame. + + Parameters + ---------- + frame : bytes + The scrambled CZM JPEG 2000 data frame as found in the DICOM dataset. + + Returns + ------- + bytearray + The unscrambled JPEG 2000 data. + """ + # Fix the 0x5A XORing + frame = bytearray(frame) + for ii in range(0, len(frame), 7): + frame[ii] = frame[ii] ^ 0x5A + + # Offset to the start of the JP2 header - empirically determined + jp2_offset = math.floor(len(frame) / 5 * 3) + + # Double check that our empirically determined jp2_offset is correct + offset = frame.find(b"\x00\x00\x00\x0C") + if offset == -1: + raise ValueError("No JP2 header found in the scrambled pixel data") + + if jp2_offset != offset: + print( + f"JP2 header found at offset {offset} rather than the expected " + f"{jp2_offset}" + ) + jp2_offset = offset + + d = bytearray() + d.extend(frame[jp2_offset:jp2_offset + 253]) + d.extend(frame[993:1016]) + d.extend(frame[276:763]) + d.extend(frame[23:276]) + d.extend(frame[1016:jp2_offset]) + d.extend(frame[:23]) + d.extend(frame[763:993]) + d.extend(frame[jp2_offset + 253:]) + + assert len(d) == len(frame) + + return d + + +def tag_fixer(element: DataElement) -> DataElement: + """Given a DataElement, attempts to remove the basic + obfuscation added to various tags. If element is valid, + returns element. If element is invalid, empties value + and returns element. + + Args: + element: DICOM tag data as DataElement + + Returns: + DataElement with more-conformant values + """ + try: + element.value = element.value.split("\x00")[0] + except: + pass + try: + validate_value(element.VR, element.value, validation_mode=2) + return element + except ValueError: + element.value = "" + return element + + +def process_file(input_file: str, output_filename: str) -> None: + """Utilizes Pydicom to read the dataset, check that the + dataset is CZM, applies fixers, and outputs a deobfuscated + DICOM. + + Args: + input_file: Path to input file as a string + output_filename: Name under which to save the DICOM + """ + # Read and check the dataset is CZM + ds = dcmread(input_file) + meta = ds.file_meta + if meta.TransferSyntaxUID != JPEG2000Lossless: + raise ValueError( + "Only DICOM datasets with a 'Transfer Syntax UID' of JPEG 2000 " + "(Lossless) are supported" + ) + + if not ds.Manufacturer.startswith("Carl Zeiss Meditec"): + raise ValueError("Only CZM DICOM datasets are supported") + + if "PixelData" not in ds: + raise ValueError("No 'Pixel Data' found in the DICOM dataset") + + # Specific tag fixers + ds.PixelSpacing = ds.PixelSpacing.split("\x00")[0].split(",") + ds.OperatorsName = ds.OperatorsName.original_string.split(b"\x00")[0].decode() + ds.PatientName = ds.PatientName.original_string.split(b"=")[0].decode() + ds.Modality = "OPT" + ds.ImageOrientationPatient = [1, 0, 0, 0, 1, 0] + + lat_map = {"OD": "R", "OS": "L", "": "", None: ""} + ds.Laterality = lat_map.get(ds.Laterality, None) + + # Clean obfuscated tags + for element in ds: + if element.VR not in ["SQ", "OB"]: + tag_fixer(element) + elif element.VR == "SQ": + for sequence in element: + for e in sequence: + tag_fixer(e) + for element in meta: + tag_fixer(element) + + # Make sure file_meta is conformant + validate_file_meta(meta, enforce_standard=True) + + # Iterate through the frames, unscramble and write to file + if "NumberOfFrames" in ds: + all_frames = [] + frames = generate_pixel_data_frame(ds.PixelData, int(ds.NumberOfFrames)) + for idx, frame in enumerate(frames): + all_frames.append(unscramble_czm(frame)) + ds.PixelData = encapsulate(all_frames) + else: + frame = unscramble_czm(ds.PixelData) + ds.PixelData = encapsulate([frame]) + + # And finally, convert pixel data to ExplicitVRLittleEndian + ds.decompress() + meta.TransferSyntaxUID = ExplicitVRLittleEndian + ds.is_implicit_VR = False + ds.is_little_endian = True + + ds.save_as(output_filename) From b9dbb2f721d4ed633bd63f6a9dc7bb1a54d59144 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 10 Oct 2023 13:55:08 -0500 Subject: [PATCH 07/25] Split up dicom converter into separate functions --- oct_converter/dicom/dicom.py | 384 ++++++++++++++++++++++++++++------- 1 file changed, 311 insertions(+), 73 deletions(-) diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index fc7d2c1..434bbe0 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -174,7 +174,7 @@ def opt_shared_functional_groups(ds: Dataset, meta: DicomMetadata) -> Dataset: def write_opt_dicom( meta: DicomMetadata, frames: t.List[np.ndarray], filepath: Path ) -> Path: - """Writes required DICOM metadata and pixel data to .dcm file. + """Writes required DICOM metadata and oct pixel data to .dcm file. Args: meta: DICOM metadata information @@ -246,10 +246,122 @@ def write_opt_dicom( return filepath +def write_fundus_dicom( + meta: DicomMetadata, frames: t.List[np.ndarray], filepath: Path +) -> Path: + """Writes required DICOM metadata and fundus pixel data to .dcm file. + + Args: + meta: DICOM metadata information + frames: list of frames of pixel data + filepath: Path to where output file is being saved + Returns: + Path to created DICOM file + """ + ds = opt_base_dicom(filepath) + ds = populate_patient_info(ds, meta) + ds = populate_manufacturer_info(ds, meta) + ds = populate_opt_series(ds, meta) + ds.Modality = "OP" + ds = populate_ocular_region(ds, meta) + ds = opt_shared_functional_groups(ds, meta) + + # TODO: Frame of reference if fundus image present + + # OPT Image Module PS3.3 C.8.17.7 + ds.ImageType = ["DERIVED", "SECONDARY"] + ds.SamplesPerPixel = 1 + ds.AcquisitionDateTime = ( + meta.series_info.acquisition_date.strftime("%Y%m%d%H%M%S.%f") + if meta.series_info.acquisition_date + else "" + ) + ds.AcquisitionNumber = 1 + ds.PhotometricInterpretation = "MONOCHROME2" + # Unsigned integer + ds.PixelRepresentation = 0 + # Use 16 bit pixel + ds.BitsAllocated = 16 + ds.BitsStored = ds.BitsAllocated + ds.HighBit = ds.BitsAllocated - 1 + ds.SamplesPerPixel = 1 + ds.NumberOfFrames = 1 + + # Multi-frame Functional Groups Module PS3.3 C.7.6.16 + dt = datetime.now() + ds.ContentDate = dt.strftime("%Y%m%d") + timeStr = dt.strftime("%H%M%S.%f") # long format with micro seconds + ds.ContentTime = timeStr + ds.InstanceNumber = 1 + pixel_data = np.array(frames).astype(np.uint16) + ds.Rows = pixel_data.shape[0] + ds.Columns = pixel_data.shape[1] + + ds.PixelData = pixel_data.tobytes() + ds.save_as(filepath) + return filepath + + +def write_color_fundus_dicom( + meta: DicomMetadata, frames: t.List[np.ndarray], filepath: Path +) -> Path: + """Writes required DICOM metadata and RGB fundus pixel data to .dcm file. + + Args: + meta: DICOM metadata information + frames: list of frames of pixel data + filepath: Path to where output file is being saved + Returns: + Path to created DICOM file + """ + ds = opt_base_dicom(filepath) + ds = populate_patient_info(ds, meta) + ds = populate_manufacturer_info(ds, meta) + ds = populate_opt_series(ds, meta) + ds.Modality = "OP" + ds = populate_ocular_region(ds, meta) + ds = opt_shared_functional_groups(ds, meta) + + # TODO: Frame of reference if fundus image present + + # OPT Image Module PS3.3 C.8.17.7 + ds.ImageType = ["DERIVED", "SECONDARY"] + ds.SamplesPerPixel = 1 + ds.AcquisitionDateTime = ( + meta.series_info.acquisition_date.strftime("%Y%m%d%H%M%S.%f") + if meta.series_info.acquisition_date + else "" + ) + ds.AcquisitionNumber = 1 + ds.PhotometricInterpretation = "RGB" + # Unsigned integer + ds.PixelRepresentation = 0 + # Use 16 bit pixel + ds.BitsAllocated = 16 + ds.BitsStored = ds.BitsAllocated + ds.HighBit = ds.BitsAllocated - 1 + ds.SamplesPerPixel = 1 + ds.NumberOfFrames = 1 + + # Multi-frame Functional Groups Module PS3.3 C.7.6.16 + dt = datetime.now() + ds.ContentDate = dt.strftime("%Y%m%d") + timeStr = dt.strftime("%H%M%S.%f") # long format with micro seconds + ds.ContentTime = timeStr + ds.InstanceNumber = 1 + + pixel_data = np.array(frames).astype(np.uint16) + ds.Rows = pixel_data.shape[0] + ds.Columns = pixel_data.shape[1] + + ds.PixelData = pixel_data.tobytes() + ds.save_as(filepath) + return filepath + + def create_dicom_from_oct( input_file: str, output_dir: str = None, - output_filename: str = None, rows: int = 1024, cols: int = 512, interlaced: bool = False, @@ -262,10 +374,6 @@ def create_dicom_from_oct( output_dir: Output directory, will be created if not currently exists. Default None places file in current working directory. - output_filename: Name to save the file under, i.e. - `filename.dcm`. Default None saves the file under - the input filename (if input_file = `test.fds`, - output_filename = `test.dcm`) rows: If .img file, allows for manually setting rows cols: If .img file, allows for manually setting cols interlaced: If .img file, allows for setting interlaced @@ -273,81 +381,35 @@ def create_dicom_from_oct( Returns: Path to DICOM file """ - file_suffix = input_file.split(".")[-1].lower() - vol_dict = None - if file_suffix == "fds": - fds = FDS(input_file) - oct = fds.read_oct_volume() - meta = fds_dicom_metadata(oct) - elif file_suffix == "fda": - fda = FDA(input_file) - oct = fda.read_oct_volume() - meta = fda_dicom_metadata(oct) - elif file_suffix == "img": - img = IMG(input_file) - oct = img.read_oct_volume(rows, cols, interlaced) - meta = img_dicom_metadata(oct) - elif file_suffix == "oct": - # May need to adjust this to double check that this is Optovue - poct = POCT(input_file) - octs = poct.read_oct_volume() - if len(octs) == 1: - oct = octs[0] - meta = poct_dicom_metadata(oct) - else: - vol_dict = dict() - for count, oct in enumerate(octs): - meta = poct_dicom_metadata(oct) - vol_dict[count] = (oct, meta) - elif file_suffix == "e2e": - e2e = E2E(input_file) - octs = e2e.read_oct_volume() - if len(octs) == 0: - raise ValueError( - "No OCT volumes found in e2e input file. Fundus images are " - "not currently supported for DICOM conversion." - ) - elif len(octs) == 1: - oct = octs[0] - meta = e2e_dicom_metadata(oct) - else: - vol_dict = dict() - for count, oct in enumerate(octs): - meta = e2e_dicom_metadata(oct) - vol_dict[count] = (oct, meta) - else: - raise TypeError( - f"DICOM conversion for {file_suffix} is not supported. " - "Currently supported filetypes are .e2e, .fds, .fda, .img, .OCT." - ) - if output_dir: output_dir = Path(output_dir) output_dir.mkdir(parents=True, exist_ok=True) else: output_dir = Path.cwd() - if vol_dict: - files = [] - for vol_num in vol_dict.keys(): - if not output_filename: - filename = f"{Path(input_file).stem}_{str(vol_num)}.dcm" - else: - # Needs refining. Filename might include additional .'s - # or other problems. - filename_parts = output_filename.split(".") - filename = f"{filename_parts[0]}_{str(vol_num)}.{filename_parts[-1]}" - filepath = Path(output_dir, filename) - oct, meta = vol_dict[vol_num] - file = write_opt_dicom(meta, oct.volume, filepath) - files.append(file) + file_suffix = input_file.split(".")[-1] + + if file_suffix.lower() == "fds": + dcm = create_dicom_from_fds(input_file, output_dir) + return dcm + elif file_suffix.lower() == "fda": + dcm = create_dicom_from_fda(input_file, output_dir) + return dcm + elif file_suffix.lower() == "img": + dcm = create_dicom_from_img(input_file, output_dir, rows, cols, interlaced) + return dcm + elif file_suffix == "OCT": + # OCT and oct are different filetypes, must be case-sensitive + files = create_dicom_from_poct(input_file, output_dir) + return files + elif file_suffix.lower() == "e2e": + files = create_dicom_from_e2e(input_file=input_file, output_dir=output_dir) return files else: - if not output_filename: - output_filename = f"{Path(input_file).stem}.dcm" - filepath = Path(output_dir, output_filename) - file = write_opt_dicom(meta, oct.volume, filepath) - return file + raise TypeError( + f"DICOM conversion for {file_suffix} is not supported. " + "Currently supported filetypes are .e2e, .fds, .fda, .img, .OCT." + ) def normalize_volume(vol: list[np.ndarray]) -> list[np.ndarray]: @@ -365,3 +427,179 @@ def normalize_volume(vol: list[np.ndarray]) -> list[np.ndarray]: temp = ((i - arr.min()) / diff_arr) * 100 norm_vol.append(temp) return norm_vol + + +def create_dicom_from_e2e( + input_file: str, + output_dir: str = None, +) -> list: + """Creates DICOM file(s) with the data parsed from + the input file. + + Args: + input_file: E2E file with OCT data + output_dir: Output directory + + Returns: + Path to DICOM file + """ + e2e = E2E(input_file) + oct_volumes = e2e.read_oct_volume() + fundus_images = e2e.read_fundus_image() + if len(oct_volumes) == 0 and len(fundus_images) == 0: + raise ValueError( + "No OCT volumes or fundus images found in e2e input file." + ) + + files = [] + + if len(fundus_images) > 0: + for count, fundus in enumerate(fundus_images): + meta = e2e_dicom_metadata(fundus) + filename = f"{Path(input_file).stem}_fundus_{str(count)}.dcm" + filepath = Path(output_dir, filename) + file = write_fundus_dicom(meta, fundus.image, filepath) + files.append(file) + + if len(oct_volumes) > 0: + for count, oct in enumerate(oct_volumes): + meta = e2e_dicom_metadata(oct) + filename = f"{Path(input_file).stem}_oct_{str(count)}.dcm" + filepath = Path(output_dir, filename) + file = write_opt_dicom(meta, oct.volume, filepath) + files.append(file) + + return files + + +def create_dicom_from_fda( + input_file: str, + output_dir: str, +) -> Path: + """Creates DICOM file(s) with the data parsed from + the input file. + + Args: + input_file: FDA file with OCT data + output_dir: Output directory + + Returns: + List of path(s) to DICOM file + """ + files = [] + fda = FDA(input_file) + oct = fda.read_oct_volume() + meta = fda_dicom_metadata(oct) + output_filename = f"{Path(input_file).stem}.dcm" + filepath = Path(output_dir, output_filename) + file = write_opt_dicom(meta, oct.volume, filepath) + files.append(file) + + # Attempt to parse fundus images + fundus = fda.read_fundus_image() + if fundus: + output_filename = f"{Path(input_file).stem}_fundus.dcm" + filepath = Path(output_dir, output_filename) + meta.image_geometry.pixel_spacing = [1, 1] + file = write_color_fundus_dicom(meta, fundus.image, filepath) + files.append(file) + + fundus_grayscale = fda.read_fundus_image_gray_scale() + if fundus_grayscale: + output_filename = f"{Path(input_file).stem}_fundus_grayscale.dcm" + filepath = Path(output_dir, output_filename) + meta.image_geometry.pixel_spacing = [1, 1] + file = write_fundus_dicom(meta, fundus_grayscale.image, filepath) + files.append(file) + + return files + + +def create_dicom_from_fds( + input_file: str, + output_dir: str, +) -> Path: + """Creates DICOM file(s) with the data parsed from + the input file. + + Args: + input_file: FDS file with OCT data + output_dir: Output directory + + Returns: + List of path(s) to DICOM file + """ + files = [] + fds = FDS(input_file) + oct = fds.read_oct_volume() + meta = fds_dicom_metadata(oct) + output_filename = f"{Path(input_file).stem}.dcm" + filepath = Path(output_dir, output_filename) + file = write_opt_dicom(meta, oct.volume, filepath) + files.append(file) + + # Attempt to parse fundus images + fundus = fds.read_fundus_image() + if fundus: + output_filename = f"{Path(input_file).stem}_fundus.dcm" + filepath = Path(output_dir, output_filename) + file = write_color_fundus_dicom(meta, fundus.image, filepath) + files.append(file) + + return files + + +def create_dicom_from_img( + input_file: str, + output_dir: str, + rows: int = 1024, + cols: int = 512, + interlaced: bool = False, +) -> Path: + """Creates a DICOM file with the data parsed from + the input file. + + Args: + input_file: .img file with OCT data + output_dir: Output directory + rows: Optional, for manually setting rows. Default 1024. + cols: Optional, for manually setting cols. Default 512. + interlaced: Optional, for setting interlaced. Default False. + + Returns: + Path to DICOM file + """ + img = IMG(input_file) + oct = img.read_oct_volume(rows, cols, interlaced) + meta = img_dicom_metadata(oct) + output_filename = f"{Path(input_file).stem}.dcm" + filepath = Path(output_dir, output_filename) + file = write_opt_dicom(meta, oct.volume, filepath) + return file + + +def create_dicom_from_poct( + input_file: str, + output_dir: str, +) -> list: + """Creates DICOM file(s) with the data parsed from + the input file. + + Args: + input_file: File with POCT data + output_dir: Output directory + + Returns: + List of path(s) to DICOM file + """ + poct = POCT(input_file) + octs = poct.read_oct_volume() + files = [] + for count, oct in enumerate(octs): + meta = poct_dicom_metadata(oct) + filename = f"{Path(input_file).stem}_{str(count)}.dcm" + filepath = Path(output_dir, filename) + file = write_opt_dicom(meta, oct.volume, filepath) + files.append(file) + + return files From eda2ef86c099052352ddbf5f46c3f3e80e4f3b2d Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 10 Oct 2023 13:56:01 -0500 Subject: [PATCH 08/25] Accommodate for fundus images in e2e --- oct_converter/dicom/e2e_meta.py | 77 ++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/oct_converter/dicom/e2e_meta.py b/oct_converter/dicom/e2e_meta.py index 908ca46..80641a9 100644 --- a/oct_converter/dicom/e2e_meta.py +++ b/oct_converter/dicom/e2e_meta.py @@ -9,46 +9,51 @@ PatientMeta, SeriesMeta, ) -from oct_converter.image_types import OCTVolumeWithMetaData +from oct_converter.image_types import ( + FundusImageWithMetaData, + OCTVolumeWithMetaData, +) -def e2e_patient_meta(oct: OCTVolumeWithMetaData) -> PatientMeta: - """Creates PatientMeta from e2e info stored in OCTVolumeWithMetaData +def e2e_patient_meta(meta: dict) -> PatientMeta: + """Creates PatientMeta from e2e info stored in raw metadata Args: - oct: OCTVolumeWithMetaData obj with patient info attributes + meta: Nested dictionary of metadata accumulated by the E2E reader Returns: PatientMeta: Patient metadata populated by oct """ patient = PatientMeta() - patient.first_name = oct.first_name - patient.last_name = oct.surname - # E2E has conflicting patient_ids, between the one in the - # patient chunk and the chunk and subdirectory headers. - # The patient chunk data is used here. - patient.patient_id = oct.patient_id - patient.patient_sex = oct.sex - patient.patient_dob = oct.DOB + patient_data = meta.get("patient_data", [{}]) + + patient.first_name = patient_data[0].get("first_name") + patient.last_name = patient_data[0].get("surname") + patient.patient_id = patient_data[0].get("patient_id") + patient.patient_sex = patient_data[0].get("sex") + # DOB needs work + # patient.patient_dob = None return patient -def e2e_series_meta(oct: OCTVolumeWithMetaData) -> SeriesMeta: - """Creates SeriesMeta from e2e info stored in OCTVolumeWithMetaData +def e2e_series_meta(id, laterality, acquisition_date) -> SeriesMeta: + """Creates SeriesMeta from info parsed by the E2E reader Args: - oct: OCTVolumeWithMetaData obj with series info attributes + id: Equivalent to oct.volume_id or fundus.image_id + laterality: R or L, from image.laterality + acquisition_date: Scan date for OCT, or None for fundus Returns: SeriesMeta: Series metadata populated by oct """ - patient_id, study_id, series_id = oct.volume_id.split("_") + patient_id, study_id, series_id = id.split("_") series = SeriesMeta() series.study_id = study_id series.series_id = series_id - series.laterality = oct.laterality - series.acquisition_date = oct.acquisition_date + series.laterality = laterality + series.acquisition_date = acquisition_date series.opt_anatomy = OPTAnatomyStructure.Retina return series @@ -64,8 +69,8 @@ def e2e_manu_meta() -> ManufacturerMeta: """ manufacture = ManufacturerMeta() - manufacture.manufacturer = "Heidelberg" - manufacture.manufacturer_model = "" + manufacture.manufacturer = "Heidelberg Engineering" + manufacture.manufacturer_model = "Spectralis" manufacture.device_serial = "" manufacture.software_version = "" @@ -81,9 +86,7 @@ def e2e_image_geom(pixel_spacing: list) -> ImageGeometry: ImageGeometry: Geometry data populated by pixel_spacing """ image_geom = ImageGeometry() - image_geom.pixel_spacing = [pixel_spacing[0], pixel_spacing[1]] - # ScaleX, ScaleY - # ScaleY seems to be found, and also constant. + image_geom.pixel_spacing = [pixel_spacing[1], pixel_spacing[0]] image_geom.slice_thickness = pixel_spacing[2] image_geom.image_orientation = [1, 0, 0, 0, 1, 0] @@ -106,27 +109,33 @@ def e2e_image_params() -> OCTImageParams: image_params.IlluminationBandwidth = 50 image_params.DepthSpatialResolution = 7 image_params.MaximumDepthDistortion = 0.5 - image_params.AlongscanSpatialResolution = "" - image_params.MaximumAlongscanDistortion = "" - image_params.AcrossscanSpatialResolution = "" - image_params.MaximumAcrossscanDistortion = "" + image_params.AlongscanSpatialResolution = 13 + image_params.MaximumAlongscanDistortion = 0.5 + image_params.AcrossscanSpatialResolution = 13 + image_params.MaximumAcrossscanDistortion = 0.5 return image_params -def e2e_dicom_metadata(oct: OCTVolumeWithMetaData) -> DicomMetadata: - """Creates DicomMetadata and populates each module +def e2e_dicom_metadata(image: FundusImageWithMetaData | OCTVolumeWithMetaData) -> DicomMetadata: + """Creates DicomMetadata for oct or fundus image and populates each module Args: - oct: OCTVolumeWithMetaData created by the E2E reader + image: Oct or Fundus image type created by the E2E reader Returns: - DicomMetadata: Populated DicomMetadata created with OCT metadata + DicomMetadata: Populated DicomMetadata created with fundus or oct metadata """ + meta = DicomMetadata - meta.patient_info = e2e_patient_meta(oct) - meta.series_info = e2e_series_meta(oct) + meta.patient_info = e2e_patient_meta(image.metadata) meta.manufacturer_info = e2e_manu_meta() - meta.image_geometry = e2e_image_geom(oct.pixel_spacing) meta.oct_image_params = e2e_image_params() + if type(image) == OCTVolumeWithMetaData: + meta.series_info = e2e_series_meta(image.volume_id, image.laterality, image.acquisition_date) + meta.image_geometry = e2e_image_geom(image.pixel_spacing) + else: # type(image) == FundusImageWithMetaData + meta.series_info = e2e_series_meta(image.image_id, image.laterality, None) + meta.image_geometry = e2e_image_geom([1, 1, 1]) + return meta From fe1733aac2eacea257af3a2b624408c6a1cf9b91 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 10 Oct 2023 13:56:26 -0500 Subject: [PATCH 09/25] Add self.metadata to Fundus image type --- oct_converter/image_types/fundus.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oct_converter/image_types/fundus.py b/oct_converter/image_types/fundus.py index b351dc8..4e6bbb8 100644 --- a/oct_converter/image_types/fundus.py +++ b/oct_converter/image_types/fundus.py @@ -32,12 +32,14 @@ def __init__( patient_id: str | None = None, image_id: str | None = None, patient_dob: str | None = None, + metadata: dict | None = None, ) -> None: self.image = image self.laterality = laterality self.patient_id = patient_id self.image_id = image_id self.DOB = patient_dob + self.metadata = metadata def save(self, filepath: str | Path) -> None: """Saves fundus image. From c18cc9190befa2fc90459ca8e80acd2a127191ca Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 10 Oct 2023 13:56:59 -0500 Subject: [PATCH 10/25] Update pixel spacing and fundus metadata --- oct_converter/readers/e2e.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/oct_converter/readers/e2e.py b/oct_converter/readers/e2e.py index a5cbdf5..5276118 100644 --- a/oct_converter/readers/e2e.py +++ b/oct_converter/readers/e2e.py @@ -149,8 +149,9 @@ def _make_lut(): if self.acquisition_date is None: self.acquisition_date = acquisition_datetime.date() if self.pixel_spacing is None: - # 0.004's are placeholders - self.pixel_spacing = [0.004, bscan_metadata.scaley, 0.004] + # scaley found, x and z not yet found in file + # but taken from E2E reader settings + self.pixel_spacing = [0.011484, bscan_metadata.scaley, 0.244673] elif chunk.type == 11: # laterality data raw = f.read(20) @@ -360,6 +361,10 @@ def read_fundus_image(self) -> list[FundusImageWithMetaData]: image_array_dict[image_string] = image # here assumes laterality stored in chunk before the image itself laterality_dict[image_string] = laterality + + # Read metadata to attach to FundusImageWithMetaData + metadata = self.read_all_metadata() + fundus_images = [] for key, image in image_array_dict.items(): fundus_images.append( @@ -370,6 +375,7 @@ def read_fundus_image(self) -> list[FundusImageWithMetaData]: laterality=laterality_dict[key] if key in laterality_dict.keys() else None, + metadata=metadata, ) ) From ae286d92bc0b935d644df01238c66a4ffbebdbbf Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 10 Oct 2023 13:58:33 -0500 Subject: [PATCH 11/25] Update examples --- examples/demo_e2e_extraction.py | 8 ++++++++ examples/demo_fda_extraction.py | 2 -- examples/demo_fds_extraction.py | 2 -- examples/demo_img_extraction.py | 2 -- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/demo_e2e_extraction.py b/examples/demo_e2e_extraction.py index 033a693..7b16ed2 100644 --- a/examples/demo_e2e_extraction.py +++ b/examples/demo_e2e_extraction.py @@ -1,5 +1,6 @@ import json +from oct_converter.dicom import create_dicom_from_oct from oct_converter.readers import E2E filepath = "../sample_files/sample.E2E" @@ -21,3 +22,10 @@ metadata = file.read_all_metadata() with open("metadata.json", "w") as outfile: outfile.write(json.dumps(metadata, indent=4)) + +# create a DICOM from E2E +dcm = create_dicom_from_oct(filepath) +# Output dir can be specified, otherwise will +# default to current working directory. +# If multiple volumes are present in the E2E file, +# multiple files will be created. \ No newline at end of file diff --git a/examples/demo_fda_extraction.py b/examples/demo_fda_extraction.py index 73ae9fd..e142aac 100644 --- a/examples/demo_fda_extraction.py +++ b/examples/demo_fda_extraction.py @@ -38,5 +38,3 @@ dcm = create_dicom_from_oct(filepath) # Output dir can be specified, otherwise will # default to current working directory. -# Output filename can be specified, otherwise -# will default to the input filename. diff --git a/examples/demo_fds_extraction.py b/examples/demo_fds_extraction.py index 1d57243..ea30026 100644 --- a/examples/demo_fds_extraction.py +++ b/examples/demo_fds_extraction.py @@ -32,5 +32,3 @@ dcm = create_dicom_from_oct(filepath) # Output dir can be specified, otherwise will # default to current working directory. -# Output filename can be specified, otherwise -# will default to the input filename. diff --git a/examples/demo_img_extraction.py b/examples/demo_img_extraction.py index e0b9033..82b4fe3 100644 --- a/examples/demo_img_extraction.py +++ b/examples/demo_img_extraction.py @@ -13,8 +13,6 @@ dcm = create_dicom_from_oct(filepath) # Output dir can be specified, otherwise will # default to current working directory. -# Output filename can be specified, otherwise -# will default to the input filename. # Additionally, rows, columns, and interlaced can # be specified to more accurately create an image. dcm = create_dicom_from_oct(filepath, interlaced=True) From c5191a69598d80b2b84f1880a999abc1457ba491 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 10 Oct 2023 14:26:31 -0500 Subject: [PATCH 12/25] minor linting --- examples/demo_e2e_extraction.py | 2 +- oct_converter/dicom/dicom.py | 20 +++++++++----------- oct_converter/dicom/e2e_meta.py | 14 +++++++------- oct_converter/readers/czm_dcm_unscrambler.py | 18 +++++++++--------- oct_converter/readers/e2e.py | 2 +- 5 files changed, 27 insertions(+), 29 deletions(-) diff --git a/examples/demo_e2e_extraction.py b/examples/demo_e2e_extraction.py index 7b16ed2..573846c 100644 --- a/examples/demo_e2e_extraction.py +++ b/examples/demo_e2e_extraction.py @@ -28,4 +28,4 @@ # Output dir can be specified, otherwise will # default to current working directory. # If multiple volumes are present in the E2E file, -# multiple files will be created. \ No newline at end of file +# multiple files will be created. diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index 434bbe0..104362e 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -447,12 +447,10 @@ def create_dicom_from_e2e( oct_volumes = e2e.read_oct_volume() fundus_images = e2e.read_fundus_image() if len(oct_volumes) == 0 and len(fundus_images) == 0: - raise ValueError( - "No OCT volumes or fundus images found in e2e input file." - ) - + raise ValueError("No OCT volumes or fundus images found in e2e input file.") + files = [] - + if len(fundus_images) > 0: for count, fundus in enumerate(fundus_images): meta = e2e_dicom_metadata(fundus) @@ -468,7 +466,7 @@ def create_dicom_from_e2e( filepath = Path(output_dir, filename) file = write_opt_dicom(meta, oct.volume, filepath) files.append(file) - + return files @@ -494,7 +492,7 @@ def create_dicom_from_fda( filepath = Path(output_dir, output_filename) file = write_opt_dicom(meta, oct.volume, filepath) files.append(file) - + # Attempt to parse fundus images fundus = fda.read_fundus_image() if fundus: @@ -511,9 +509,9 @@ def create_dicom_from_fda( meta.image_geometry.pixel_spacing = [1, 1] file = write_fundus_dicom(meta, fundus_grayscale.image, filepath) files.append(file) - + return files - + def create_dicom_from_fds( input_file: str, @@ -545,7 +543,7 @@ def create_dicom_from_fds( filepath = Path(output_dir, output_filename) file = write_color_fundus_dicom(meta, fundus.image, filepath) files.append(file) - + return files @@ -601,5 +599,5 @@ def create_dicom_from_poct( filepath = Path(output_dir, filename) file = write_opt_dicom(meta, oct.volume, filepath) files.append(file) - + return files diff --git a/oct_converter/dicom/e2e_meta.py b/oct_converter/dicom/e2e_meta.py index 80641a9..3bed067 100644 --- a/oct_converter/dicom/e2e_meta.py +++ b/oct_converter/dicom/e2e_meta.py @@ -9,10 +9,7 @@ PatientMeta, SeriesMeta, ) -from oct_converter.image_types import ( - FundusImageWithMetaData, - OCTVolumeWithMetaData, -) +from oct_converter.image_types import FundusImageWithMetaData, OCTVolumeWithMetaData def e2e_patient_meta(meta: dict) -> PatientMeta: @@ -117,7 +114,9 @@ def e2e_image_params() -> OCTImageParams: return image_params -def e2e_dicom_metadata(image: FundusImageWithMetaData | OCTVolumeWithMetaData) -> DicomMetadata: +def e2e_dicom_metadata( + image: FundusImageWithMetaData | OCTVolumeWithMetaData, +) -> DicomMetadata: """Creates DicomMetadata for oct or fundus image and populates each module Args: @@ -131,11 +130,12 @@ def e2e_dicom_metadata(image: FundusImageWithMetaData | OCTVolumeWithMetaData) - meta.manufacturer_info = e2e_manu_meta() meta.oct_image_params = e2e_image_params() if type(image) == OCTVolumeWithMetaData: - meta.series_info = e2e_series_meta(image.volume_id, image.laterality, image.acquisition_date) + meta.series_info = e2e_series_meta( + image.volume_id, image.laterality, image.acquisition_date + ) meta.image_geometry = e2e_image_geom(image.pixel_spacing) else: # type(image) == FundusImageWithMetaData meta.series_info = e2e_series_meta(image.image_id, image.laterality, None) meta.image_geometry = e2e_image_geom([1, 1, 1]) - return meta diff --git a/oct_converter/readers/czm_dcm_unscrambler.py b/oct_converter/readers/czm_dcm_unscrambler.py index 9062db9..39183d9 100644 --- a/oct_converter/readers/czm_dcm_unscrambler.py +++ b/oct_converter/readers/czm_dcm_unscrambler.py @@ -2,10 +2,10 @@ import math from pydicom import dcmread -from pydicom.dataelem import validate_value, DataElement +from pydicom.dataelem import DataElement, validate_value from pydicom.dataset import validate_file_meta -from pydicom.encaps import generate_pixel_data_frame, encapsulate -from pydicom.uid import JPEG2000Lossless, ExplicitVRLittleEndian +from pydicom.encaps import encapsulate, generate_pixel_data_frame +from pydicom.uid import ExplicitVRLittleEndian, JPEG2000Lossless def unscramble_czm(frame: bytes) -> bytearray: @@ -42,14 +42,14 @@ def unscramble_czm(frame: bytes) -> bytearray: jp2_offset = offset d = bytearray() - d.extend(frame[jp2_offset:jp2_offset + 253]) + d.extend(frame[jp2_offset : jp2_offset + 253]) d.extend(frame[993:1016]) d.extend(frame[276:763]) d.extend(frame[23:276]) d.extend(frame[1016:jp2_offset]) d.extend(frame[:23]) d.extend(frame[763:993]) - d.extend(frame[jp2_offset + 253:]) + d.extend(frame[jp2_offset + 253 :]) assert len(d) == len(frame) @@ -61,10 +61,10 @@ def tag_fixer(element: DataElement) -> DataElement: obfuscation added to various tags. If element is valid, returns element. If element is invalid, empties value and returns element. - + Args: element: DICOM tag data as DataElement - + Returns: DataElement with more-conformant values """ @@ -103,7 +103,7 @@ def process_file(input_file: str, output_filename: str) -> None: if "PixelData" not in ds: raise ValueError("No 'Pixel Data' found in the DICOM dataset") - + # Specific tag fixers ds.PixelSpacing = ds.PixelSpacing.split("\x00")[0].split(",") ds.OperatorsName = ds.OperatorsName.original_string.split(b"\x00")[0].decode() @@ -144,5 +144,5 @@ def process_file(input_file: str, output_filename: str) -> None: meta.TransferSyntaxUID = ExplicitVRLittleEndian ds.is_implicit_VR = False ds.is_little_endian = True - + ds.save_as(output_filename) diff --git a/oct_converter/readers/e2e.py b/oct_converter/readers/e2e.py index 5276118..0ac5f07 100644 --- a/oct_converter/readers/e2e.py +++ b/oct_converter/readers/e2e.py @@ -361,7 +361,7 @@ def read_fundus_image(self) -> list[FundusImageWithMetaData]: image_array_dict[image_string] = image # here assumes laterality stored in chunk before the image itself laterality_dict[image_string] = laterality - + # Read metadata to attach to FundusImageWithMetaData metadata = self.read_all_metadata() From 3c5702c4182412ba928d13109a5aef0c3c358f5f Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Tue, 24 Oct 2023 17:02:06 -0500 Subject: [PATCH 13/25] Add skeleton BOCT to DICOM beginnings --- oct_converter/dicom/boct_meta.py | 128 +++++++++++++++++++++++++++++++ oct_converter/dicom/dicom.py | 97 ++++++++++++++++------- 2 files changed, 196 insertions(+), 29 deletions(-) create mode 100644 oct_converter/dicom/boct_meta.py diff --git a/oct_converter/dicom/boct_meta.py b/oct_converter/dicom/boct_meta.py new file mode 100644 index 0000000..f724095 --- /dev/null +++ b/oct_converter/dicom/boct_meta.py @@ -0,0 +1,128 @@ +from oct_converter.dicom.metadata import ( + DicomMetadata, + ImageGeometry, + ManufacturerMeta, + OCTDetectorType, + OCTImageParams, + OPTAcquisitionDevice, + OPTAnatomyStructure, + PatientMeta, + SeriesMeta, +) +from oct_converter.image_types import OCTVolumeWithMetaData + + +def boct_patient_meta() -> PatientMeta: + """Creates empty PatientMeta + + Args: + None + Returns: + PatientMeta: Patient metadata populated by boct_metadata + """ + + patient = PatientMeta() + + patient.first_name = "" + patient.last_name = "" + patient.patient_id = "" # Might be in filename + patient.patient_sex = "" + patient.patient_dob = "" + + return patient + + +def boct_series_meta(boct: OCTVolumeWithMetaData) -> SeriesMeta: + """Creates SeriesMeta from Bioptigen OCT metadata + + Args: + boct: OCTVolumeWithMetaData with laterality and acquisition_date attributes + Returns: + SeriesMeta: Series metadata populated with laterality and acquisition_date + """ + series = SeriesMeta() + + series.study_id = "" + series.series_id = 0 + series.laterality = "" # Might be in filename + series.acquisition_date = boct.acquisition_date + series.opt_anatomy = OPTAnatomyStructure.Retina + + return series + + +def boct_manu_meta() -> ManufacturerMeta: + """Creates base ManufacturerMeta for Bioptigen + + Args: + None + Returns: + ManufacturerMeta: base Bioptigen manufacture metadata + """ + + manufacture = ManufacturerMeta() + + manufacture.manufacturer = "Bioptigen" + manufacture.manufacturer_model = "" + manufacture.device_serial = "" + manufacture.software_version = "" + + return manufacture + + +def boct_image_geom(pixel_spacing: list) -> ImageGeometry: + """Creates ImageGeometry from Bioptigen OCT metadata + + Args: + pixel_spacing: Pixel spacing calculated in the boct reader + Returns: + ImageGeometry: Geometry data populated by pixel_spacing + """ + image_geom = ImageGeometry() + image_geom.pixel_spacing = [0.02, 0.02] # Placeholder value + image_geom.slice_thickness = 0.02 # Placeholder value + image_geom.image_orientation = [1, 0, 0, 0, 1, 0] + + return image_geom + + +def boct_image_params() -> OCTImageParams: + """Creates OCTImageParams specific to Bioptigen + + Args: + None + Returns: + OCTImageParams: Image params populated with Bioptigen defaults + """ + image_params = OCTImageParams() + image_params.opt_acquisition_device = OPTAcquisitionDevice.OCTScanner + image_params.DetectorType = OCTDetectorType.CCD + image_params.IlluminationWaveLength = 880 + image_params.IlluminationPower = 1200 + image_params.IlluminationBandwidth = 50 + image_params.DepthSpatialResolution = 7 + image_params.MaximumDepthDistortion = 0.5 + image_params.AlongscanSpatialResolution = [] + image_params.MaximumAlongscanDistortion = [] + image_params.AcrossscanSpatialResolution = [] + image_params.MaximumAcrossscanDistortion = [] + + return image_params + + +def boct_dicom_metadata(boct: OCTVolumeWithMetaData) -> DicomMetadata: + """Creates DicomMetadata and populates each module + + Args: + oct: OCTVolumeWithMetaData created by the boct reader + Returns: + DicomMetadata: Populated DicomMetadata created with OCT metadata + """ + meta = DicomMetadata + meta.patient_info = boct_patient_meta() + meta.series_info = boct_series_meta(boct) + meta.manufacturer_info = boct_manu_meta() + meta.image_geometry = boct_image_geom() + meta.oct_image_params = boct_image_params() + + return meta diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index 104362e..60583b1 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -11,13 +11,14 @@ generate_uid, ) +from oct_converter.dicom.boct_meta import boct_dicom_metadata from oct_converter.dicom.e2e_meta import e2e_dicom_metadata from oct_converter.dicom.fda_meta import fda_dicom_metadata from oct_converter.dicom.fds_meta import fds_dicom_metadata from oct_converter.dicom.img_meta import img_dicom_metadata from oct_converter.dicom.metadata import DicomMetadata from oct_converter.dicom.poct_meta import poct_dicom_metadata -from oct_converter.readers import E2E, FDA, FDS, IMG, POCT +from oct_converter.readers import BOCT, E2E, FDA, FDS, IMG, POCT # Deterministic implentation UID based on package name and version version = metadata.version("oct_converter") @@ -365,7 +366,8 @@ def create_dicom_from_oct( rows: int = 1024, cols: int = 512, interlaced: bool = False, -) -> Path: + diskbuffered: bool = False, +) -> list: """Creates a DICOM file with the data parsed from the input file. @@ -377,9 +379,10 @@ def create_dicom_from_oct( rows: If .img file, allows for manually setting rows cols: If .img file, allows for manually setting cols interlaced: If .img file, allows for setting interlaced + diskbuffered: If Bioptigen .OCT, allows for setting diskbuffered Returns: - Path to DICOM file + list: list of Path(s) to DICOM file """ if output_dir: output_dir = Path(output_dir) @@ -387,30 +390,33 @@ def create_dicom_from_oct( else: output_dir = Path.cwd() - file_suffix = input_file.split(".")[-1] - - if file_suffix.lower() == "fds": - dcm = create_dicom_from_fds(input_file, output_dir) - return dcm - elif file_suffix.lower() == "fda": - dcm = create_dicom_from_fda(input_file, output_dir) - return dcm - elif file_suffix.lower() == "img": - dcm = create_dicom_from_img(input_file, output_dir, rows, cols, interlaced) - return dcm - elif file_suffix == "OCT": - # OCT and oct are different filetypes, must be case-sensitive - files = create_dicom_from_poct(input_file, output_dir) - return files - elif file_suffix.lower() == "e2e": - files = create_dicom_from_e2e(input_file=input_file, output_dir=output_dir) - return files + file_suffix = input_file.split(".")[-1].lower() + + if file_suffix == "fds": + files = create_dicom_from_fds(input_file, output_dir) + elif file_suffix == "fda": + files = create_dicom_from_fda(input_file, output_dir) + elif file_suffix == "img": + files = create_dicom_from_img(input_file, output_dir, rows, cols, interlaced) + elif file_suffix == "oct": + # Bioptigen and Octovue both use .OCT. + # BOCT._validate can check if Bioptigen, else Optivue + try: + BOCT(input_file) + files = create_dicom_from_boct(input_file, output_dir, diskbuffered) + except: + # if BOCT raises, treat as POCT + files = create_dicom_from_poct(input_file, output_dir) + elif file_suffix == "e2e": + files = create_dicom_from_e2e(input_file, output_dir) else: raise TypeError( f"DICOM conversion for {file_suffix} is not supported. " "Currently supported filetypes are .e2e, .fds, .fda, .img, .OCT." ) + return files + def normalize_volume(vol: list[np.ndarray]) -> list[np.ndarray]: """Normalizes pixel intensities within a range of 0-100. @@ -429,6 +435,39 @@ def normalize_volume(vol: list[np.ndarray]) -> list[np.ndarray]: return norm_vol +def create_dicom_from_boct( + input_file: str, + output_dir: str = None, + diskbuffered: bool = False, +) -> list: + """Creates DICOM file(s) with the data parsed from + the input file. + + Args: + input_file: Bioptigen OCT file + output_dir: Output directory + diskbuffered: If True, reduces memory usage by storing volume on disk using HDF5. + + Returns: + list: List of path(s) to DICOM file(s)""" + + boct = BOCT(input_file) + oct_volumes = boct.read_oct_volume(diskbuffered) + if len(oct_volumes) == 0: + raise ValueError("No OCT volumes found in OCT input file.") + + files = [] + + for count, oct in enumerate(oct_volumes): + meta = boct_dicom_metadata(oct) + filename = f"{Path(input_file).stem}_{str(count)}.dcm" + filepath = Path(output_dir, filename) + file = write_opt_dicom(meta, oct.volume, filepath) + files.append(file) + + return files + + def create_dicom_from_e2e( input_file: str, output_dir: str = None, @@ -441,7 +480,7 @@ def create_dicom_from_e2e( output_dir: Output directory Returns: - Path to DICOM file + list: List of path(s) to DICOM file(s) """ e2e = E2E(input_file) oct_volumes = e2e.read_oct_volume() @@ -473,7 +512,7 @@ def create_dicom_from_e2e( def create_dicom_from_fda( input_file: str, output_dir: str, -) -> Path: +) -> list: """Creates DICOM file(s) with the data parsed from the input file. @@ -482,7 +521,7 @@ def create_dicom_from_fda( output_dir: Output directory Returns: - List of path(s) to DICOM file + list: List of path(s) to DICOM file(s) """ files = [] fda = FDA(input_file) @@ -516,7 +555,7 @@ def create_dicom_from_fda( def create_dicom_from_fds( input_file: str, output_dir: str, -) -> Path: +) -> list: """Creates DICOM file(s) with the data parsed from the input file. @@ -525,7 +564,7 @@ def create_dicom_from_fds( output_dir: Output directory Returns: - List of path(s) to DICOM file + list: List of path(s) to DICOM file(s) """ files = [] fds = FDS(input_file) @@ -565,7 +604,7 @@ def create_dicom_from_img( interlaced: Optional, for setting interlaced. Default False. Returns: - Path to DICOM file + list: List of path(s) to DICOM file(s) """ img = IMG(input_file) oct = img.read_oct_volume(rows, cols, interlaced) @@ -573,7 +612,7 @@ def create_dicom_from_img( output_filename = f"{Path(input_file).stem}.dcm" filepath = Path(output_dir, output_filename) file = write_opt_dicom(meta, oct.volume, filepath) - return file + return [file] def create_dicom_from_poct( @@ -588,7 +627,7 @@ def create_dicom_from_poct( output_dir: Output directory Returns: - List of path(s) to DICOM file + list: List of path(s) to DICOM file(s) """ poct = POCT(input_file) octs = poct.read_oct_volume() From b3928e0ed60a705712497fe1d0a66df317a429ef Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Wed, 25 Oct 2023 10:47:09 -0500 Subject: [PATCH 14/25] Polishing --- oct_converter/dicom/dicom.py | 9 +++------ oct_converter/dicom/e2e_meta.py | 7 ++++--- oct_converter/dicom/poct_meta.py | 2 +- oct_converter/readers/e2e.py | 5 +++++ oct_converter/readers/poct.py | 3 +-- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index 60583b1..fd083f3 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -18,6 +18,7 @@ from oct_converter.dicom.img_meta import img_dicom_metadata from oct_converter.dicom.metadata import DicomMetadata from oct_converter.dicom.poct_meta import poct_dicom_metadata +from oct_converter.exceptions import InvalidOCTReaderError from oct_converter.readers import BOCT, E2E, FDA, FDS, IMG, POCT # Deterministic implentation UID based on package name and version @@ -267,8 +268,6 @@ def write_fundus_dicom( ds = populate_ocular_region(ds, meta) ds = opt_shared_functional_groups(ds, meta) - # TODO: Frame of reference if fundus image present - # OPT Image Module PS3.3 C.8.17.7 ds.ImageType = ["DERIVED", "SECONDARY"] ds.SamplesPerPixel = 1 @@ -323,8 +322,6 @@ def write_color_fundus_dicom( ds = populate_ocular_region(ds, meta) ds = opt_shared_functional_groups(ds, meta) - # TODO: Frame of reference if fundus image present - # OPT Image Module PS3.3 C.8.17.7 ds.ImageType = ["DERIVED", "SECONDARY"] ds.SamplesPerPixel = 1 @@ -400,11 +397,11 @@ def create_dicom_from_oct( files = create_dicom_from_img(input_file, output_dir, rows, cols, interlaced) elif file_suffix == "oct": # Bioptigen and Octovue both use .OCT. - # BOCT._validate can check if Bioptigen, else Optivue + # BOCT._validate on init can check if Bioptigen, else Optivue try: BOCT(input_file) files = create_dicom_from_boct(input_file, output_dir, diskbuffered) - except: + except InvalidOCTReaderError: # if BOCT raises, treat as POCT files = create_dicom_from_poct(input_file, output_dir) elif file_suffix == "e2e": diff --git a/oct_converter/dicom/e2e_meta.py b/oct_converter/dicom/e2e_meta.py index 3bed067..a791c34 100644 --- a/oct_converter/dicom/e2e_meta.py +++ b/oct_converter/dicom/e2e_meta.py @@ -28,8 +28,9 @@ def e2e_patient_meta(meta: dict) -> PatientMeta: patient.last_name = patient_data[0].get("surname") patient.patient_id = patient_data[0].get("patient_id") patient.patient_sex = patient_data[0].get("sex") - # DOB needs work - # patient.patient_dob = None + # TODO patient.patient_dob + # Currently, E2E's patient_dob is incorrect, see + # the E2E reader for more context. return patient @@ -44,7 +45,7 @@ def e2e_series_meta(id, laterality, acquisition_date) -> SeriesMeta: Returns: SeriesMeta: Series metadata populated by oct """ - patient_id, study_id, series_id = id.split("_") + patient_db_id, study_id, series_id = id.split("_") series = SeriesMeta() series.study_id = study_id diff --git a/oct_converter/dicom/poct_meta.py b/oct_converter/dicom/poct_meta.py index 84e2648..3d6449d 100644 --- a/oct_converter/dicom/poct_meta.py +++ b/oct_converter/dicom/poct_meta.py @@ -40,7 +40,7 @@ def poct_series_meta(poct: OCTVolumeWithMetaData) -> SeriesMeta: Returns: SeriesMeta: Series metadata populated with laterality and acquisition_date """ - # Can probably save this in the OCT obj instead of in the metadata dict + series = SeriesMeta() series.study_id = "" diff --git a/oct_converter/readers/e2e.py b/oct_converter/readers/e2e.py index 0ac5f07..15e72b3 100644 --- a/oct_converter/readers/e2e.py +++ b/oct_converter/readers/e2e.py @@ -133,6 +133,11 @@ def _make_lut(): self.surname = patient_data.surname julian_birthdate = (patient_data.birthdate / 64) - 14558805 self.birthdate = self.julian_to_ymd(julian_birthdate) + # TODO: There are conflicting ideas of how to parse E2E's birthdate + # https://bitbucket.org/uocte/uocte/wiki/Heidelberg%20File%20Format suggests the above, + # whereas https://github.com/neurodial/LibE2E/blob/master/E2E/dataelements/patientdataelement.cpp + # suggests that DOB is given as a Windows date. Neither option seems accurate to + # test files with known-correct birthdates. More investigation is needed. self.patient_id = patient_data.patient_id except Exception: pass diff --git a/oct_converter/readers/poct.py b/oct_converter/readers/poct.py index 402da63..7048f61 100644 --- a/oct_converter/readers/poct.py +++ b/oct_converter/readers/poct.py @@ -68,8 +68,7 @@ def _read_filespec(self) -> None: float(file_info["video_width"]) ) - # Also, from the filename, grab acquisition date - # And maybe patient ID? + # Attempt to find acquisition date in filename acq = ( list( re.search( From 9b259c296c58ad2e129545162aafecef59e1c276 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Wed, 25 Oct 2023 14:30:44 -0500 Subject: [PATCH 15/25] More minor adjustments --- oct_converter/dicom/boct_meta.py | 28 +++---------------- oct_converter/dicom/poct_meta.py | 22 +-------------- oct_converter/readers/boct.py | 47 +++++++++++++++++++++++++++++++- oct_converter/readers/img.py | 3 +- 4 files changed, 52 insertions(+), 48 deletions(-) diff --git a/oct_converter/dicom/boct_meta.py b/oct_converter/dicom/boct_meta.py index f724095..fe20a7e 100644 --- a/oct_converter/dicom/boct_meta.py +++ b/oct_converter/dicom/boct_meta.py @@ -12,26 +12,6 @@ from oct_converter.image_types import OCTVolumeWithMetaData -def boct_patient_meta() -> PatientMeta: - """Creates empty PatientMeta - - Args: - None - Returns: - PatientMeta: Patient metadata populated by boct_metadata - """ - - patient = PatientMeta() - - patient.first_name = "" - patient.last_name = "" - patient.patient_id = "" # Might be in filename - patient.patient_sex = "" - patient.patient_dob = "" - - return patient - - def boct_series_meta(boct: OCTVolumeWithMetaData) -> SeriesMeta: """Creates SeriesMeta from Bioptigen OCT metadata @@ -44,7 +24,7 @@ def boct_series_meta(boct: OCTVolumeWithMetaData) -> SeriesMeta: series.study_id = "" series.series_id = 0 - series.laterality = "" # Might be in filename + series.laterality = boct.laterality series.acquisition_date = boct.acquisition_date series.opt_anatomy = OPTAnatomyStructure.Retina @@ -70,7 +50,7 @@ def boct_manu_meta() -> ManufacturerMeta: return manufacture -def boct_image_geom(pixel_spacing: list) -> ImageGeometry: +def boct_image_geom() -> ImageGeometry: """Creates ImageGeometry from Bioptigen OCT metadata Args: @@ -80,7 +60,7 @@ def boct_image_geom(pixel_spacing: list) -> ImageGeometry: """ image_geom = ImageGeometry() image_geom.pixel_spacing = [0.02, 0.02] # Placeholder value - image_geom.slice_thickness = 0.02 # Placeholder value + image_geom.slice_thickness = 0.2 # Placeholder value image_geom.image_orientation = [1, 0, 0, 0, 1, 0] return image_geom @@ -119,7 +99,7 @@ def boct_dicom_metadata(boct: OCTVolumeWithMetaData) -> DicomMetadata: DicomMetadata: Populated DicomMetadata created with OCT metadata """ meta = DicomMetadata - meta.patient_info = boct_patient_meta() + meta.patient_info = PatientMeta() meta.series_info = boct_series_meta(boct) meta.manufacturer_info = boct_manu_meta() meta.image_geometry = boct_image_geom() diff --git a/oct_converter/dicom/poct_meta.py b/oct_converter/dicom/poct_meta.py index 3d6449d..96b71b1 100644 --- a/oct_converter/dicom/poct_meta.py +++ b/oct_converter/dicom/poct_meta.py @@ -12,26 +12,6 @@ from oct_converter.image_types import OCTVolumeWithMetaData -def poct_patient_meta() -> PatientMeta: - """Creates empty PatientMeta - - Args: - None - Returns: - PatientMeta: Patient metadata populated by poct_metadata - """ - - patient = PatientMeta() - - patient.first_name = "" - patient.last_name = "" - patient.patient_id = "" # Might be in filename - patient.patient_sex = "" - patient.patient_dob = "" - - return patient - - def poct_series_meta(poct: OCTVolumeWithMetaData) -> SeriesMeta: """Creates SeriesMeta from Optovue OCT metadata @@ -120,7 +100,7 @@ def poct_dicom_metadata(poct: OCTVolumeWithMetaData) -> DicomMetadata: DicomMetadata: Populated DicomMetadata created with OCT metadata """ meta = DicomMetadata - meta.patient_info = poct_patient_meta() + meta.patient_info = PatientMeta() meta.series_info = poct_series_meta(poct) meta.manufacturer_info = poct_manu_meta() meta.image_geometry = poct_image_geom(poct.pixel_spacing) diff --git a/oct_converter/readers/boct.py b/oct_converter/readers/boct.py index b7c9b67..30e4710 100644 --- a/oct_converter/readers/boct.py +++ b/oct_converter/readers/boct.py @@ -1,6 +1,8 @@ from __future__ import annotations +import re import tempfile +from datetime import datetime from pathlib import Path from typing import BinaryIO @@ -26,6 +28,7 @@ class BOCT(object): bioptigen_scan_type_map = {0: "linear", 1: "rect", 3: "rad"} file_structure = boct_binary.bioptigen_file_structure + # TODO this should contain the datetimes header_structure = boct_binary.bioptigen_oct_header_struct def __init__(self, filepath: Path | str) -> None: @@ -84,6 +87,20 @@ def read_oct_volume( else: self.vol = np.empty(self.volume_shape, dtype=np.uint16) + # Grab the acquisition datetime, + dt = oct.data[0].header.framedatetime.value + self.acquisition_datetime = datetime( + year=dt.year, + month=dt.month, + day=dt.day, + hour=dt.hour, + minute=dt.minute, + second=dt.second, + ) + + # Attempt to parse laterality from filename + self.get_laterality_from_filename() + return self.load_oct_volume() def _create_disk_buffer( @@ -109,13 +126,41 @@ def load_oct_volume(self) -> list[OCTVolumeWithMetaData]: print(e) print("Stopping load") return [ - OCTVolumeWithMetaData(self.vol[t, :, :, :]) + OCTVolumeWithMetaData( + self.vol[t, :, :, :], + acquisition_date=self.acquisition_datetime, + laterality=self.laterality, + ) for t in range(self.vol.shape[0]) ] def read_fundus_image(self) -> None: return + def get_laterality_from_filename(self) -> None: + """Attempts to find laterality within the filename, + if found, sets self.laterality + + Returns: + None + """ + filename = Path(self.filepath).name + + lat_from_filename = ( + re.search(r"O[D|S]", filename).group(0) + if re.search(r"O[D|S]", filename) + else None + ) + + if lat_from_filename == "OD": + self.laterality = "R" + elif lat_from_filename == "OS": + self.laterality = "L" + else: + self.laterality = "" + + return + class OCTFrame(object): def __init__(self, frame: Struct) -> None: diff --git a/oct_converter/readers/img.py b/oct_converter/readers/img.py index 568d967..d307e51 100644 --- a/oct_converter/readers/img.py +++ b/oct_converter/readers/img.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import re from datetime import datetime from pathlib import Path @@ -67,7 +66,7 @@ def get_metadata_from_filename(self) -> dict: Returns: meta: dict of information extracted from filename """ - filename = os.path.basename(self.filepath) + filename = Path(self.filepath).name meta = {} meta["patient_id"] = ( re.search(r"^P\d+", filename).group(0) From 496b29ddca110d4af582c062d04e9963ad120bf1 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Wed, 25 Oct 2023 14:34:46 -0500 Subject: [PATCH 16/25] Update demos --- examples/demo_boct_extraction.py | 11 +++++++++++ examples/demo_optovue_extraction.py | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/examples/demo_boct_extraction.py b/examples/demo_boct_extraction.py index b5168ba..305b52b 100644 --- a/examples/demo_boct_extraction.py +++ b/examples/demo_boct_extraction.py @@ -1,3 +1,4 @@ +from oct_converter.dicom import create_dicom_from_oct from oct_converter.readers import BOCT filepath = "../sample_files/sample.OCT" @@ -11,3 +12,13 @@ oct.save( "boct_testing.png" ) # save volume as a set of sequential images, fds_testing_[1...N].png + +# create DICOM from .OCT +dcm = create_dicom_from_oct(filepath) +# Output dir can be specified, otherwise will +# default to current working directory. +# If multiple volumes are identified within the file, +# multiple DICOMs will be outputted. +# Additionally, diskbuffered can be specified to store +# volume on disk using HDF5 to reduce memory usage +dcm = create_dicom_from_oct(filepath, diskbuffered=True) \ No newline at end of file diff --git a/examples/demo_optovue_extraction.py b/examples/demo_optovue_extraction.py index 05b4ca6..8d8c9a2 100644 --- a/examples/demo_optovue_extraction.py +++ b/examples/demo_optovue_extraction.py @@ -1,3 +1,4 @@ +from oct_converter.dicom import create_dicom_from_oct from oct_converter.readers import POCT filepath = "../sample_files/sample.OCT" @@ -7,3 +8,10 @@ for volume in oct_volumes: volume.peek() # plots a montage of the volume print("debug") + +# create DICOM from .OCT +dcm = create_dicom_from_oct(filepath) +# Output dir can be specified, otherwise will +# default to current working directory. +# If multiple volumes are identified within the file, +# multiple DICOMs will be outputted. \ No newline at end of file From d677df77676168914ce14c10ce81a42c1c566a6c Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Wed, 25 Oct 2023 15:06:15 -0500 Subject: [PATCH 17/25] Catch StreamError --- oct_converter/dicom/dicom.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index fd083f3..8cdf8e5 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -10,6 +10,7 @@ OphthalmicTomographyImageStorage, generate_uid, ) +from construct import StreamError from oct_converter.dicom.boct_meta import boct_dicom_metadata from oct_converter.dicom.e2e_meta import e2e_dicom_metadata @@ -401,7 +402,7 @@ def create_dicom_from_oct( try: BOCT(input_file) files = create_dicom_from_boct(input_file, output_dir, diskbuffered) - except InvalidOCTReaderError: + except (InvalidOCTReaderError, StreamError): # if BOCT raises, treat as POCT files = create_dicom_from_poct(input_file, output_dir) elif file_suffix == "e2e": From 3701e3b24838302a6b6174a50ee30fe2ee14b3d7 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Wed, 25 Oct 2023 15:06:43 -0500 Subject: [PATCH 18/25] RM czm_dcm for now --- oct_converter/readers/czm_dcm_unscrambler.py | 148 ------------------- 1 file changed, 148 deletions(-) delete mode 100644 oct_converter/readers/czm_dcm_unscrambler.py diff --git a/oct_converter/readers/czm_dcm_unscrambler.py b/oct_converter/readers/czm_dcm_unscrambler.py deleted file mode 100644 index 39183d9..0000000 --- a/oct_converter/readers/czm_dcm_unscrambler.py +++ /dev/null @@ -1,148 +0,0 @@ -"""With many, many thanks to scaramallion, https://github.com/pydicom/pydicom/discussions/1618""" -import math - -from pydicom import dcmread -from pydicom.dataelem import DataElement, validate_value -from pydicom.dataset import validate_file_meta -from pydicom.encaps import encapsulate, generate_pixel_data_frame -from pydicom.uid import ExplicitVRLittleEndian, JPEG2000Lossless - - -def unscramble_czm(frame: bytes) -> bytearray: - """Return an unscrambled image frame. - - Parameters - ---------- - frame : bytes - The scrambled CZM JPEG 2000 data frame as found in the DICOM dataset. - - Returns - ------- - bytearray - The unscrambled JPEG 2000 data. - """ - # Fix the 0x5A XORing - frame = bytearray(frame) - for ii in range(0, len(frame), 7): - frame[ii] = frame[ii] ^ 0x5A - - # Offset to the start of the JP2 header - empirically determined - jp2_offset = math.floor(len(frame) / 5 * 3) - - # Double check that our empirically determined jp2_offset is correct - offset = frame.find(b"\x00\x00\x00\x0C") - if offset == -1: - raise ValueError("No JP2 header found in the scrambled pixel data") - - if jp2_offset != offset: - print( - f"JP2 header found at offset {offset} rather than the expected " - f"{jp2_offset}" - ) - jp2_offset = offset - - d = bytearray() - d.extend(frame[jp2_offset : jp2_offset + 253]) - d.extend(frame[993:1016]) - d.extend(frame[276:763]) - d.extend(frame[23:276]) - d.extend(frame[1016:jp2_offset]) - d.extend(frame[:23]) - d.extend(frame[763:993]) - d.extend(frame[jp2_offset + 253 :]) - - assert len(d) == len(frame) - - return d - - -def tag_fixer(element: DataElement) -> DataElement: - """Given a DataElement, attempts to remove the basic - obfuscation added to various tags. If element is valid, - returns element. If element is invalid, empties value - and returns element. - - Args: - element: DICOM tag data as DataElement - - Returns: - DataElement with more-conformant values - """ - try: - element.value = element.value.split("\x00")[0] - except: - pass - try: - validate_value(element.VR, element.value, validation_mode=2) - return element - except ValueError: - element.value = "" - return element - - -def process_file(input_file: str, output_filename: str) -> None: - """Utilizes Pydicom to read the dataset, check that the - dataset is CZM, applies fixers, and outputs a deobfuscated - DICOM. - - Args: - input_file: Path to input file as a string - output_filename: Name under which to save the DICOM - """ - # Read and check the dataset is CZM - ds = dcmread(input_file) - meta = ds.file_meta - if meta.TransferSyntaxUID != JPEG2000Lossless: - raise ValueError( - "Only DICOM datasets with a 'Transfer Syntax UID' of JPEG 2000 " - "(Lossless) are supported" - ) - - if not ds.Manufacturer.startswith("Carl Zeiss Meditec"): - raise ValueError("Only CZM DICOM datasets are supported") - - if "PixelData" not in ds: - raise ValueError("No 'Pixel Data' found in the DICOM dataset") - - # Specific tag fixers - ds.PixelSpacing = ds.PixelSpacing.split("\x00")[0].split(",") - ds.OperatorsName = ds.OperatorsName.original_string.split(b"\x00")[0].decode() - ds.PatientName = ds.PatientName.original_string.split(b"=")[0].decode() - ds.Modality = "OPT" - ds.ImageOrientationPatient = [1, 0, 0, 0, 1, 0] - - lat_map = {"OD": "R", "OS": "L", "": "", None: ""} - ds.Laterality = lat_map.get(ds.Laterality, None) - - # Clean obfuscated tags - for element in ds: - if element.VR not in ["SQ", "OB"]: - tag_fixer(element) - elif element.VR == "SQ": - for sequence in element: - for e in sequence: - tag_fixer(e) - for element in meta: - tag_fixer(element) - - # Make sure file_meta is conformant - validate_file_meta(meta, enforce_standard=True) - - # Iterate through the frames, unscramble and write to file - if "NumberOfFrames" in ds: - all_frames = [] - frames = generate_pixel_data_frame(ds.PixelData, int(ds.NumberOfFrames)) - for idx, frame in enumerate(frames): - all_frames.append(unscramble_czm(frame)) - ds.PixelData = encapsulate(all_frames) - else: - frame = unscramble_czm(ds.PixelData) - ds.PixelData = encapsulate([frame]) - - # And finally, convert pixel data to ExplicitVRLittleEndian - ds.decompress() - meta.TransferSyntaxUID = ExplicitVRLittleEndian - ds.is_implicit_VR = False - ds.is_little_endian = True - - ds.save_as(output_filename) From 4c2380a92e82f51d07bc91a3833ffe59023751a7 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Wed, 25 Oct 2023 15:07:30 -0500 Subject: [PATCH 19/25] Linting --- examples/demo_boct_extraction.py | 2 +- examples/demo_optovue_extraction.py | 2 +- oct_converter/dicom/dicom.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/demo_boct_extraction.py b/examples/demo_boct_extraction.py index 305b52b..5fc5b89 100644 --- a/examples/demo_boct_extraction.py +++ b/examples/demo_boct_extraction.py @@ -21,4 +21,4 @@ # multiple DICOMs will be outputted. # Additionally, diskbuffered can be specified to store # volume on disk using HDF5 to reduce memory usage -dcm = create_dicom_from_oct(filepath, diskbuffered=True) \ No newline at end of file +dcm = create_dicom_from_oct(filepath, diskbuffered=True) diff --git a/examples/demo_optovue_extraction.py b/examples/demo_optovue_extraction.py index 8d8c9a2..b777508 100644 --- a/examples/demo_optovue_extraction.py +++ b/examples/demo_optovue_extraction.py @@ -14,4 +14,4 @@ # Output dir can be specified, otherwise will # default to current working directory. # If multiple volumes are identified within the file, -# multiple DICOMs will be outputted. \ No newline at end of file +# multiple DICOMs will be outputted. diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index 8cdf8e5..c6bc443 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -4,13 +4,13 @@ from pathlib import Path import numpy as np +from construct import StreamError from pydicom.dataset import Dataset, FileDataset, FileMetaDataset from pydicom.uid import ( ExplicitVRLittleEndian, OphthalmicTomographyImageStorage, generate_uid, ) -from construct import StreamError from oct_converter.dicom.boct_meta import boct_dicom_metadata from oct_converter.dicom.e2e_meta import e2e_dicom_metadata From 988ddc889c9222319fe92cc3ad570532709d16b8 Mon Sep 17 00:00:00 2001 From: Mark Graham Date: Fri, 22 Sep 2023 06:39:55 -0600 Subject: [PATCH 20/25] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 42479ca..74b65bc 100644 --- a/README.md +++ b/README.md @@ -54,12 +54,19 @@ fundus_image.save('fds_testing_fundus.jpg') metadata = fds.read_all_metadata(verbose=True) # extracts all other metadata with open("fds_metadata.json", "w") as outfile: outfile.write(json.dumps(metadata, indent=4)) + +# create and save a DICOM (.fda/.fds only for now) +dcm = create_dicom_from_oct(filepath) ``` ## Contributions Are welcome! Here is a [development roadmap](https://github.com/marksgraham/OCT-Converter/issues/86), including some easy first issues. Please open a [new issue](https://github.com/marksgraham/OCT-Converter/issues/new) to discuss any potential contributions. ## Updates + +22 September 2023 +- DICOM support: can now save .fda/.fds files as DICOMs with metadata populated. + 28 March 2023 - Metadata extraction for .fds expanded to match that of .fda file. From 91b1154e162b43bb97e8a24697c72c88ea14b529 Mon Sep 17 00:00:00 2001 From: Mark Graham Date: Fri, 22 Sep 2023 13:43:07 +0100 Subject: [PATCH 21/25] bump version 0.5.12 -> 0.6.0 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 32091eb..c198772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "oct_converter" -version = "0.5.12" +version = "0.6.0" description = "Extract OCT and fundus data from proprietary file formats." readme = "README.md" authors = [{ name = "Mark Graham", email = "markgraham539@gmail.com" }] @@ -39,7 +39,7 @@ include = ["oct_converter*","examples"] exclude= ["my_example_volumes*"] [tool.bumpver] -current_version = "0.5.12" +current_version = "0.6.0" version_pattern = "MAJOR.MINOR.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = true From 599beddc1b3aab8c0c44397f54780e9575e8500a Mon Sep 17 00:00:00 2001 From: Mark Graham Date: Fri, 22 Sep 2023 06:51:56 -0600 Subject: [PATCH 22/25] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 74b65bc..6ed62bf 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,8 @@ Are welcome! Here is a [development roadmap](https://github.com/marksgraham/OCT- ## Updates 22 September 2023 -- DICOM support: can now save .fda/.fds files as DICOMs with metadata populated. +- DICOM support: can now save .fda/.fds files as DICOMs with correct headers. +- Much more complete extraction of .fda/.fds metadata. 28 March 2023 - Metadata extraction for .fds expanded to match that of .fda file. From 8db09ff243cf945dc7cc2710d2a71b6c78b83d6c Mon Sep 17 00:00:00 2001 From: Mark Graham Date: Mon, 23 Oct 2023 09:21:10 -0600 Subject: [PATCH 23/25] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ed62bf..b9c870b 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ Are welcome! Here is a [development roadmap](https://github.com/marksgraham/OCT- - More patient info extracted from .e2e files (name, sex, birthdate, patient ID). 24 Aug 2021 -- Reading the Bioptigen .OCT format is now supported/ +- Reading the Bioptigen .OCT format is now supported. 11 June 2021 - Can now specify whether Zeiss .img data needs to be de-interlaced during reading. From 34cedeb74c122ad2f0c346da380fb2a9cb29de45 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Wed, 25 Oct 2023 15:54:50 -0500 Subject: [PATCH 24/25] RM TODO that got done --- oct_converter/readers/boct.py | 1 - 1 file changed, 1 deletion(-) diff --git a/oct_converter/readers/boct.py b/oct_converter/readers/boct.py index 30e4710..6a13f2d 100644 --- a/oct_converter/readers/boct.py +++ b/oct_converter/readers/boct.py @@ -28,7 +28,6 @@ class BOCT(object): bioptigen_scan_type_map = {0: "linear", 1: "rect", 3: "rad"} file_structure = boct_binary.bioptigen_file_structure - # TODO this should contain the datetimes header_structure = boct_binary.bioptigen_oct_header_struct def __init__(self, filepath: Path | str) -> None: From a2a90c183b2e5be490dfe8a8db8d302616d3c3a2 Mon Sep 17 00:00:00 2001 From: Susannah Trevino Date: Mon, 6 Nov 2023 08:33:13 -0600 Subject: [PATCH 25/25] PR touchups --- oct_converter/dicom/dicom.py | 2 +- oct_converter/dicom/e2e_meta.py | 2 ++ oct_converter/dicom/img_meta.py | 2 ++ oct_converter/readers/img.py | 8 ++++---- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/oct_converter/dicom/dicom.py b/oct_converter/dicom/dicom.py index c6bc443..8fd1345 100644 --- a/oct_converter/dicom/dicom.py +++ b/oct_converter/dicom/dicom.py @@ -22,7 +22,7 @@ from oct_converter.exceptions import InvalidOCTReaderError from oct_converter.readers import BOCT, E2E, FDA, FDS, IMG, POCT -# Deterministic implentation UID based on package name and version +# Deterministic implementation UID based on package name and version version = metadata.version("oct_converter") implementation_uid = generate_uid(entropy_srcs=["oct_converter", version]) diff --git a/oct_converter/dicom/e2e_meta.py b/oct_converter/dicom/e2e_meta.py index a791c34..8c3de76 100644 --- a/oct_converter/dicom/e2e_meta.py +++ b/oct_converter/dicom/e2e_meta.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from oct_converter.dicom.metadata import ( DicomMetadata, ImageGeometry, diff --git a/oct_converter/dicom/img_meta.py b/oct_converter/dicom/img_meta.py index 0e5c7ef..918faa5 100644 --- a/oct_converter/dicom/img_meta.py +++ b/oct_converter/dicom/img_meta.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import datetime from oct_converter.dicom.metadata import ( diff --git a/oct_converter/readers/img.py b/oct_converter/readers/img.py index d307e51..c14a0d0 100644 --- a/oct_converter/readers/img.py +++ b/oct_converter/readers/img.py @@ -54,8 +54,8 @@ def read_oct_volume( oct_volume = OCTVolumeWithMetaData( [volume[:, :, i] for i in range(volume.shape[2])], patient_id=meta.get("patient_id"), - acquisition_date=meta.get("acq"), - laterality=lat_map[meta.get("lat", None)], + acquisition_date=meta.get("acquisition_date"), + laterality=lat_map[meta.get("laterality", None)], metadata=meta, ) return oct_volume @@ -87,7 +87,7 @@ def get_metadata_from_filename(self) -> dict: else None ) if acq: - meta["acq"] = datetime( + meta["acquisition_date"] = datetime( year=int(acq[2]), month=int(acq[0]), day=int(acq[1]), @@ -95,7 +95,7 @@ def get_metadata_from_filename(self) -> dict: minute=int(acq[4]), second=int(acq[5]), ) - meta["lat"] = ( + meta["laterality"] = ( re.search(r"O[D|S]", filename).group(0) if re.search(r"O[D|S]", filename) else None