Merge branch 'main' into main

.gitignore

@@ -113,3 +113,8 @@ test_scripts/
 /SciFiReaders/readers/microscopy/spm/afm/mdt_reader/nt_mdt.svg
 /SciFiReaders/readers/microscopy/spm/afm/mdt_reader/MDTdeclaration.py
 /SciFiReaders/readers/microscopy/spm/afm/mdt_reader/MDTfile.py
+.pyTEMlib.files.pkl
+tests/readers/microscopy/em/tem/EMDReader_Spectrum_FEI.emd
+tests/readers/converters/server_user_id.txt
+tests/readers/microscopy/em/tem/DMReader_Image_SI-Survey.dm3
+tests/readers/microscopy/em/tem/server_user_id.txt

README.rst

@@ -26,6 +26,6 @@ SciFiReaders
     :alt: Coverage
 
 Tools for extracting data and metadata from scientific data files.
-Extracted information are returned as `sidpy.Dataset <https://pycroscopy.github.io/sidpy/_autosummary/sidpy.sid.dataset.Dataset.html#sidpy.sid.dataset.Dataset>`_ objects.
+Extracted information are returned as a dictionary of `sidpy.Dataset <https://pycroscopy.github.io/sidpy/_autosummary/sidpy.sid.dataset.Dataset.html#sidpy.sid.dataset.Dataset>`_ objects.
 
 Please see `SciFiReaders documentation website <https://pycroscopy.github.io/SciFiReaders/index.html>`_ for more information.

SciFiReaders/readers/microscopy/em/tem/dm_reader.py

@@ -126,13 +126,21 @@ def read_string(dm_file, length=1):
 
 class DMReader(sidpy.Reader):
     """
-    file_path: filepath to dm3 or dm4 file.
+    Reader of Digital Micrograph image and spectroscopy data
 
-    warn('This Reader will eventually be moved to the ScopeReaders package'
-         '. Be prepared to change your import statements',
-         FutureWarning)
-    """
+    This reader reads (attribute read) all the different data in the file and returns it as a dictionary 
+    of sidpy.Datasets
+
+    Parameter:
+    ---------
+    file_path: str
+        filepath to dm3 or dm4 file.
 
+    Return:
+    ------
+    datasets: dict
+        dictionary of sidpy datasets
+    """
     def __init__(self, file_path, verbose=False):
         super().__init__(file_path)
 
@@ -227,7 +235,8 @@ def read(self):
             print(key, dataset)
             if 'urvey' in dataset.title:
                 main_dataset_key = key
-        print(main_dataset_key)
+        if self.verbose:
+            print(main_dataset_key)
         self.datasets[main_dataset_key].original_metadata.update(self.__stored_tags)
         self.close()
         return self.datasets

SciFiReaders/readers/microscopy/em/tem/edax_reader.py

@@ -135,6 +135,22 @@ def read_image(base_group, dataset_item):
     return dataset
 
 class EDAXReader(sidpy.Reader):
+    """
+    Creates an instance of EDAXReader which can read one or more HDF5
+    datasets formatted in the EDAX format
+
+    We can read Images, and SpectrumStreams (SpectrumImages and Spectra).
+    Please note that all original metadata are retained in each sidpy dataset.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to a EDAX file
+    Return
+    ------
+    datasets: dict
+        dictionary of sidpy.Datasets
+    """
 
     def __init__(self, file_path, verbose=False):
         """

SciFiReaders/readers/microscopy/em/tem/emd_reader.py

@@ -35,26 +35,32 @@
 
 
 class EMDReader(sidpy.Reader):
+
+    """
+    Creates an instance of EMDReader which can read one or more HDF5
+    datasets formatted in the FEI Velox style EDM file
+
+    We can read Images, and SpectrumStreams (SpectrumImages and Spectra).
+    Please note that all original metadata are retained in each sidpy dataset.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to a HDF5 file
+    Return
+    ------
+    datasets: dict
+        dictionary of sidpy.Datasets
+    """
     def __init__(self, file_path):
-        """
-        Creates an instance of EMDReader which can read one or more HDF5
-        datasets formatted in the FEI Velox style EDM file
-
-        We can read Images, and SpectrumStreams (SpectrumImages and Spectra).
-        Please note that all original metadata are retained in each sidpy dataset.
-
-        Parameters
-        ----------
-        file_path : str
-            Path to a HDF5 file
-        """
-
         super(EMDReader, self).__init__(file_path)
 
         # Let h5py raise an OS error if a non-HDF5 file was provided
         self._h5_file = h5py.File(file_path, mode='r+')
 
-        self.datasets = []
+        self.datasets = {}
+        self.channel_number = 0
+        self.key = f"Channel_{int(self.channel_number):03d}"
         self.data_array = None
         self.metadata = None
         self.label_dict = {}
@@ -94,13 +100,13 @@ def read(self, eds_stream=False):
         if 'Data' not in self._h5_file:
             raise TypeError('Velox EMD File is empty')
 
-        number_of_datasets=0
+        number_of_datasets = 0
         use_tqdm = False
         for key in self._h5_file['Data']:
             if key == 'SpectrumStream':
                 number_of_datasets += len(self._h5_file['Data']['SpectrumStream'].keys())
         if number_of_datasets > 1:
-            progress_bar = tqdm(total=number_of_datasets) # Initialise
+            progress_bar = tqdm(total=number_of_datasets)  # Initialise
             use_tqdm = tqdm_available
         for key in self._h5_file['Data']:
             self.image_key = 'None'
@@ -109,12 +115,12 @@ def read(self, eds_stream=False):
                 for self.image_key in self._h5_file['Data']['Image']:
                     self.get_data('Data/Image/' + self.image_key)
                     self.get_image()
-                    self.extract_crucial_metadata(-1)
+                    self.extract_crucial_metadata(self.key)
             elif key == 'SpectrumStream':
                 for stream_key in self._h5_file['Data']['SpectrumStream']:
                     self.get_data('Data/SpectrumStream/' + stream_key)
                     self.get_eds(eds_stream)
-                    self.extract_crucial_metadata(-1)
+                    self.extract_crucial_metadata(self.key)
                     if use_tqdm:
                         progress_bar.update(1)
         if use_tqdm:
@@ -143,12 +149,15 @@ def _parse_image_display(self):
     def get_eds(self, eds_stream=False):
         if 'AcquisitionSettings' not in self.metadata:
             eds_stream = True
+        key = f"Channel_{int(self.channel_number):03d}"
+        self.key = key
+        self.channel_number += 1
         if eds_stream:
-            self.datasets.append(sidpy.Dataset.from_array(self.data_array),)
+            self.datasets[key] = sidpy.Dataset.from_array(self.data_array)
         else:
             data_array = self.get_eds_spectrum()
             if data_array.shape[0] == 1 and data_array.shape[1] == 1:
-                data_array = np.array(data_array).flatten()
+                data_array = np.squeeze(data_array)
                 chunks = 1
             else:
                 chunks= [32, 32, data_array.shape[2]]
@@ -157,14 +166,14 @@ def get_eds(self, eds_stream=False):
                 if data_array.shape[1]> chunks[1]:
                     chunks[1] = data_array.shape[1]
 
-            self.datasets.append(sidpy.Dataset.from_array(data_array,  chunks=chunks))
-        # print(self.datasets[-1])
-
+            self.datasets[key] = sidpy.Dataset.from_array(data_array, chunks=chunks)
+            
+       
         self.data_array=np.zeros([1,1])
 
-        self.datasets[-1].original_metadata = self.metadata
+        self.datasets[key].original_metadata = self.metadata
 
-        detectors = self.datasets[-1].original_metadata['Detectors']
+        detectors = self.datasets[key].original_metadata['Detectors']
         if eds_stream:
             pass
         else:
@@ -177,39 +186,42 @@ def get_eds(self, eds_stream=False):
                     if 'Dispersion' in detector:
                         dispersion = float(detector['Dispersion'])
 
-            self.datasets[-1].units = 'counts'
-            self.datasets[-1].quantity = 'intensity'
-            energy_scale = np.arange(self.datasets[-1].shape[-1]) * dispersion + offset
+            self.datasets[key].units = 'counts'
+            self.datasets[key].quantity = 'intensity'
+            energy_scale = np.arange(self.datasets[key].shape[-1]) * dispersion + offset
 
-            if self.datasets[-1].ndim == 1:
-                self.datasets[-1].data_type = 'spectrum'
+            if self.datasets[key].ndim == 1:
+                self.datasets[key].data_type = 'spectrum'
 
-                self.datasets[-1].set_dimension(0, sidpy.Dimension(energy_scale,
+                self.datasets[key].set_dimension(0, sidpy.Dimension(energy_scale,
                                                                    name='energy_scale', units='eV',
                                                                    quantity='energy',
                                                                    dimension_type='spectral'))
 
             else:
-                self.datasets[-1].data_type = 'spectral_image'
-                self.datasets[-1].set_dimension(2, sidpy.Dimension(energy_scale,
-                                                                   name='energy_scale', units='eV',
-                                                                   quantity='energy',
-                                                                   dimension_type='spectral'))
+                self.datasets[key].data_type = 'spectral_image'
+                print(self.datasets[key].shape)
+
                 scale_x = float(self.metadata['BinaryResult']['PixelSize']['width']) * 1e9
                 scale_y = float(self.metadata['BinaryResult']['PixelSize']['height']) * 1e9
 
-                self.datasets[-1].set_dimension(0, sidpy.Dimension(np.arange(self.datasets[-1].shape[0]) * scale_x,
+                self.datasets[key].set_dimension(0, sidpy.Dimension(np.arange(self.datasets[key].shape[0]) * scale_x,
                                                                    name='x', units='nm',
                                                                    quantity='distance',
                                                                    dimension_type='spatial'))
-                self.datasets[-1].set_dimension(1, sidpy.Dimension(np.arange(self.datasets[-1].shape[1]) * scale_y,
+                self.datasets[key].set_dimension(1, sidpy.Dimension(np.arange(self.datasets[key].shape[1]) * scale_y,
                                                                    name='y', units='nm',
                                                                    quantity='distance',
                                                                    dimension_type='spatial'))
+                self.datasets[key].set_dimension(2, sidpy.Dimension(energy_scale,
+                                                                   name='energy_scale', units='eV',
+                                                                   quantity='energy',
+                                                                   dimension_type='spectral'))
+
 
     def get_eds_spectrum(self):
         acquisition = self.metadata['AcquisitionSettings']
-        # print(acquisition)
+
         size_x = 1
         size_y = 1
         if 'Scan' in self.metadata:
@@ -234,18 +246,21 @@ def get_eds_spectrum(self):
         return np.reshape(data, (size_x, size_y, spectrum_size))
 
     def get_image(self):
+        key = f"Channel_{int(self.channel_number):03d}"
+        self.key = key
+        self.channel_number += 1
 
         scale_x = float(self.metadata['BinaryResult']['PixelSize']['width']) * 1e9
         scale_y = float(self.metadata['BinaryResult']['PixelSize']['height']) * 1e9
 
         if self.data_array.shape[2] == 1:
-            self.datasets.append(sidpy.Dataset.from_array(self.data_array[:, :, 0]))
-            self.datasets[-1].data_type = 'image'
-            self.datasets[-1].set_dimension(0, sidpy.Dimension(np.arange(self.data_array.shape[0]) * scale_x,
+            self.datasets[key] = sidpy.Dataset.from_array(self.data_array[:, :, 0])
+            self.datasets[key].data_type = 'image'
+            self.datasets[key].set_dimension(0, sidpy.Dimension(np.arange(self.data_array.shape[0]) * scale_x,
                                                                name='x', units='nm',
                                                                quantity='distance',
                                                                dimension_type='spatial'))
-            self.datasets[-1].set_dimension(1, sidpy.Dimension(np.arange(self.data_array.shape[1]) * scale_y,
+            self.datasets[key].set_dimension(1, sidpy.Dimension(np.arange(self.data_array.shape[1]) * scale_y,
                                                                name='y', units='nm',
                                                                quantity='distance',
                                                                dimension_type='spatial'))
@@ -258,29 +273,29 @@ def get_image(self):
             self.data_array = np.rollaxis(data_array, axis=2)
             # np.moveaxis(data_array, source=[0, 1, 2], destination=[2, 0, 1])
 
-            self.datasets.append(sidpy.Dataset.from_array(self.data_array))
-            self.datasets[-1].data_type = 'image_stack'
-            self.datasets[-1].set_dimension(0, sidpy.Dimension(np.arange(self.data_array.shape[0]),
+            self.datasets[key] = sidpy.Dataset.from_array(self.data_array)
+            self.datasets[key].data_type = 'image_stack'
+            self.datasets[key].set_dimension(0, sidpy.Dimension(np.arange(self.data_array.shape[0]),
                                                                name='frame', units='frame',
                                                                quantity='time',
                                                                dimension_type='temporal'))
-            self.datasets[-1].set_dimension(1, sidpy.Dimension(np.arange(self.data_array.shape[1]) * scale_x,
+            self.datasets[key].set_dimension(1, sidpy.Dimension(np.arange(self.data_array.shape[1]) * scale_x,
                                                                name='x', units='nm',
                                                                quantity='distance',
                                                                dimension_type='spatial'))
-            self.datasets[-1].set_dimension(2, sidpy.Dimension(np.arange(self.data_array.shape[2]) * scale_y,
+            self.datasets[key].set_dimension(2, sidpy.Dimension(np.arange(self.data_array.shape[2]) * scale_y,
                                                                name='y', units='nm',
                                                                quantity='distance',
                                                                dimension_type='spatial'))
-        self.datasets[-1].original_metadata = self.metadata
+        self.datasets[key].original_metadata = self.metadata
 
-        self.datasets[-1].units = 'counts'
-        self.datasets[-1].quantity = 'intensity'
+        self.datasets[key].units = 'counts'
+        self.datasets[key].quantity = 'intensity'
         if self.image_key in self.label_dict:
-            self.datasets[-1].title = self.label_dict[self.image_key]
+            self.datasets[key].title = self.label_dict[self.image_key]
 
-    def extract_crucial_metadata(self, index):
-        metadata = self.datasets[index].original_metadata
+    def extract_crucial_metadata(self, key):
+        metadata = self.datasets[key].original_metadata
         experiment = {'detector': metadata['BinaryResult']['Detector'],
                       'acceleration_voltage': float(metadata['Optics']['AccelerationVoltage']),
                       'microscope': metadata['Instrument']['InstrumentClass'],
@@ -300,9 +315,9 @@ def extract_crucial_metadata(self, index):
                                "tilt": {"alpha": float(metadata['Stage']['AlphaTilt']),
                                         "beta": float(metadata['Stage']['BetaTilt'])}}
 
-        self.datasets[index].metadata['experiment'] = experiment
-        if self.datasets[index].title == 'generic':
-            self.datasets[index].title = experiment['detector']
+        self.datasets[key].metadata['experiment'] = experiment
+        if self.datasets[key].title == 'generic':
+            self.datasets[key].title = experiment['detector']
 
     def close(self):
         self._h5_file.close()

SciFiReaders/readers/microscopy/em/tem/nion_reader.py

@@ -92,12 +92,22 @@ def parse_zip(fp):
 
 
 class NionReader(sidpy.Reader):
+    """
+    Reader for Nion ndata and h5 files
 
-    def __init__(self, file_path, verbose=False):
-        """
-        file_path: filepath to dm3 file.
-        """
+    Parameter:
+    ---------
+    file_path: str
+        filepath to Nion file.
+
+    Return:
+    ------
+    datasets: dict
+        dictionary of sidpy datasets
+    """
 
+    def __init__(self, file_path, verbose=False):
+
         super().__init__(file_path)
 
         # initialize variables ##
@@ -222,7 +232,7 @@ def read(self):
         dataset.modality = 'STEM data'
         dataset.h5_dataset = None
 
-        return dataset
+        return {'Channel_000': dataset}
 
     def set_data_type(self, dataset):
 

notebooks/01_using_readers/read_dm3_files.ipynb

@@ -790,7 +790,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.6"
+   "version": "3.11.7"
   },
   "toc": {
    "base_numbering": 1,