setup: add setup.py in preparation for publishing CHAP as a package.

Move package code to the CHAP/ directory.
Add a wrapper script for runner.py that setup.py will install in the bin area.
Organize example pipeline files in the examples/ directory.
Remove redundant files already contained in test-data/.
Remove environment.yml which does not represent the most generic run environment for CHAP.

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# text editor backups
+*~

CHAP/__main__.py

@@ -0,0 +1,4 @@
+from CHAP.runner import main
+
+if __name__ == '__main__':
+    main()

async.py → CHAP/async.py

@@ -11,7 +11,7 @@
 import asyncio
 
 # local modules
-from processor import Processor, PrintProcessor
+from CHAP.processor import Processor, PrintProcessor
 
 
 async def task(mgr, doc):

models/integration.py → CHAP/models/integration.py

@@ -6,13 +6,12 @@
 from time import time
 from typing import Literal, Optional
 
-from msnctools.general import input_menu
-from multiprocessing.pool import ThreadPool
-from nexusformat.nexus import (NXdata,
-                               NXdetector,
-                               NXfield,
-                               NXprocess,
-                               NXroot)
+# from multiprocessing.pool import ThreadPool
+# from nexusformat.nexus import (NXdata,
+#                                NXdetector,
+#                                NXfield,
+#                                NXprocess,
+#                                NXroot)
 import numpy as np
 from pydantic import (BaseModel,
                       validator,
@@ -21,10 +20,13 @@
                       conint,
                       confloat,
                       FilePath)
-import pyFAI, pyFAI.multi_geometry, pyFAI.units
-from pyspec.file.tiff import TiffFile
+#import pyFAI, pyFAI.multi_geometry, pyFAI.units
+from pyFAI import load as pyfai_load
+from pyFAI.multi_geometry import MultiGeometry
+from pyFAI.units import AZIMUTHAL_UNITS, RADIAL_UNITS
+#from pyspec.file.tiff import TiffFile
 
-from .map import MapConfig, SpecScans
+#from .map import MapConfig, SpecScans
 
 
 class Detector(BaseModel):
@@ -98,12 +100,14 @@ def mask_array(self):
 def azimuthal_integrator(poni_file:str):
     if not isinstance(poni_file, str):
         poni_file = str(poni_file)
-    return(pyFAI.load(poni_file))
+    return(pyfai_load(poni_file))
 @cache
 def get_mask_array(mask_file:str, poni_file:str):
     if mask_file is not None:
         if not isinstance(mask_file, str):
             mask_file = str(mask_file)
+
+        from pyspec.file.tiff import TiffFile
         with TiffFile(mask_file) as tiff:
             mask_array = tiff.asarray()
     else:
@@ -166,10 +170,10 @@ def validate_radial_units(cls, radial_units):
         :return: validated radial units
         :rtype: str
         """
-        if radial_units in pyFAI.units.RADIAL_UNITS.keys():
+        if radial_units in RADIAL_UNITS.keys():
             return(radial_units)
         else:
-            raise(ValueError(f'Invalid radial units: {radial_units}. Must be one of {", ".join(pyFAI.units.RADIAL_UNITS.keys())}'))
+            raise(ValueError(f'Invalid radial units: {radial_units}. Must be one of {", ".join(RADIAL_UNITS.keys())}'))
     @validator('azimuthal_units', allow_reuse=True)
     def validate_azimuthal_units(cls, azimuthal_units):
         """
@@ -182,10 +186,10 @@ def validate_azimuthal_units(cls, azimuthal_units):
         :return: The original supplied value, if is one of the keys in `pyFAI.units.AZIMUTHAL_UNITS`.
         :rtype: str
         """
-        if azimuthal_units in pyFAI.units.AZIMUTHAL_UNITS.keys():
+        if azimuthal_units in AZIMUTHAL_UNITS.keys():
             return(azimuthal_units)
         else:
-            raise(ValueError(f'Invalid azimuthal units: {azimuthal_units}. Must be one of {", ".join(pyFAI.units.AZIMUTHAL_UNITS.keys())}'))
+            raise(ValueError(f'Invalid azimuthal units: {azimuthal_units}. Must be one of {", ".join(AZIMUTHAL_UNITS.keys())}'))
     def validate_range_max(range_name:str):
         """Validate the maximum value of an integration range.
 
@@ -213,7 +217,7 @@ def _validate_range_max(cls, range_max, values):
         return(_validate_range_max)
     _validate_radial_max = validator('radial_max', allow_reuse=True)(validate_range_max('radial'))
     _validate_azimuthal_max = validator('azimuthal_max', allow_reuse=True)(validate_range_max('azimuthal'))        
-    def validate_for_map_config(self, map_config:MapConfig):
+    def validate_for_map_config(self, map_config:BaseModel):
         """
         Validate the existence of the detector data file for all scan points in `map_config`.
         
@@ -269,7 +273,7 @@ def get_multi_geometry_integrator(self):
         radial_range = (self.radial_min, self.radial_max)
         azimuthal_range = (self.azimuthal_min, self.azimuthal_max)
         return(get_multi_geometry_integrator(poni_files, self.radial_units, radial_range, azimuthal_range))
-    def get_azimuthally_integrated_data(self, spec_scans:SpecScans, scan_number:int, scan_step_index:int):
+    def get_azimuthally_integrated_data(self, spec_scans:BaseModel, scan_number:int, scan_step_index:int):
         """Return azimuthally-integrated data for the scan step specified.
         
         :param spec_scans: An instance of `SpecScans` containing the scan step requested.
@@ -289,7 +293,7 @@ def get_azimuthally_integrated_data(self, spec_scans:SpecScans, scan_number:int,
             return(result.intensity)
         else:
             return(result.intensity, result.sigma)
-    def get_radially_integrated_data(self, spec_scans:SpecScans, scan_number:int, scan_step_index:int):
+    def get_radially_integrated_data(self, spec_scans:BaseModel, scan_number:int, scan_step_index:int):
         """Return radially-integrated data for the scan step specified.
         
         :param spec_scans: An instance of `SpecScans` containing the scan step requested.
@@ -327,7 +331,7 @@ def get_radially_integrated_data(self, spec_scans:SpecScans, scan_number:int, sc
             # Get the standard deviation of the summed detectors' intensities
             sigma = np.sqrt(np.nansum(variance_each_detector, axis=0))
             return(I, sigma)
-    def get_cake_integrated_data(self, spec_scans:SpecScans, scan_number:int, scan_step_index:int):
+    def get_cake_integrated_data(self, spec_scans:BaseModel, scan_number:int, scan_step_index:int):
         """Return cake-integrated data for the scan step specified.
         
         :param spec_scans: An instance of `SpecScans` containing the scan step requested.
@@ -350,7 +354,7 @@ def get_cake_integrated_data(self, spec_scans:SpecScans, scan_number:int, scan_s
             return(result.intensity)
         else:
             return(result.intensity, result.sigma)
-    def get_integrated_data(self, spec_scans:SpecScans, scan_number:int, scan_step_index:int):
+    def get_integrated_data(self, spec_scans:BaseModel, scan_number:int, scan_step_index:int):
         """Return integrated data for the scan step specified.
         
         :param spec_scans: An instance of `SpecScans` containing the scan step requested.
@@ -471,12 +475,12 @@ def get_multi_geometry_integrator(poni_files:tuple, radial_unit:str, radial_rang
     """
     chi_min, chi_max, chi_offset, chi_disc = get_azimuthal_adjustments(*azimuthal_range)
     ais = copy.deepcopy(get_azimuthal_integrators(poni_files, chi_offset=chi_offset))
-    multi_geometry = pyFAI.multi_geometry.MultiGeometry(ais,
-                                                        unit=radial_unit,
-                                                        radial_range=radial_range,
-                                                        azimuth_range=(chi_min,chi_max),
-                                                        wavelength=sum([ai.wavelength for ai in ais])/len(ais),
-                                                        chi_disc=chi_disc)
+    multi_geometry = MultiGeometry(ais,
+                                   unit=radial_unit,
+                                   radial_range=radial_range,
+                                   azimuth_range=(chi_min,chi_max),
+                                   wavelength=sum([ai.wavelength for ai in ais])/len(ais),
+                                   chi_disc=chi_disc)
     return(multi_geometry)
 @cache
 def get_integrated_data_coordinates(azimuthal_range:tuple=None, azimuthal_npt:int=None, radial_range:tuple=None, radial_npt:int=None):

processor.py → CHAP/processor.py

@@ -312,7 +312,7 @@ def get_map_config(self, data):
         :rtype: MapConfig
         '''
 
-        from models.map import MapConfig
+        from CHAP.models.map import MapConfig
 
         map_config = False
         if isinstance(data, list):
@@ -461,8 +461,8 @@ def get_configs(self, data):
         self.logger.debug('Getting configuration objects')
         t0 = time()
 
-        from models.map import MapConfig
-        from models.integration import IntegrationConfig
+        from CHAP.models.map import MapConfig
+        from CHAP.models.integration import IntegrationConfig
 
         map_config = False
         integration_config = False
@@ -635,7 +635,7 @@ def get_config(self, data):
         :rtype: MCACeriaCalibrationConfig
         '''
 
-        from models.edd import MCACeriaCalibrationConfig
+        from CHAP.models.edd import MCACeriaCalibrationConfig
 
         calibration_config = False
         if isinstance(data, list):
@@ -787,8 +787,8 @@ def get_configs(self, data):
         :rtype: tuple[MapConfig, MCACeriaCalibrationConfig]
         '''
 
-        from models.map import MapConfig
-        from models.edd import MCACeriaCalibrationConfig
+        from CHAP.models.map import MapConfig
+        from CHAP.models.edd import MCACeriaCalibrationConfig
 
         map_config = False
         calibration_config = False

runner.py → CHAP/runner.py

@@ -15,7 +15,7 @@
 import yaml
 
 # local modules
-from pipeline import Pipeline
+from CHAP.pipeline import Pipeline
 
 
 class OptionParser():
@@ -63,7 +63,7 @@ def runner(opts):
             name = item
             kwargs = {}
         modName, clsName = name.split('.')
-        module = __import__(modName)
+        module = __import__(f'CHAP.{modName}', fromlist=[clsName])
         obj = getattr(module, clsName)()
         obj.logger.setLevel(log_level)
         obj.logger.addHandler(log_handler)

edd/ceria_calibration_config.yaml → examples/edd/ceria_calibration_config.yaml

@@ -1,13 +1,13 @@
-spec_file: edd/ceria_2222-2/spec.log
+spec_file: examples/edd/ceria_2222-2/spec.log
 scan_number: 1
 
-flux_file: edd/flux.dft
+flux_file: examples/edd/flux.dft
 
 detector_name: mca1
 num_bins: 2048
 max_energy_kev: 150
 
-hexrd_h5_material_file: edd/materials.h5
+hexrd_h5_material_file: examples/edd/materials.h5
 
 tth_max: 90.0
 hkl_tth_tol: 0.15

edd/map.yaml → examples/edd/map.yaml

@@ -4,7 +4,7 @@ experiment_type: EDD
 sample:
   name: set2_c1-1
 spec_scans:
-  - spec_file: edd/set2_c1-1/spec.log
+  - spec_file: examples/edd/set2_c1-1/spec.log
     scan_numbers: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
 independent_dimensions:
   - label: sample_y

edd/pipeline.yaml → examples/edd/pipeline.yaml

@@ -7,41 +7,41 @@ pipeline:
         PRIVATE-TOKEN: # your token here
   - processor.URLResponseProcessor
   - writer.ExtractArchiveWriter:
-      filename: edd
+      filename: examples/edd
 
   # Calibrate detector
   - reader.YAMLReader:
-      filename: edd/ceria_calibration_config.yaml
+      filename: examples/edd/ceria_calibration_config.yaml
       schema: MCACeriaCalibrationConfig
   - processor.MCACeriaCalibrationProcessor
   - writer.YAMLWriter:
-      filename: edd/ceria_calibrated.yaml
+      filename: examples/edd/ceria_calibrated.yaml
       force_overwrite: true
 
   # Gather calibrated detector data
   - reader.MultipleReader:
       readers:
         - YAMLReader:
-            filename: edd/map.yaml
+            filename: examples/edd/map.yaml
             schema: MapConfig
         - YAMLReader:
-            filename: edd/ceria_calibrated.yaml
+            filename: examples/edd/ceria_calibrated.yaml
             schema: MCACeriaCalibrationConfig
   - processor.MCADataProcessor
   - writer.NexusWriter:
-      filename: edd/map_detector_data.nxs
+      filename: examples/edd/map_detector_data.nxs
       force_overwrite: true
 
   # Compute sample strain map
   - reader.MultipleReader:
       readers:
+        - NexusReader:
+            filename: examples/edd/map_detector_data.nxs
         - YAMLReader:
-            filename: edd/map_detector_data.nxs
-        - YAMLReader:
-            filename: edd/strain_analysis_config.yaml
+            filename: examples/edd/strain_analysis_config.yaml
             schema: StrainAnalysisConfig
   - processor.StrainAnalysisProcessor
   - writer.YAMLWriter:
-      filename: edd/map_strain_data.yaml
+      filename: examples/edd/map_strain_data.yaml
       force_overwrite: true
 

edd/strain_analysis_config.yaml → examples/edd/strain_analysis_config.yaml

@@ -3,7 +3,7 @@ energy_range:
 - 112
 lattice_parameters:
 - 5.41153
-material_file: materials.h5
+material_file: examples/edd/materials.h5
 material_name: CeO2
 selected_peaks:
 - 3

inference/pipeline.yaml → examples/inference/pipeline.yaml

@@ -6,4 +6,4 @@ pipeline:
       model: mnist
       verbose: true
   - writer.Writer:
-      filename: predictions.json
+      filename: examples/inference/predictions.json

saxswaxs/integration_saxs_azimuthal.yaml → examples/saxswaxs/integration_saxs_azimuthal.yaml

@@ -3,8 +3,8 @@ title: saxs_azimuthal
 integration_type: azimuthal
 detectors:
 - prefix: PIL5
-  poni_file: saxswaxs/PIL5.poni
-  mask_file: saxswaxs/PIL5.tif
+  poni_file: examples/saxswaxs/PIL5.poni
+  mask_file: examples/saxswaxs/PIL5.tif
 radial_units: q_A^-1
 radial_min: 0.0
 radial_max: 0.21821

saxswaxs/integration_waxs_azimuthal.yaml → examples/saxswaxs/integration_waxs_azimuthal.yaml

@@ -3,11 +3,11 @@ title: waxs_azimuthal
 integration_type: azimuthal
 detectors:
 - prefix: PIL9
-  poni_file: saxswaxs/PIL9.poni
-  mask_file: saxswaxs/PIL9.tif
+  poni_file: examples/saxswaxs/PIL9.poni
+  mask_file: examples/saxswaxs/PIL9.tif
 - prefix: PIL11
-  poni_file: saxswaxs/PIL11.poni
-  mask_file: saxswaxs/PIL11.tif
+  poni_file: examples/saxswaxs/PIL11.poni
+  mask_file: examples/saxswaxs/PIL11.tif
 radial_units: q_A^-1
 radial_min: 0.0
 radial_max: 3.33209

saxswaxs/map_1d.yaml → examples/saxswaxs/map_1d.yaml

@@ -4,7 +4,7 @@ experiment_type: SAXSWAXS
 sample:
   name: sample_14_align
 spec_scans:
-- spec_file: saxswaxs/test_1d
+- spec_file: examples/saxswaxs/test_1d
   scan_numbers:
   - 1
 independent_dimensions:

saxswaxs/map_2d.yaml → examples/saxswaxs/map_2d.yaml

@@ -4,7 +4,7 @@ experiment_type: SAXSWAXS
 sample:
   name: sample_14
 spec_scans:
-- spec_file: saxswaxs/test_2d
+- spec_file: examples/saxswaxs/test_2d
   scan_numbers:
   - 1
 independent_dimensions: