Merge pull request #273 from scipion-em/devel

3.5.0

.github/workflows/python-package.yml

@@ -1,48 +1,14 @@
-# Workflow to send master to pypi and tag  the branch:
-# You need to edit FOLDER_WITH_VERSION with the folder that has the __version__ value. 
-
+# Workflow to send master to pypi and tag the branch
 name: master to pypi with comments and tag
 
-
-# Controls when the action will run. Triggers the workflow on push or pull request
-# events but only for the master branch
+# Triggers the workflow on push to the master branch
 on:
   push:
     branches: [ master ]
 
-env:
-  FOLDER_WITH_VERSION: imod
-# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  deploy:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up MPI
-      uses: mpi4py/setup-mpi@v1
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.8'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine 
-        pip install scipion-pyworkflow 
-        pip install scipion-em
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist
-        twine upload dist/* -c "${{ secrets.PYPI_COMMENT }}"
-    - name: Get version and tag
-      env:
-        FOLDER_WITH_VERSION: imod
-      run: |
-        export PACKAGE_VERSION=$(python -c "import $FOLDER_WITH_VERSION; print('VERSION', 'v'+$FOLDER_WITH_VERSION.__version__)" | grep VERSION | sed "s/VERSION //g")
-        git tag $PACKAGE_VERSION
-        git push origin $PACKAGE_VERSION
+  call-publish-workflow:
+    uses: scipion-em/.github/.github/workflows/publish_and_tag.yml@master
+    with:
+      folder: imod
+    secrets: inherit

CHANGES.txt

@@ -1,3 +1,19 @@
+3.5.0:
+ Users:
+  - Protocols' help added.
+  - Help of the parameters was enhanced.
+  - New feature: patch tracking alignment added.
+  - Trimming options in fiducial model added
+  - The order of the parameters in the forms was slightly modified according to the user actions.
+  - Protocol CTF correction:
+    * Now it can deal with excluded views in the CTF and/or in the TS.
+    * The same when entering with interpolated or not aligned TS.
+    * The xf, defocus, and tlt files are generated inside each TS extra sub-directory.
+ Developers:
+  - Update the acquisition order in the CTFTomo objects (field added to that class in scipion-em-tomo v3.7.0).
+  - Refactorization: avoid the excesive usage of getFirstItem.
+  - File generation methods (most of them in utils) adapted to the CTF-TS intersection functionality.
+  - Fixing sorted dictionary in ctf correction. How to access the defocus list in utils.py
 3.3.0:
   - bugfix for import ctf, set missing defocus flag
   - move plugin-specific import CTF protocol to the core

MANIFEST.in

@@ -4,7 +4,7 @@ include *.rst
 # Include the license file
 include LICENSE
 
-# Include the changelog
+# Include the changelog and requirements
 include *.txt
 
 # Include conf file

README.rst

@@ -24,12 +24,6 @@ This plugin provides wrappers for several programs of `IMOD <https://bio3d.color
         :target: https://pypi.python.org/pypi/scipion-em-imod
         :alt: Downloads
 
-Current development
--------------------
-
-This plugin is currently in **BETA** mode.
-
-
 Installation
 ------------
 

imod/__init__.py

@@ -35,7 +35,7 @@
 
 from .constants import IMOD_HOME, ETOMO_CMD, DEFAULT_VERSION, VERSIONS, IMOD_VIEWER_BINNING
 
-__version__ = '3.4.1'
+__version__ = '3.5.0'
 _logo = "icon.png"
 _references = ['Kremer1996', 'Mastronarde2017']
 

imod/protocols.conf

@@ -5,7 +5,7 @@ Tomography = [
 	]},
 	{"tag": "section", "text": "Tilt-series", "children": [
 		{"tag": "protocol_group", "text": "Tilt-series preprocess", "openItem": "False", "children": [
-			{"tag": "protocol", "value": "ProtImodTSNormalization", "text": "default"},
+			{"tag": "protocol", "value": "ProtImodTsNormalization", "text": "default"},
 			{"tag": "protocol", "value": "ProtImodXraysEraser", "text": "default"},
 			{"tag": "protocol", "value": "ProtImodDoseFilter", "text": "default"},
 			{"tag": "protocol", "value": "ProtImodExcludeViews", "text": "default"}

imod/protocols/__init__.py

@@ -40,10 +40,10 @@
 from .protocol_fiducialAlignment import ProtImodFiducialAlignment
 from .protocol_fiducialModel import ProtImodFiducialModel
 from .protocol_goldBeadPicker3d import ProtImodGoldBeadPicker3d
-from .protocol_tomoNormalization import ProtImodTomoNormalization
+from .protocol_tomoPreprocess import ProtImodTomoNormalization
 from .protocol_tomoProjection import ProtImodTomoProjection
 from .protocol_tomoReconstruction import ProtImodTomoReconstruction
-from .protocol_tsNormalization import ProtImodTSNormalization
+from .protocol_tsPreprocess import ProtImodTsNormalization
 from .protocol_xCorrPrealignment import ProtImodXcorrPrealignment
 from .protocol_xRaysEraser import ProtImodXraysEraser
 

imod/protocols/deprecated_20240411/__init__.py

@@ -0,0 +1 @@
+# Gold bead eraser: it does not seem to work fine. We may back to it in the future...

imod/protocols/protocol_goldBeadEraser.py → imod/protocols/deprecated_20240411/protocol_goldBeadEraser.py

@@ -34,8 +34,8 @@
 from tomo.protocols import ProtTomoBase
 import tomo.objects as tomoObj
 
-from .. import Plugin, utils
-from .protocol_base import OUTPUT_TILTSERIES_NAME
+from imod import Plugin, utils
+from imod.protocols.protocol_base import OUTPUT_TILTSERIES_NAME
 
 
 class ProtImodGoldBeadEraser(EMProtocol, ProtTomoBase):

imod/protocols/protocol_applyTransformationMatrix.py

@@ -25,23 +25,27 @@
 # *****************************************************************************
 
 import os
-
 from pyworkflow import BETA
 import pyworkflow.protocol.params as params
-import pyworkflow.utils.path as path
 from pyworkflow.object import Set
 from pwem.emlib.image import ImageHandler
 from tomo.objects import TiltSeries, TiltImage
-
 from .. import Plugin, utils
-from .protocol_base import ProtImodBase, EXT_MRCS_TS_EVEN_NAME, EXT_MRCS_TS_ODD_NAME
+from .protocol_base import ProtImodBase, XF_EXT, ODD, EVEN
 
 
 class ProtImodApplyTransformationMatrix(ProtImodBase):
     """
     Compute the interpolated tilt-series from its transform matrix.
+    The protocol makes use of the IMod command newstack
     More info:
         https://bio3d.colorado.edu/imod/doc/man/newstack.html
+
+    Generally, the tilt series has an associated transformation matrix
+    which contains the alignment information. The transformation matrix
+    is usually associated but not applied to avoid to accumulate interpolation
+    errors during the image processing. This protocol allows to apply
+    the transformation matrix to the tilt series
     """
 
     _label = 'Apply transformation'
@@ -54,16 +58,36 @@ def _defineParams(self, form):
                       params.PointerParam,
                       pointerClass='SetOfTiltSeries',
                       important=True,
-                      label='Input set of tilt-series')
+                      label='Tilt-series to apply the transformation matrix')
 
         form.addParam('binning', params.IntParam,
                       default=1,
-                      label='Binning',
-                      help='Binning to be applied to the interpolated '
-                           'tilt-series in IMOD convention. Images will be '
-                           'binned by the given factor. Must be an integer '
-                           'bigger than 1')
-
+                      label='Binning for the interpolated',
+                      help='Binning to be applied to the interpolated tilt-series in IMOD '
+                           'convention. \n'
+                           'Binning is an scaling factor given by an integer greater than 1. '
+                           'IMOD uses ordinary binning (with antialiasing filter) to reduce images in size by the given factor. '
+                           'The value of a binned pixel is the average of pixel values in each block '
+                           'of pixels being binned. Binning is applied before all other image '
+                           'transformations.')
+
+        form.addParam('taperInside',
+                      params.BooleanParam,
+                      expertLevel=params.LEVEL_ADVANCED,
+                      default=True,
+                      label='Taper inwards from the edge?',
+                      help='When the image is transformed areas with no information are filled in (e.g. because of rotation).'
+                      'Decide whether tapering is done inwards or outwards from the edge.')
+
+        form.addParam('linear',
+                      params.BooleanParam,
+                      expertLevel=params.LEVEL_ADVANCED,
+                      default=False,
+                      label='Linear interpolation?',
+                      help='From newstack man page: Use linear instead of cubic interpolation to transform images. '
+                            'Linear interpolation is more suitable when images are very noisy, '
+                            'but cubic interpolation will preserve fine detail better when noise is not an issue.')
+
         form.addParam('processOddEven',
                       params.BooleanParam,
                       expertLevel=params.LEVEL_ADVANCED,
@@ -74,47 +98,49 @@ def _defineParams(self, form):
 
     # -------------------------- INSERT steps functions -----------------------
     def _insertAllSteps(self):
-        self._failedTs = []
-
-        for ts in self.inputSetOfTiltSeries.get():
-            self._insertFunctionStep(self.generateTransformFileStep, ts.getObjId())
-            self._insertFunctionStep(self.computeAlignmentStep, ts.getObjId())
-            self._insertFunctionStep(self.generateOutputStackStep, ts.getObjId())
-            self._insertFunctionStep(self.createOutputFailedSet, ts.getObjId())
+        self._initialize()
+        for tsId in self.tsDict.keys():
+            self._insertFunctionStep(self.generateTransformFileStep, tsId)
+            self._insertFunctionStep(self.computeAlignmentStep, tsId)
+            self._insertFunctionStep(self.generateOutputStackStep, tsId)
+            self._insertFunctionStep(self.createOutputFailedStep, tsId)
         self._insertFunctionStep(self.closeOutputSetsStep)
 
     # --------------------------- STEPS functions ------------------------------
-    def generateTransformFileStep(self, tsObjId):
-        ts = self.inputSetOfTiltSeries.get()[tsObjId]
-        tsId = ts.getTsId()
-        extraPrefix = self._getExtraPath(tsId)
-        path.makePath(extraPrefix)
-        utils.formatTransformFile(ts,
-                                  os.path.join(extraPrefix,
-                                               ts.getFirstItem().parseFileName(extension=".xf")))
+    def _initialize(self):
+        self._failedTs = []
+        self.tsDict = {ts.getTsId(): ts.clone(ignoreAttrs=[]) for ts in self.inputSetOfTiltSeries.get()}
+
+    def generateTransformFileStep(self, tsId):
+        ts = self.tsDict[tsId]
+        self.genTsPaths(tsId)
+        utils.genXfFile(ts, self.getExtraOutFile(tsId, ext=XF_EXT))
 
     @ProtImodBase.tryExceptDecorator
-    def computeAlignmentStep(self, tsObjId):
-        ts = self.inputSetOfTiltSeries.get()[tsObjId]
-        tsId = ts.getTsId()
-        extraPrefix = self._getExtraPath(tsId)
+    def computeAlignmentStep(self, tsId):
+        ts = self.tsDict[tsId]
         firstItem = ts.getFirstItem()
         binning = self.binning.get()
 
         paramsAlignment = {
             'input': firstItem.getFileName(),
-            'output': os.path.join(extraPrefix, firstItem.parseFileName()),
-            'xform': os.path.join(extraPrefix, firstItem.parseFileName(extension=".xf")),
+            'output': self.getExtraOutFile(tsId),
+            'xform': self.getExtraOutFile(tsId, ext=XF_EXT),
             'bin': binning,
-            'imagebinned': 1.0
+            'imagebinned': 1.0,
+            'taper': "1,1" if self.taperInside else "1,0"
         }
 
         argsAlignment = "-input %(input)s " \
                         "-output %(output)s " \
                         "-xform %(xform)s " \
                         "-bin %(bin)d " \
                         "-antialias -1 " \
-                        "-imagebinned %(imagebinned)s "
+                        "-imagebinned %(imagebinned)s " \
+                        "-taper %(taper)s "
+
+        if self.linear.get():
+            argsAlignment += "-linear "
 
         rotationAngle = ts.getAcquisition().getTiltAxisAngle()
 
@@ -139,19 +165,15 @@ def computeAlignmentStep(self, tsObjId):
             oddFn = firstItem.getOdd().split('@')[1]
             evenFn = firstItem.getEven().split('@')[1]
             paramsAlignment['input'] = oddFn
-            paramsAlignment['output'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_ODD_NAME)
+            paramsAlignment['output'] = self.getExtraOutFile(tsId, suffix=ODD)
             Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment)
             paramsAlignment['input'] = evenFn
-            paramsAlignment['output'] = os.path.join(extraPrefix, tsId+EXT_MRCS_TS_EVEN_NAME)
+            paramsAlignment['output'] = self.getExtraOutFile(tsId, suffix=EVEN)
             Plugin.runImod(self, 'newstack', argsAlignment % paramsAlignment)
 
-    def generateOutputStackStep(self, tsObjId):
-
-        ts = self.inputSetOfTiltSeries.get()[tsObjId]
-        tsId = ts.getTsId()
-        extraPrefix = self._getExtraPath(tsId)
-
-        outputLocation = os.path.join(extraPrefix, ts.getFirstItem().parseFileName())
+    def generateOutputStackStep(self, tsId):
+        ts = self.tsDict[tsId]
+        outputLocation = self.getExtraOutFile(tsId)
 
         if os.path.exists(outputLocation):
             output = self.getOutputInterpolatedSetOfTiltSeries(self.inputSetOfTiltSeries.get())
@@ -181,8 +203,8 @@ def generateOutputStackStep(self, tsObjId):
                     newTi.setAcquisition(acq)
                     newTi.setLocation(index, outputLocation)
                     if self.applyToOddEven(ts):
-                        locationOdd = index, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_ODD_NAME))
-                        locationEven = index, (os.path.join(extraPrefix, tsId + EXT_MRCS_TS_EVEN_NAME))
+                        locationOdd = index, (self.getExtraOutFile(tsId, suffix=ODD))
+                        locationEven = index, (self.getExtraOutFile(tsId, suffix=EVEN))
                         newTi.setOddEven([ih.locationToXmipp(locationOdd), ih.locationToXmipp(locationEven)])
                     else:
                         newTi.setOddEven([])
@@ -201,6 +223,10 @@ def generateOutputStackStep(self, tsObjId):
             output.write()
             self._store()
 
+    def createOutputFailedStep(self, tsId):
+        ts = self.tsDict[tsId]
+        super().createOutputFailedSet(ts)
+
     def closeOutputSetsStep(self):
         for _, output in self.iterOutputAttributes():
             output.setStreamState(Set.STREAM_CLOSED)