ENH: Add OTP processing option

.circleci/config.yml

@@ -2,16 +2,37 @@ version: 2
 jobs:
   build_docs:
     docker:
-      - image: circleci/python:3.7-buster
+      - image: cimg/base:stable-20.04
     steps:
       # Get our data and merge with upstream
       - checkout
-      - run: sudo apt-get install graphviz-dev graphviz
-      - run: python -m pip install --quiet --upgrade --user pip
-      - run: python -m pip install --quiet --upgrade --user numpy scipy https://github.com/mne-tools/mne-python/zipball/main numpydoc sphinx sphinx_fontawesome sphinx_bootstrap_theme pygraphviz
-      - run: python -c "import mne; mne.sys_info()"
-      - run: python setup.py develop --user
-      - run: cd doc && make html
+      - run:
+          name: Setup system
+          command: |
+            set -e
+            echo "set -e" >> $BASH_ENV
+            sudo apt update -qq
+            sudo apt install -qq graphviz-dev graphviz python3-venv python3.9-venv python3.9-dev
+            python3.9 -m venv ~/python_env
+            source ~/python_env/bin/activate
+            echo "source ~/python_env/bin/activate" >> $BASH_ENV
+            mkdir -p ~/.local/bin
+            ln -s ~/python_env/bin/python ~/.local/bin/python
+      - run:
+          name: Setup Python env
+          command: |
+            echo "Pip..."
+            pip install --quiet --upgrade pip
+            echo "Dependencies..."
+            pip install --quiet --upgrade numpy scipy https://github.com/mne-tools/mne-python/zipball/main numpydoc sphinx sphinx_fontawesome sphinx_bootstrap_theme
+            echo "PyGraphViz..."
+            pip install --upgrade pygraphviz
+            python -c "import mne; mne.sys_info()"
+            pip install -ve .
+      - run:
+          name: Build docs
+          command: |
+            make -C doc html
 
       - store_artifacts:
           path: doc/_build/html/
@@ -25,7 +46,7 @@ jobs:
 
   deploy:
     docker:
-      - image: circleci/python:3.6-stretch
+      - image: cimg/base:stable-20.04
     steps:
       - add_ssh_keys:
           fingerprints:

.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: 'linux'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
+  cancel-in-progress: true
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  job:
+    name: 'py3.10'
+    runs-on: ubuntu-20.04
+    defaults:
+      run:
+        shell: bash
+    env:
+      MNE_LOGGING_LEVEL: 'info'
+      MNE_STIM_CHANNEL: 'STI101'
+      OPENBLAS_NUM_THREADS: '1'
+      PYTHON_VERSION: '3.10'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+        name: 'Setup python'
+      - shell: bash -el {0}
+        run: |
+          pip install --upgrade --only-binary=":all:" numpy scipy matplotlib
+          pip install --upgrade pyyaml patsy pytest pytest-cov codecov flake8 pydocstyle numpydoc https://github.com/mne-tools/mne-python/archive/main.zip
+          pip install -ve .
+      - shell: bash -el {0}
+        run: pytest mnefun
+        name: 'Run tests'
+      - shell: bash -el {0}
+        run: make flake
+      - uses: codecov/codecov-action@v1
+        if: success()
+        name: 'Upload coverage to CodeCov'

doc/overview.rst

@@ -133,9 +133,19 @@ score : callable | None
 on_process : callable
     Called at each processing step.
 
-.. _do_sss:
+3. do_otp
+---------
+Run OTP processing.
+
+1. Run OTP locally using :func:`mne.preprocessing.oversampled_temporal_projection`.
+
+``preprocessing: otp``: OTP parameters
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+otp_dur : float | None
+    The window duration (in seconds) to use. Must be > or equal to tsss_dur if combining with tSSS processing.
 
-3. do_sss
+
+4. do_sss
 ---------
 
 .. warning:: Before running SSS, set ``params.mf_prebad[SUBJ]`` to a
@@ -281,12 +291,12 @@ cont_as_esss : bool
     Only supported when Python is used for SSS.
 
 
-4. do_ch_fix
+5. do_ch_fix
 ------------
 
 Fix EEG channel ordering, and also anonymize files.
 
-5. gen_ssp
+6. gen_ssp
 ----------
 
 .. warning:: Before running SSP, examine SSS'ed files and make
@@ -441,12 +451,12 @@ plot_drop_logs : bool
     If True, plot drop logs after preprocessing.
 
 
-6. apply_ssp
+7. apply_ssp
 ------------
 Apply SSP vectors and filtering to the files.
 
 
-7. write_epochs
+8. write_epochs
 ---------------
 Write epochs to disk.
 
@@ -537,7 +547,7 @@ allow_resample : bool
     matching sample rate. This is useful when recordings were errantly
     performed at different sample rates.
 
-8. gen_covs
+9. gen_covs
 -----------
 Generate covariances.
 
@@ -566,8 +576,8 @@ force_erm_cov_rank_full : bool
     is short and/or there are a lot of head movements.
 
 
-9. gen_fwd
-----------
+10. gen_fwd
+-----------
 .. warning:: Make SUBJ/trans/SUBJ-trans.fif using :ref:`mne:mne coreg`.
 
 Generate forward solutions (and source space if necessary).
@@ -591,7 +601,7 @@ fwd_mindist : float
     Minimum distance (mm) for sources in the brain from the skull in order
     for them to be included in the forward solution source space.
 
-10. gen_inv
+11. gen_inv
 -----------
 
 Generate inverses.
@@ -604,7 +614,7 @@ inv_runs : list of int
     Runs to use for each inverse.
 
 
-11. gen_report
+12. gen_report
 --------------
 
 Write :class:`mne.Report` HTML of results to disk.
@@ -673,10 +683,14 @@ bad_tag : str
     Tag for bid channel filename, usually "_post-sss.txt".
 raw_dir : str
     Raw directory, usually "raw_fif".
+otp_dir : str
+    Directory for OTP-processed files, usually "otp_fif".
 keep_orig : bool
     Keep original files after anonymization.
 raw_fif_tag : str
     File tag for raw data, usually "_raw.fif".
+otp_fif_tag : str
+    File tag for OTP-processed files, usually "_raw_otp.fif".
 sss_fif_tag : str
     File tag for SSS-processed files, usually "_raw_sss.fif".
 sss_dir : str

mnefun/_mnefun.py

@@ -25,6 +25,7 @@
 from ._status import print_proc_status
 from ._paths import _get_config_file
 from ._utils import timestring
+from ._otp import run_otp
 
 
 # Class adapted from:
@@ -156,6 +157,7 @@ def __init__(self, tmin=None, tmax=None, t_adjust=0, bmin=-0.2, bmax=0.0,
         self.trans_dir = 'trans'
         self.bad_dir = 'bads'
         self.raw_dir = 'raw_fif'
+        self.otp_dir = 'otp_fif'
         self.sss_dir = 'sss_fif'
         self.pca_dir = 'sss_pca_fif'
 
@@ -167,6 +169,7 @@ def __init__(self, tmin=None, tmax=None, t_adjust=0, bmin=-0.2, bmax=0.0,
         self.inv_erm_tag = '-erm'
         self.eq_tag = 'eq'
         self.sss_fif_tag = '_raw_sss.fif'
+        self.otp_fif_tag = '_raw_otp.fif'
         self.bad_tag = '_post-sss.txt'
         self.keep_orig = False
         # This is used by fix_eeg_channels to fix original files
@@ -178,6 +181,7 @@ def __init__(self, tmin=None, tmax=None, t_adjust=0, bmin=-0.2, bmax=0.0,
         self.hp_type = 'maxfilter'
         self.mf_args = ''
         self.tsss_dur = 60.
+        self.otp_dur = None
         self.trans_to = 'median'  # where to transform head positions to
         self.sss_format = 'float'  # output type for MaxFilter
         self.movecomp = 'inter'
@@ -366,8 +370,8 @@ def _set_static(p):
 
 
 def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
-                  do_sss=False, fetch_sss=False, do_ch_fix=False,
-                  gen_ssp=False, apply_ssp=False,
+                  do_otp=False, do_sss=False, fetch_sss=False,
+                  do_ch_fix=False, gen_ssp=False, apply_ssp=False,
                   write_epochs=False, gen_covs=False, gen_fwd=False,
                   gen_inv=False, gen_report=False, print_status=True):
     """Do M/EEG data processing.
@@ -382,6 +386,8 @@ def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
         Do scoring.
     push_raw : bool
         Push raw recording files to SSS workstation.
+    do_otp: bool
+        Apply OTP locally.
     do_sss : bool
         Run SSS remotely on SSS workstation.
     fetch_sss : bool
@@ -413,6 +419,7 @@ def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
     bools = [fetch_raw,
              do_score,
              push_raw,
+             do_otp,
              do_sss,
              fetch_sss,
              do_ch_fix,
@@ -428,6 +435,7 @@ def do_processing(p, fetch_raw=False, do_score=False, push_raw=False,
     texts = ['Pulling raw files from acquisition machine',
              'Scoring subjects',
              'Pushing raw files to remote workstation',
+             'Running OTP',
              'Running SSS using %s' % p.sss_type,
              'Pulling SSS files from remote workstation',
              'Fixing EEG order',
@@ -449,6 +457,7 @@ def score_fun(p, subjects, run_indices):
     funcs = [fetch_raw_files,
              score_fun,
              push_raw_files,
+             run_otp,
              run_sss,
              fetch_sss_files,
              fix_eeg_files,

mnefun/_otp.py

@@ -0,0 +1,46 @@
+import os
+import os.path as op
+import time
+from ._paths import get_raw_fnames
+from mne.preprocessing import oversampled_temporal_projection
+from ._sss import _read_raw_prebad
+
+def run_otp(p, subjects, run_indices):
+    """ Run Oversampled Temporal Projection (OTP) on raw data.
+    """
+
+    assert isinstance(p.otp_dur, float) and p.tsss_dur > 0
+    duration = p.otp_dur
+    if p.otp_dur is not None:
+        assert p.otp_dur >= p.tsss_dur
+
+    for si, subj in enumerate(subjects):
+        if p.disp_files:
+            print('    Denoising subject %g/%g (%s).'
+                  % (si + 1, len(subjects), subj))
+        # locate raw files with splits
+        otp_dir = op.join(p.work_dir, subj, p.otp_dir)
+        if not op.isdir(otp_dir):
+            os.mkdir(otp_dir)
+        raw_files = get_raw_fnames(p, subj, 'raw', erm=False,
+                                   run_indices=run_indices[si])
+        raw_files_out = get_raw_fnames(p, subj, 'otp', erm=False,
+                                       run_indices=run_indices[si])
+        erm_files = get_raw_fnames(p, subj, 'raw', 'only')
+        erm_files_out = get_raw_fnames(p, subj, 'otp', 'only')
+
+        #  process raw files
+        assert len(raw_files) > 0
+        for ii, (r, o) in enumerate(zip(raw_files + erm_files,
+                                        raw_files_out + erm_files_out)):
+            if not op.isfile(r):
+                raise NameError('File not found (' + r + ')')
+            raw = _read_raw_prebad(p, subj, r, disp=True).load_data()
+
+            # apply maxwell filter
+            t0 = time.time()
+            print('        Running OTP ...', end='')
+            raw_otp = oversampled_temporal_projection(
+                raw, duration=duration)
+            print('%i sec' % (time.time() - t0,))
+            raw_otp.save(o, overwrite=True, buffer_size_sec=None)

mnefun/_paths.py

@@ -82,7 +82,7 @@ def get_raw_fnames(p, subj, which='raw', erm=True, add_splits=False,
     subj : str
         Subject name.
     which : str
-        Type of raw filenames. Must be 'sss', 'raw', or 'pca'.
+        Type of raw filenames. Must be 'sss', 'raw', 'pca' or 'otp'.
     erm : bool | str
         If True, include empty-room files (appended to end). If 'only', then
         only return empty-room files.
@@ -98,7 +98,7 @@ def get_raw_fnames(p, subj, which='raw', erm=True, add_splits=False,
     fnames : list
         List of filenames.
     """
-    assert which in ('sss', 'raw', 'pca')
+    assert which in ('sss', 'raw', 'pca', 'otp')
     if which == 'sss':
         raw_dir = op.join(p.work_dir, subj, p.sss_dir)
         tag = p.sss_fif_tag
@@ -108,6 +108,9 @@ def get_raw_fnames(p, subj, which='raw', erm=True, add_splits=False,
     elif which == 'pca':
         raw_dir = op.join(p.work_dir, subj, p.pca_dir)
         tag = p.pca_extra + p.sss_fif_tag
+    elif which == 'otp':
+        raw_dir = op.join(p.work_dir, subj, p.otp_dir)
+        tag = p.otp_fif_tag
 
     if run_indices is None:
         run_indices = np.arange(len(p.run_names))

mnefun/_yaml.py

@@ -29,8 +29,8 @@
 _KEYS_TO_FLATTEN = {
     'general', 'naming', 'fetch_raw', 'scoring', 'bads', 'raw', 'annotations',
     'multithreading',
-    'preprocessing', 'head_position_estimation', 'sss', 'filtering', 'ssp',
-    'epochs', 'epoching',
+    'preprocessing', 'head_position_estimation', 'otp', 'sss', 'filtering',
+    'ssp', 'epochs', 'epoching',
     'covariance', 'forward', 'inverse',
 }