Merge pull request #280 from CAMBI-tech/2.0.0rc3

2.0.0rc3

.github/workflows/main.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -36,8 +36,10 @@ jobs:
         sudo apt-get install libsndfile*
         sudo apt-get install xvfb
         python -m pip install --upgrade pip
+        pip install attrdict3
         pip install -r dev_requirements.txt
         pip install -e .
+        pip install kenlm==0.1 --global-option="--max_order=12"
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -64,20 +66,25 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install dependencies for python3.9
+      if: matrix.python-version == 3.9
+      run: |
+        python -m pip install --upgrade pip
+        pip install ./.bcipy/downloads/pyWinhook-1.6.2-cp39-cp39-win_amd64.whl 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install -r dev_requirements.txt
-        pip install WxPython==4.1.1
         pip install -e .
+        pip install kenlm==0.1 --global-option="--max_order=12"
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -93,7 +100,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2
@@ -105,8 +112,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install -r dev_requirements.txt
-        pip install WxPython==4.1.1
         pip install -e .
+        pip install kenlm==0.1 --global-option="--max_order=12"
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names

.gitignore

@@ -12,7 +12,7 @@ venv*/
 __pycache__/
 .cache/
 
-# BciPy files and directories 
+# BciPy files and directories
 calibration.csv
 rawdata.csv
 buffer.db
@@ -25,4 +25,9 @@ bcipy.egg-info/
 build/
 dist/
 
-bcipy/parameters/parameters_*
+bcipy/parameters/parameters_*
+bcipy_cache
+
+bcipy/language/lms/lm_dec19_char_large_12gram.*
+bcipy/language/lms/out_*
+bcipy/language/out/

CHANGELOG.md

@@ -1,5 +1,54 @@
+# 2.0.0-rc.3
+
+## Contributions
+
+- GUIs
+    - Parameter GUI `save as` functionality #255
+    - Parameter GUI user editable parameters defined in `bcipy/parameters.json` #256
+    - GUI history using a `.bcipy_cache` file #257
+- Acquisition
+    - Set channel spec information in devices.json. Removed aliases. #266 Updated default analysis channels for Wearable Sensing devices #279
+    - Refactor to allow multiple devices to be configured for multimodal acquisition. #277
+    - Refinements to LSL server to use the device ChannelSpec information for generating metadata. #282
+- Matrix
+    - Matrix calibration refinements #262
+    - Matrix Copy Phrase Task #261
+- RSVP
+    - RSVP Calibration pulls out inquiry preview parameters and calls the display with them if enabled
+    - RSVP Display `do_inquiry` accepts a preview_calibration argument that will present the full inquiry to come if True after the prompt and before the fixation
+    - RSVP Calibration now support Inquiry Preview #267
+- Signal Processing
+    - update from `sosfilt` to `sosfiltfilt` #269 add prestimulus to trial reshaping in the InquiryReshaper #279
+    - add continuous wavelet transform to signal processing #279
+- MNE
+    - convert to mne can now handle conversion to Volts from microvolts, and no channel map #259
+- Dependencies
+    - Upgrade PsychoPy, transformers (#268) and pandas, add support for python3.9, deprecate python3.7 #272 upgrade scikit-learn and add pywavelets #279
+- BciPy client
+    - Add `-nv` (no visualizations) and `-f` (fake data) client options #272
+- Session
+    - After each session run using `bci_main` an ERP visualization is generated by default using new function `visualize_session_data` #260 add screen_resolution to logs and an alert if the screen resolution is too low #283
+- Language Model
+    - removed previous `gpt2.py` implementation, replaced with generic Causal model `causal.py` #268
+    - added KenLM model `kenlm.py` #268
+    - added mixture model `mixture.py` and script to tune weights `mixture_tuning.py` #268
+    - added script to evaluate language model performance `lm_eval.py` #268
+- Signal Model
+    - added `RdaKdeModel` and restructured to pull out common elements from the PcaRdaKdeModel #279
+- Bug Fixes
+    - Missing inits and lock some dependencies #258 Fix Windows Builds (pin pygame version) #265 Fix Mac Os Builds (pin pyo version) #432
+    - Update cross_validation.py #271
+    - Add jitter to matrix and time tests #281
+- Documentation
+    - Update README.md with correct installation instructions for development requirements #263 and new language model installation instructions #268
+- Misc Features!
+    - Add `tobii_to_norm` and `norm_to_tobii` methods to `helpers/convert.py` #278
+    - Add a GUI TaskBar display component for Psychopy windows #270
+
 # 2.0.1-rc.2
 
+## Contributions
+
 Small patch release to fix a missing import in helpers/parameters.py
 
 # 2.0.0-rc.2

README.md

@@ -18,22 +18,36 @@ Memmott, T., Koçanaoğulları, A., Lawhead, M., Klee, D., Dudy, S., Fried-Oken,
 
 ## Dependencies
 ---------------
-This project requires Python 3.7 or 3.8. All other dependencies defined in the requirements.txt.
+This project requires Python 3.8 or 3.9. Please see notes below for additional OS specific dependencies before installation can be completed.
 
+### Linux
+
+You will need to install the prerequisites defined in `scripts\shell\linux_requirements.sh` as well as `pip install attrdict3`.
+
+### Windows
+
+If you are using a Windows machine, you will need to install the [Microsoft Visual C++ Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/).
+
+*python 3.9 only!*
+You will need to install pyWinhook manually. See [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pywinhook) for the appropriate wheel file (`pyWinhook‑1.6.2‑cp39‑cp39‑win_amd64.whl`). Then run `pip install <path_to_wheel_file>`. We also include the 64-bit wheel file in the `.bcipy/downloads/` directory.
+
+### Mac
+
+If you are using a Mac, you will need to install XCode and enable command line tools. `xcode-select --install`. 
 
 ## Installation
 ---------------
 
 #### BciPy Setup
 
-In order to run BciPy on your computer, first install **Python 3** [from here.](https://www.python.org/downloads/)
+In order to run BciPy on your computer, after following the dependencies above, you will need to install the BciPy package.
 
-To use all the goodies locally (including the GUI and demo scripts)
+To install for use locally,
 1. Git clone https://github.com/BciPy/BciPy.git
 2. Change directory in your terminal to the repo
 3. Run `pip install -e .`
-4. If using Mac, you will need to install XCode and enable command line tools. `xcode-select --install`
-5. If you're on Windows, you may need to uninstall pygame (`pip uninstall pygame`). Psychopy, for historical reasons, keeps pygame but it just spams your console logs if you only want to use pyglet (which we use in this repository!)
+4. To use the KenLMLanguageModel class, you must manually install the kenlm package. `pip install kenlm==0.1 --global-option="--max_order=12"`.
+
 
 If wanting the latest version from PyPi:
 1. `pip install bcipy`
@@ -47,15 +61,12 @@ make dev-install
 
 #### Usage Locally
 
-Start by running `python bcipy/gui/BCInterface.py` in your command prompt or terminal. This will run the GUI. You may also use the command `make bci-gui`. You may also invoke the experiment directly using command line tools from bcipy.
-
-Ex. `bcipy` *this will use default parameters, user, experiment and task*
-
-You can pass it attributes with flags, if desired.
-
-Ex. `bcipy --user "bci_user" --task "RSVP Calibration"`
-
-Use the help flag to see other available input options: `bcipy --help`
+Two ways to get started using BciPy for data collection:
+	1. Run `python bcipy/gui/BCInterface.py` in your command prompt or terminal from from base BciPy directory. This will execute the main BCI GUI. You may also use the command `make bci-gui`. 
+	2. Invoke the experiment directly using command line utility `bcipy`.
+		- You can pass it attributes with flags, if desired.
+				Ex. `bcipy --user "bci_user" --task "RSVP Calibration"`
+		- Use the help flag to see other available input options: `bcipy --help`
 
 ##### Example usage as a package
 
@@ -141,15 +152,15 @@ For example, you may run the main BciPy demo by:
 
 `python demo/bci_main_demo.py`
 
-This demo will load in parameters and execute a demo task defined in the file. There are demo files for all modules listed above except language_model, helpers, and utils. Run them as a python script!
+This demo will load in parameters and execute a demo task defined in the file. There are demo files for all modules listed above except helpers and utils. Run them as a python script!
 
 
 ## Testing
 ----------
 
 When writing tests, put them in the correct module, in a tests folder, and prefix the file and test itself with `test_` in order for pytest to discover it. See other module tests for examples!
 
-Development requirements must be installed before running: `pip install dev_requirements.txt`
+Development requirements must be installed before running: `pip install -r dev_requirements.txt`
 
 To run all tests, in the command line:
 

bcipy/acquisition/__init__.py

@@ -1,8 +1,9 @@
-from bcipy.acquisition.protocols.lsl.lsl_client import LslAcquisitionClient
+from bcipy.acquisition.protocols.lsl.lsl_client import (LslAcquisitionClient,
+                                                        discover_device_spec)
 from bcipy.acquisition.datastream.lsl_server import LslDataServer, await_start
+from bcipy.acquisition.multimodal import ClientManager
 
 __all__ = [
-    'LslAcquisitionClient',
-    'LslDataServer',
-    'await_start'
+    'LslAcquisitionClient', 'LslDataServer', 'await_start',
+    'discover_device_spec', 'ClientManager'
 ]

bcipy/acquisition/datastream/lsl_server.py

@@ -26,7 +26,7 @@ class LslDataServer(StoppableThread):
     using pylsl. See https://github.com/sccn/labstreaminglayer/wiki.
 
     In parameters.json, if the fake_data parameter is set to true and the
-    device is set to LSL, this server will be used to mock data. Alternatively,
+    device is set to DSI-24, this server will be used to mock data. Alternatively,
     fake_data can be set to false and this module can be run standalone in its
     own python instance.
 
@@ -63,18 +63,14 @@ def __init__(self, device_spec: DeviceSpec, generator: Generator = None, include
                           f'{device_spec.content_type.lower()}_{uuid.uuid1()}')
 
         if include_meta:
-            # TODO: different types of metadata depending on the content type
-            unit = 'unknown'
-            channel_type = 'unknown'
-            if self.device_spec.content_type == 'EEG':
-                unit = 'microvolts'
-                channel_type = 'EEG'
             meta_channels = info.desc().append_child('channels')
             for channel in device_spec.channel_specs:
-                meta_channels.append_child('channel') \
-                    .append_child_value('label', channel.name) \
-                    .append_child_value('unit', channel.units or unit) \
-                    .append_child_value('type', channel.type or channel_type)
+                channel_node = meta_channels.append_child('channel')
+                channel_node.append_child_value('label', channel.name)
+                if channel.units:
+                    channel_node.append_child_value('unit', channel.units)
+                if channel.type:
+                    channel_node.append_child_value('type', channel.type)
 
         self.outlet = StreamOutlet(info)
 
@@ -181,7 +177,7 @@ def main():
                         help="file containing data to be streamed; "
                         "if missing, random data will be served.")
     parser.add_argument('-m', '--markers', action="store_true", default=False)
-    parser.add_argument('-n', '--name', default='LSL', help='Name of the device spec to mock.')
+    parser.add_argument('-n', '--name', default='DSI-24', help='Name of the device spec to mock.')
     args = parser.parse_args()
 
     if args.filename:

bcipy/acquisition/demo/demo_lsl_acq_client.py

@@ -12,7 +12,7 @@ def main():
     The client can be stopped with a Keyboard Interrupt (Ctl-C)."""
 
     # Start the server with the command:
-    # python bcipy/acquisition/datastream/lsl_server.py --name LSL
+    # python bcipy/acquisition/datastream/lsl_server.py --name 'DSI-24'
 
     client = LslAcquisitionClient(max_buffer_len=1, save_directory='.')
 

bcipy/acquisition/demo/demo_lsl_server.py

@@ -16,7 +16,7 @@ def main():
 
     parser.add_argument('-n',
                         '--name',
-                        default='LSL',
+                        default='DSI-24',
                         help='Name of the device spec to mock.')
     args = parser.parse_args()
     device_spec = preconfigured_device(args.name)

bcipy/acquisition/devices.py

@@ -162,33 +162,63 @@ def make_device_spec(config: dict) -> DeviceSpec:
                           'excluded_from_analysis', []))
 
 
-def load(config_path: Path = Path(DEFAULT_CONFIG)) -> Dict[str, DeviceSpec]:
+def load(config_path: Path = Path(DEFAULT_CONFIG), replace: bool = False) -> Dict[str, DeviceSpec]:
     """Load the list of supported hardware for data acquisition from the given
-    configuration file."""
+    configuration file.
+
+    Parameters
+    ----------
+        config_path - path to the devices json file
+        replace - optional; if true, existing devices are replaced; if false,
+            values will be overwritten or appended.
+    """
     global _SUPPORTED_DEVICES
-    with open(config_path, 'r', encoding=DEFAULT_ENCODING) as json_file:
-        specs = [make_device_spec(entry) for entry in json.load(json_file)]
-        _SUPPORTED_DEVICES = {spec.name: spec for spec in specs}
-        return _SUPPORTED_DEVICES
+
+    if config_path.is_file():
+        with open(config_path, 'r', encoding=DEFAULT_ENCODING) as json_file:
+            specs = [make_device_spec(entry) for entry in json.load(json_file)]
+            if specs and replace:
+                _SUPPORTED_DEVICES.clear()
+            for spec in specs:
+                _SUPPORTED_DEVICES[spec.name] = spec
+    return _SUPPORTED_DEVICES
 
 
 def preconfigured_devices() -> Dict[str, DeviceSpec]:
-    """Returns the preconfigured devices keyed by name."""
+    """Returns the preconfigured devices keyed by name. If no devices have yet
+    been configured, loads and returns the DEFAULT_CONFIG."""
     global _SUPPORTED_DEVICES
     if not _SUPPORTED_DEVICES:
         load()
     return _SUPPORTED_DEVICES
 
 
-def preconfigured_device(name: str) -> DeviceSpec:
+def preconfigured_device(name: str, strict: bool = True) -> DeviceSpec:
     """Retrieve the DeviceSpec with the given name. An exception is raised
     if the device is not found."""
     device = preconfigured_devices().get(name, None)
-    if not device:
-        raise ValueError(f"Device not found: {name}")
+    if strict and not device:
+        current = ', '.join(
+            [f"'{key}'" for key, _ in preconfigured_devices().items()])
+        msg = (
+            f"Device not found: {name}."
+            "\n\n"
+            f"The current list of devices includes the following: {current}."
+            "\n"
+            "You may register new devices using the device module `register` function or in bulk"
+            " using `load`.")
+        raise ValueError(msg)
     return device
 
 
+def with_content_type(content_type: str) -> List[DeviceSpec]:
+    """Retrieve the list of DeviceSpecs with the given content_type."""
+    return [
+        spec for spec in preconfigured_devices().values()
+        if spec.content_type == content_type
+    ]
+
+
 def register(device_spec: DeviceSpec):
     """Register the given DeviceSpec."""
     config = preconfigured_devices()