Strictly internal fix for SpikeGLX stub

environments/environment-Linux.yml

@@ -15,7 +15,10 @@ dependencies:
       - flask == 2.3.2
       - flask-cors == 4.0.0
       - flask_restx == 1.1.0
-      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[dandi,compressors,ecephys,ophys,behavior,text]
+      # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility
+      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text]
+      # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation
+      - spikeinterface == 0.100.5
       - scikit-learn == 1.4.0  # Tutorial data generation
       - tqdm_publisher >= 0.0.1  # Progress bars
       - tzlocal >= 5.2  # Frontend timezone handling

environments/environment-MAC-apple-silicon.yml

@@ -23,7 +23,10 @@ dependencies:
       - flask_restx == 1.1.0
       # NOTE: the NeuroConv wheel on PyPI includes sonpy which is not compatible with arm64, so build and install
       # NeuroConv from GitHub, which will remove the sonpy dependency when building from Mac arm64
-      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[dandi,compressors,ecephys,ophys,behavior,text]
+      # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility
+      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text]
+      # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation
+      - spikeinterface == 0.100.5
       - scikit-learn == 1.4.0  # Tutorial data generation
       - tqdm_publisher >= 0.0.1  # Progress bars
       - tzlocal >= 5.2  # Frontend timezone handling

environments/environment-MAC-intel.yml

@@ -18,7 +18,10 @@ dependencies:
       - flask == 2.3.2
       - flask-cors == 4.0.0
       - flask_restx == 1.1.0
-      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[dandi,compressors,ecephys,ophys,behavior,text]
+      # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility
+      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text]
+      # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation
+      - spikeinterface == 0.100.5
       - scikit-learn == 1.4.0  # Tutorial data generation
       - tqdm_publisher >= 0.0.1  # Progress bars
       - tzlocal >= 5.2  # Frontend timezone handling

environments/environment-Windows.yml

@@ -18,7 +18,10 @@ dependencies:
       - flask == 2.3.2
       - flask-cors === 3.0.10
       - flask_restx == 1.1.0
-      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@main#neuroconv[dandi,compressors,ecephys,ophys,behavior,text]
+      # For stability, NeuroConv is pinned at a commit just prior to breaking SpikeInterface compatibility
+      - neuroconv @ git+https://github.com/catalystneuro/neuroconv.git@fa636458aa5c321f1c2c08f6e682b4a52d5a83f3#neuroconv[dandi,compressors,ecephys,ophys,behavior,text]
+      # For stability, pinning SpikeInterface to a version that works with NeuroConv and with tutorial generation
+      - spikeinterface == 0.100.5
       - scikit-learn == 1.4.0  # Tutorial data generation
       - tqdm_publisher >= 0.0.1  # Progress bars
       - tzlocal >= 5.2  # Frontend timezone handling

src/pyflask/manageNeuroconv/manage_neuroconv.py

@@ -2,6 +2,7 @@
 
 import copy
 import hashlib
+import inspect
 import json
 import math
 import os
@@ -13,6 +14,7 @@
 from shutil import copytree, rmtree
 from typing import Any, Dict, List, Optional, Union
 
+from pynwb import NWBFile
 from tqdm_publisher import TQDMProgressHandler
 
 from .info import (
@@ -364,6 +366,30 @@ class CustomNWBConverter(NWBConverter):
         def temporally_align_data_interfaces(self):
             set_interface_alignment(self, alignment_info=alignment_info)
 
+        # From previous issue regarding SpikeGLX not generating previews of correct size
+        def add_to_nwbfile(self, nwbfile: NWBFile, metadata, conversion_options: Optional[dict] = None) -> None:
+            conversion_options = conversion_options or dict()
+            for interface_key, data_interface in self.data_interface_objects.items():
+                if isinstance(data_interface, NWBConverter):
+                    subconverter_kwargs = dict(nwbfile=nwbfile, metadata=metadata)
+
+                    # Certain subconverters fully expose control over their interfaces conversion options
+                    # (such as iterator options, including progress bar details)
+                    subconverter_keyword_arguments = list(
+                        inspect.signature(data_interface.add_to_nwbfile).parameters.keys()
+                    )
+                    if "conversion_options" in subconverter_keyword_arguments:
+                        subconverter_kwargs["conversion_options"] = conversion_options.get(interface_key, None)
+                    # Others do not, and instead expose simplified global keywords similar to a classic interface
+                    else:
+                        subconverter_kwargs.update(conversion_options.get(interface_key, dict()))
+
+                    data_interface.add_to_nwbfile(**subconverter_kwargs)
+                else:
+                    data_interface.add_to_nwbfile(
+                        nwbfile=nwbfile, metadata=metadata, **conversion_options.get(interface_key, dict())
+                    )
+
     return CustomNWBConverter
 
 
@@ -920,25 +946,57 @@ def update_conversion_progress(message):
         )
 
         # Assume all interfaces have the same conversion options for now
-        available_options = converter.get_conversion_options_schema()
-        options = {interface: {} for interface in info["source_data"]}
-
-        for interface in options:
-            available_opts = available_options.get("properties").get(interface).get("properties", {})
-
-            # Specify if stub test
-            if run_stub_test:
-                if available_opts.get("stub_test"):
-                    options[interface]["stub_test"] = True
-
-            # Specify if iterator options are available
-            elif available_opts.get("iterator_opts"):
-                options[interface]["iterator_opts"] = dict(
-                    display_progress=True,
-                    progress_bar_class=TQDMProgressSubscriber,
-                    progress_bar_options=progress_bar_options,
+        conversion_options_schema = converter.get_conversion_options_schema()
+        conversion_options = {interface: dict() for interface in info["source_data"]}
+
+        for interface_or_subconverter in conversion_options:
+            conversion_options_schema_per_interface_or_converter = conversion_options_schema.get(
+                "properties", dict()
+            ).get(interface_or_subconverter, dict())
+
+            # Object is a nested converter
+            if conversion_options_schema_per_interface_or_converter.get("title", "") == "Conversion options schema":
+                subconverter = interface_or_subconverter
+
+                conversion_options_schema_per_subinterface = conversion_options_schema_per_interface_or_converter.get(
+                    "properties", dict()
                 )
 
+                for subinterface, subschema in conversion_options_schema_per_subinterface.items():
+                    conversion_options[subconverter][subinterface] = dict()
+                    options_to_update = conversion_options[subconverter][subinterface]
+
+                    properties_per_subinterface = subschema.get("properties", dict())
+
+                    if run_stub_test is True and "stub_test" in properties_per_subinterface:
+                        options_to_update["stub_test"] = True
+
+                    # Only display per-file progress updates if not running a preview
+                    if run_stub_test is False and "iterator_opts" in properties_per_subinterface:
+                        options_to_update["iterator_opts"] = dict(
+                            display_progress=True,
+                            progress_bar_class=TQDMProgressSubscriber,
+                            progress_bar_options=progress_bar_options,
+                        )
+
+            # Object is a standard interface
+            else:
+                interface = interface_or_subconverter
+
+                conversion_options_schema_per_interface = conversion_options_schema.get("properties", dict())
+                options_to_update = conversion_options[interface]
+
+                if run_stub_test is True and "stub_test" in conversion_options_schema_per_interface:
+                    options_to_update[interface]["stub_test"] = True
+
+                # Only display per-file progress updates if not running a preview
+                if run_stub_test is False and "iterator_opts" in conversion_options_schema_per_interface:
+                    options_to_update[interface]["iterator_opts"] = dict(
+                        display_progress=True,
+                        progress_bar_class=TQDMProgressSubscriber,
+                        progress_bar_options=progress_bar_options,
+                    )
+
         # Add GUIDE watermark
         package_json_file_path = resource_path("package.json" if is_packaged() else "../package.json")
         with open(file=package_json_file_path) as fp:
@@ -951,11 +1009,12 @@ def update_conversion_progress(message):
             metadata=metadata,
             nwbfile_path=nwbfile_path,
             overwrite=overwrite,
-            conversion_options=options,
+            conversion_options=conversion_options,
             backend=backend,
         )
 
-        if not run_stub_test:
+        # Only set full backend configuration if running a full conversion
+        if run_stub_test is False:
             run_conversion_kwargs.update(dict(backend_configuration=update_backend_configuration(info)))
 
         converter.run_conversion(**run_conversion_kwargs)