Merge branch 'main' into staging

NeurodataWithoutBorders · Jan 13, 2025 · ae67d2f · ae67d2f
2 parents 57fec0c + 9c45fb4
commit ae67d2f
Show file tree

Hide file tree

Showing 11 changed files with 63 additions and 65 deletions.
diff --git a/.github/workflows/testing_dev_e2e_with_live_services.yml b/.github/workflows/testing_dev_e2e_with_live_services.yml
@@ -86,7 +86,12 @@ jobs:
 
       - if: matrix.os == 'ubuntu-latest'
         name: Run tests with xvfb
-        run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:tutorial
+        run: |
+          # Adjust Chrome sandbox (used by Puppeteer) owner and permissions to bypass Ubuntu 24.04's
+          # new restrictions for AppImage apps. https://github.com/electron/electron/issues/42510
+          sudo chown root ./node_modules/electron/dist/chrome-sandbox
+          sudo chmod 4755 ./node_modules/electron/dist/chrome-sandbox
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:tutorial
 
       - name: Archive E2E Test Screenshots
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/testing_pipelines.yml b/.github/workflows/testing_pipelines.yml
@@ -74,24 +74,29 @@ jobs:
       - name: Get ephy_testing_data current head hash
         id: ephys
         run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)"
+
       - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }}
         uses: actions/cache@v4
         id: cache-ephys-datasets
         with:
           path: ./ephy_testing_data
           key: ephys-datasets-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }}
+
       - name: Get ophys_testing_data current head hash
         id: ophys
         run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)"
+
       - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
         uses: actions/cache@v4
         id: cache-ophys-datasets
         with:
           path: ./ophys_testing_data
           key: ophys-datasets-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }}
+
       - name: Get behavior_testing_data current head hash
         id: behavior
         run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)"
+
       - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }}
         uses: actions/cache@v4
         id: cache-behavior-datasets

diff --git a/docs/conf_extlinks.py b/docs/conf_extlinks.py
@@ -1,6 +1,6 @@
 # Use this for mapping to external links
 extlinks = {
-    "linux-latest": ("https://github.com/neurodatawithoutborders/nwb-guide/tree/linux-latest/%s", "%s"),
+    "linux-dev": ("https://github.com/neurodatawithoutborders/nwb-guide/tree/linux-dev/%s", "%s"),
     "releases": ("https://github.com/NeurodataWithoutBorders/nwb-guide/releases/%s", "%s"),
     "pynwb-docs": ("https://pynwb.readthedocs.io/en/stable/%s", "%s"),
     "matnwb-src": ("https://github.com/NeurodataWithoutBorders/matnwb/%s", "%s"),

diff --git a/docs/installation.rst b/docs/installation.rst
@@ -27,9 +27,9 @@ Download and run `NWB-GUIDE-arm64.dmg <https://github.com/NeurodataWithoutBorder
 Linux
 -----
 
-Please clone the :linux-latest:`linux-latest <>` branch of the NWB GUIDE and follow the :ref:`Developer Installation instructions<developer_installation>` after the "Clone the Repo" step.
+Please clone the :linux-dev:`linux-dev <>` branch of the NWB GUIDE and follow the :ref:`Developer Installation instructions<developer_installation>` after the "Clone the Repo" step.
 
 .. code-block:: bash
 
-   git clone --branch linux-latest https://github.com/NeurodataWithoutBorders/nwb-guide
+   git clone --branch linux-dev https://github.com/NeurodataWithoutBorders/nwb-guide
    cd nwb-guide
diff --git a/environments/environment-Linux.yml b/environments/environment-Linux.yml
@@ -7,16 +7,19 @@ dependencies:
   - nodejs = 18.16.1
   - numcodecs = 0.11.0
   # install these from conda-forge so that dependent packages get included in the distributable
-  - jsonschema = 4.18.0  # installs jsonschema-specifications
+  - jsonschema = 4.18.0 # installs jsonschema-specifications
   - pip
   - pip:
       - chardet == 5.1.0
       - configparser == 6.0.0
       - flask == 2.3.2
       - flask-cors == 4.0.0
       - flask_restx == 1.1.0
-      - werkzeug < 3.0  # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask.
+      - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask.
       - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5
-      - scikit-learn == 1.4.0  # Tutorial data generation
-      - tqdm_publisher >= 0.0.1  # Progress bars
-      - tzlocal >= 5.2  # Frontend timezone handling
+      - scikit-learn == 1.4.0 # Tutorial data generation
+      - tqdm_publisher >= 0.0.1 # Progress bars
+      - tzlocal >= 5.2 # Frontend timezone handling
+      - ndx-pose == 0.1.1
+      - nwbinspector == 0.6.1
+      - tables
diff --git a/environments/environment-MAC-apple-silicon.yml b/environments/environment-MAC-apple-silicon.yml
@@ -7,11 +7,11 @@ dependencies:
   - nodejs = 18.16.1
   # Install these from conda-forge so that dependent packages get included in the distributable
   - numcodecs = 0.11.0
-  - lxml = 4.9.3  # PyPI build fails due to x64/arm64 mismatch so install from conda-forge
-  - pyedflib = 0.1.32  # PyPI build fails due to x64/arm64 mismatch so install from conda-forge
-  - numpy  # May have x64/arm64 mismatch issues so install from conda-forge
+  - lxml = 4.9.3 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge
+  - pyedflib = 0.1.32 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge
+  - numpy # May have x64/arm64 mismatch issues so install from conda-forge
   - pytables = 3.8 # PyPI build fails on arm64 so install from conda-forge (used by neuroconv deps)
-  - jsonschema = 4.18.0  # Also installs jsonschema-specifications
+  - jsonschema = 4.18.0 # Also installs jsonschema-specifications
   - pip
   - pip:
       - setuptools==70.0.0
@@ -22,7 +22,11 @@ dependencies:
       - flask-cors == 4.0.0
       - flask_restx == 1.1.0
       - werkzeug < 3.0  # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask.
-      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5      - scikit-learn == 1.4.0  # Tutorial data generation
+      # NOTE: the NeuroConv wheel on PyPI includes sonpy which is not compatible with arm64, so build and install
+      # NeuroConv from GitHub, which will remove the sonpy dependency when building from Mac arm64
+      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5
       - scikit-learn == 1.4.0  # Tutorial data generation
       - tqdm_publisher >= 0.0.1  # Progress bars
       - tzlocal >= 5.2  # Frontend timezone handling
+      - ndx-pose == 0.1.1
+      - nwbinspector == 0.6.1
diff --git a/environments/environment-MAC-intel.yml b/environments/environment-MAC-intel.yml
@@ -7,7 +7,7 @@ dependencies:
   - nodejs = 18.16.1
   - numcodecs = 0.11.0
   # install these from conda-forge so that dependent packages get included in the distributable
-  - jsonschema = 4.18.0  # installs jsonschema-specifications
+  - jsonschema = 4.18.0 # installs jsonschema-specifications
   - pip
   - pip:
       - setuptools==70.0.0
@@ -19,7 +19,10 @@ dependencies:
       - flask-cors == 4.0.0
       - flask_restx == 1.1.0
       - werkzeug < 3.0  # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask.
-      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5      - scikit-learn == 1.4.0  # Tutorial data generation
+      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5
       - scikit-learn == 1.4.0  # Tutorial data generation
       - tqdm_publisher >= 0.0.1  # Progress bars
       - tzlocal >= 5.2  # Frontend timezone handling
+      - ndx-pose == 0.1.1
+      - nwbinspector == 0.6.1
+      - tables
diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml
@@ -12,7 +12,7 @@ dependencies:
   - pip:
       - setuptools==70.0.0
       - PyInstaller==6.7.0
-      - pyinstaller-hooks-contrib == 2024.6  # Fix needed for pydantic v2; otherwise imports pydantic.compiled
+      - pyinstaller-hooks-contrib == 2024.6 # Fix needed for pydantic v2; otherwise imports pydantic.compiled
       - chardet == 5.1.0
       - configparser == 6.0.0
       - flask == 2.3.2
@@ -23,3 +23,6 @@ dependencies:
       - scikit-learn == 1.4.0  # Tutorial data generation
       - tqdm_publisher >= 0.0.1  # Progress bars
       - tzlocal >= 5.2  # Frontend timezone handling
+      - ndx-pose == 0.1.1
+      - nwbinspector ==0.6.1
+      - tables
diff --git a/src/electron/frontend/assets/css/main.css b/src/electron/frontend/assets/css/main.css
@@ -310,8 +310,8 @@ button[disabled] {
     padding: 3px 10px;
     background-color: #fff;
     font-size: 13px;
-    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif,
-        "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans",
+        sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
 }
 
 .menu.file {
@@ -324,8 +324,8 @@ button[disabled] {
     background-color: #fff;
     color: black;
     font-size: 13px;
-    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif,
-        "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans",
+        sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
 }
 
 .menu.file li:hover {
@@ -355,8 +355,8 @@ button[disabled] {
     padding: 3px 10px;
     background-color: #fff;
     font-size: 13px;
-    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif,
-        "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+    font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans",
+        sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
 }
 
 .menu.reg-folder.off {

diff --git a/src/example_pipelines.yml b/src/example_pipelines.yml
@@ -26,80 +26,66 @@ SpikeGLX-Phy:
     PhySortingInterface:
       folder_path: ephy_testing_data/phy/phy_example_0
 
-
 SpikeGLX_v1_SingleProbe_AP:
   SpikeGLXRecordingInterface:
     file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin
 
-
 SpikeGLXConverter-Phy:
   SpikeGLXConverterPipe:
     folder_path: ephy_testing_data/spikeglx/Noise4Sam_g0
   PhySortingInterface:
     folder_path: ephy_testing_data/phy/phy_example_0
 
-
 Suite2P_SinglePlane_SingleChannel:
   metadata:
     NWBFile:
       session_start_time: 2024-05-09T00:00
     Ophys:
       ImagingPlane:
-        -
-          excitation_lambda: NaN
+        - excitation_lambda: NaN
           optical_channel:
-            -
-              emission_lambda: NaN
+            - emission_lambda: NaN
 
   interfaces:
     Suite2pSegmentationInterface:
       folder_path: ophys_testing_data/segmentation_datasets/suite2p
 
-
 Caiman_SinglePlane_SingleChannel:
   metadata:
     NWBFile:
       session_start_time: 2024-05-09T00:00
     Ophys:
       ImagingPlane:
-        -
-          excitation_lambda: NaN
+        - excitation_lambda: NaN
           optical_channel:
-            -
-              emission_lambda: NaN
+            - emission_lambda: NaN
 
   interfaces:
     CaimanSegmentationInterface:
       file_path: ophys_testing_data/segmentation_datasets/caiman/caiman_analysis.hdf5
 
-
 ScanImage_SinglePlane_SingleChannel:
-
   metadata:
     Ophys:
       ImagingPlane:
-        -
-          excitation_lambda: NaN
+        - excitation_lambda: NaN
           optical_channel:
-            -
-              emission_lambda: NaN
+            - emission_lambda: NaN
 
   interfaces:
     ScanImageImagingInterface:
       file_path: ophys_testing_data/imaging_datasets/ScanImage/sample_scanimage_version_3_8.tiff
 
-
 TDT:
   metadata:
-      NWBFile:
-        session_start_time: 2024-05-09T00:00
+    NWBFile:
+      session_start_time: 2024-05-09T00:00
 
   interfaces:
     TdtRecordingInterface:
       folder_path: ephy_testing_data/tdt/aep_05
       gain: 1
 
-
 CellExplorer:
   metadata:
     NWBFile:
@@ -109,7 +95,6 @@ CellExplorer:
     CellExplorerRecordingInterface:
       folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed
 
-
 CellExplorerUnits:
   metadata:
     NWBFile:
@@ -121,7 +106,6 @@ CellExplorerUnits:
     CellExplorerSortingInterface:
       file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat
 
-
 CellExplorerSortingOnly:
   metadata:
     NWBFile:
@@ -131,24 +115,19 @@ CellExplorerSortingOnly:
     CellExplorerSortingInterface:
       file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat
 
-
 IntanSingleFileRecording:
   metadata:
     NWBFile:
       session_start_time: 2024-05-09T00:00
     Ecephys:
       ElectrodeGroup:
-        -
-          location: Test location
-        -
-          location: Test location
-        -
-          location: Test location
+        - location: Test location
+        - location: Test location
+        - location: Test location
   interfaces:
     IntanRecordingInterface:
       file_path: ephy_testing_data/intan/intan_rhd_test_1.rhd
 
-
 MEArec:
   metadata:
     NWBFile:
@@ -158,32 +137,27 @@ MEArec:
     MEArecRecordingInterface:
       file_path: ephy_testing_data/mearec/mearec_test_10s.h5
 
-
 BrukerTiffMultiChannelSinglePlane:
   metadata:
     Ophys:
       ImagingPlane:
-        -
-          excitation_lambda: NaN
+        - excitation_lambda: NaN
           optical_channel:
-            -
-              emission_lambda: NaN
+            - emission_lambda: NaN
 
   interfaces:
     BrukerTiffSinglePlaneConverter:
       folder_path: ophys_testing_data/imaging_datasets/BrukerTif/NCCR32_2023_02_20_Into_the_void_t_series_baseline-000
 
-
 DeepLabCut:
   metadata:
     NWBFile:
       session_start_time: 2024-05-09T00:00
 
   interfaces:
     DeepLabCutInterface:
-      file_path: behavior_testing_data/DLC/m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5
-      config_file_path: behavior_testing_data/DLC/config.yaml
-
+      file_path: behavior_testing_data/DLC/open_field_with_video/m3v1mp4DLC_resnet50_openfieldOct30shuffle1_1000.h5
+      config_file_path: behavior_testing_data/DLC/open_field_with_video/config.yaml
 
 SLEAP:
   metadata:

diff --git a/tests/e2e/puppeteer.ts b/tests/e2e/puppeteer.ts
@@ -50,7 +50,8 @@ export const connect = () => {
     });
 
     const browserURL = `http://localhost:${electronDebugPort}`
-    const browser = output.browser = await puppeteer.launch({ headless: 'new' })
+    // See notes on no sandbox mode: https://pptr.dev/troubleshooting#setting-up-chrome-linux-sandbox
+    const browser = output.browser = await puppeteer.launch({ headless: 'new', args: ["--no-sandbox", "--disable-setuid-sandbox"] })
     const page = output.page = await browser.newPage();
     await page.goto(browserURL);
     const endpoint = await page.evaluate(() => fetch(`json/version`).then(res => res.json()).then(res => res.webSocketDebuggerUrl))