diff --git a/.github/workflows/testing_dev_e2e_with_live_services.yml b/.github/workflows/testing_dev_e2e_with_live_services.yml index 8367a8df7..6778d5bf3 100644 --- a/.github/workflows/testing_dev_e2e_with_live_services.yml +++ b/.github/workflows/testing_dev_e2e_with_live_services.yml @@ -86,7 +86,12 @@ jobs: - if: matrix.os == 'ubuntu-latest' name: Run tests with xvfb - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:tutorial + run: | + # Adjust Chrome sandbox (used by Puppeteer) owner and permissions to bypass Ubuntu 24.04's + # new restrictions for AppImage apps. https://github.com/electron/electron/issues/42510 + sudo chown root ./node_modules/electron/dist/chrome-sandbox + sudo chmod 4755 ./node_modules/electron/dist/chrome-sandbox + xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run coverage:tutorial - name: Archive E2E Test Screenshots uses: actions/upload-artifact@v4 diff --git a/.github/workflows/testing_pipelines.yml b/.github/workflows/testing_pipelines.yml index 8892eacd0..400bc20b2 100644 --- a/.github/workflows/testing_pipelines.yml +++ b/.github/workflows/testing_pipelines.yml @@ -74,24 +74,29 @@ jobs: - name: Get ephy_testing_data current head hash id: ephys run: echo "::set-output name=HASH_EPHY_DATASET::$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" + - name: Cache ephys dataset - ${{ steps.ephys.outputs.HASH_EPHY_DATASET }} uses: actions/cache@v4 id: cache-ephys-datasets with: path: ./ephy_testing_data key: ephys-datasets-${{ matrix.os }}-${{ steps.ephys.outputs.HASH_EPHY_DATASET }} + - name: Get ophys_testing_data current head hash id: ophys run: echo "::set-output name=HASH_OPHYS_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/ophys_testing_data.git HEAD | cut -f1)" + - name: Cache ophys dataset - ${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} uses: actions/cache@v4 id: cache-ophys-datasets with: path: ./ophys_testing_data key: ophys-datasets-${{ matrix.os }}-${{ steps.ophys.outputs.HASH_OPHYS_DATASET }} + - name: Get behavior_testing_data current head hash id: behavior run: echo "::set-output name=HASH_BEHAVIOR_DATASET::$(git ls-remote https://gin.g-node.org/CatalystNeuro/behavior_testing_data.git HEAD | cut -f1)" + - name: Cache behavior dataset - ${{ steps.behavior.outputs.HASH_BEHAVIOR_DATASET }} uses: actions/cache@v4 id: cache-behavior-datasets diff --git a/docs/conf_extlinks.py b/docs/conf_extlinks.py index 2ccfc42c3..d26b3ab88 100644 --- a/docs/conf_extlinks.py +++ b/docs/conf_extlinks.py @@ -1,6 +1,6 @@ # Use this for mapping to external links extlinks = { - "linux-latest": ("https://github.com/neurodatawithoutborders/nwb-guide/tree/linux-latest/%s", "%s"), + "linux-dev": ("https://github.com/neurodatawithoutborders/nwb-guide/tree/linux-dev/%s", "%s"), "releases": ("https://github.com/NeurodataWithoutBorders/nwb-guide/releases/%s", "%s"), "pynwb-docs": ("https://pynwb.readthedocs.io/en/stable/%s", "%s"), "matnwb-src": ("https://github.com/NeurodataWithoutBorders/matnwb/%s", "%s"), diff --git a/docs/installation.rst b/docs/installation.rst index 78d430f2e..a85d560fa 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -27,9 +27,9 @@ Download and run `NWB-GUIDE-arm64.dmg ` branch of the NWB GUIDE and follow the :ref:`Developer Installation instructions` after the "Clone the Repo" step. +Please clone the :linux-dev:`linux-dev <>` branch of the NWB GUIDE and follow the :ref:`Developer Installation instructions` after the "Clone the Repo" step. .. code-block:: bash - git clone --branch linux-latest https://github.com/NeurodataWithoutBorders/nwb-guide + git clone --branch linux-dev https://github.com/NeurodataWithoutBorders/nwb-guide cd nwb-guide diff --git a/environments/environment-Linux.yml b/environments/environment-Linux.yml index 68f726a96..701678242 100644 --- a/environments/environment-Linux.yml +++ b/environments/environment-Linux.yml @@ -7,7 +7,7 @@ dependencies: - nodejs = 18.16.1 - numcodecs = 0.11.0 # install these from conda-forge so that dependent packages get included in the distributable - - jsonschema = 4.18.0 # installs jsonschema-specifications + - jsonschema = 4.18.0 # installs jsonschema-specifications - pip - pip: - chardet == 5.1.0 @@ -15,8 +15,11 @@ dependencies: - flask == 2.3.2 - flask-cors == 4.0.0 - flask_restx == 1.1.0 - - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. + - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5 - - scikit-learn == 1.4.0 # Tutorial data generation - - tqdm_publisher >= 0.0.1 # Progress bars - - tzlocal >= 5.2 # Frontend timezone handling + - scikit-learn == 1.4.0 # Tutorial data generation + - tqdm_publisher >= 0.0.1 # Progress bars + - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - nwbinspector == 0.6.1 + - tables diff --git a/environments/environment-MAC-apple-silicon.yml b/environments/environment-MAC-apple-silicon.yml index 6f24dbb6e..da16ed03f 100644 --- a/environments/environment-MAC-apple-silicon.yml +++ b/environments/environment-MAC-apple-silicon.yml @@ -7,11 +7,11 @@ dependencies: - nodejs = 18.16.1 # Install these from conda-forge so that dependent packages get included in the distributable - numcodecs = 0.11.0 - - lxml = 4.9.3 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge - - pyedflib = 0.1.32 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge - - numpy # May have x64/arm64 mismatch issues so install from conda-forge + - lxml = 4.9.3 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge + - pyedflib = 0.1.32 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge + - numpy # May have x64/arm64 mismatch issues so install from conda-forge - pytables = 3.8 # PyPI build fails on arm64 so install from conda-forge (used by neuroconv deps) - - jsonschema = 4.18.0 # Also installs jsonschema-specifications + - jsonschema = 4.18.0 # Also installs jsonschema-specifications - pip - pip: - setuptools==70.0.0 @@ -22,7 +22,11 @@ dependencies: - flask-cors == 4.0.0 - flask_restx == 1.1.0 - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. - - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5 - scikit-learn == 1.4.0 # Tutorial data generation + # NOTE: the NeuroConv wheel on PyPI includes sonpy which is not compatible with arm64, so build and install + # NeuroConv from GitHub, which will remove the sonpy dependency when building from Mac arm64 + - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5 - scikit-learn == 1.4.0 # Tutorial data generation - tqdm_publisher >= 0.0.1 # Progress bars - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - nwbinspector == 0.6.1 diff --git a/environments/environment-MAC-intel.yml b/environments/environment-MAC-intel.yml index eed92cb90..a1681da4e 100644 --- a/environments/environment-MAC-intel.yml +++ b/environments/environment-MAC-intel.yml @@ -7,7 +7,7 @@ dependencies: - nodejs = 18.16.1 - numcodecs = 0.11.0 # install these from conda-forge so that dependent packages get included in the distributable - - jsonschema = 4.18.0 # installs jsonschema-specifications + - jsonschema = 4.18.0 # installs jsonschema-specifications - pip - pip: - setuptools==70.0.0 @@ -19,7 +19,10 @@ dependencies: - flask-cors == 4.0.0 - flask_restx == 1.1.0 - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask. - - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5 - scikit-learn == 1.4.0 # Tutorial data generation + - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.5 - scikit-learn == 1.4.0 # Tutorial data generation - tqdm_publisher >= 0.0.1 # Progress bars - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - nwbinspector == 0.6.1 + - tables diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml index 2d606abf1..6176fa4c4 100644 --- a/environments/environment-Windows.yml +++ b/environments/environment-Windows.yml @@ -12,7 +12,7 @@ dependencies: - pip: - setuptools==70.0.0 - PyInstaller==6.7.0 - - pyinstaller-hooks-contrib == 2024.6 # Fix needed for pydantic v2; otherwise imports pydantic.compiled + - pyinstaller-hooks-contrib == 2024.6 # Fix needed for pydantic v2; otherwise imports pydantic.compiled - chardet == 5.1.0 - configparser == 6.0.0 - flask == 2.3.2 @@ -23,3 +23,6 @@ dependencies: - scikit-learn == 1.4.0 # Tutorial data generation - tqdm_publisher >= 0.0.1 # Progress bars - tzlocal >= 5.2 # Frontend timezone handling + - ndx-pose == 0.1.1 + - nwbinspector ==0.6.1 + - tables diff --git a/src/electron/frontend/assets/css/main.css b/src/electron/frontend/assets/css/main.css index e1d791579..ab90e07c9 100644 --- a/src/electron/frontend/assets/css/main.css +++ b/src/electron/frontend/assets/css/main.css @@ -310,8 +310,8 @@ button[disabled] { padding: 3px 10px; background-color: #fff; font-size: 13px; - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, - "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", + sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; } .menu.file { @@ -324,8 +324,8 @@ button[disabled] { background-color: #fff; color: black; font-size: 13px; - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, - "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", + sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; } .menu.file li:hover { @@ -355,8 +355,8 @@ button[disabled] { padding: 3px 10px; background-color: #fff; font-size: 13px; - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, - "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", + sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; } .menu.reg-folder.off { diff --git a/src/example_pipelines.yml b/src/example_pipelines.yml index cd5803376..fe8d11541 100644 --- a/src/example_pipelines.yml +++ b/src/example_pipelines.yml @@ -26,80 +26,66 @@ SpikeGLX-Phy: PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 - SpikeGLX_v1_SingleProbe_AP: SpikeGLXRecordingInterface: file_path: ephy_testing_data/spikeglx/Noise4Sam_g0/Noise4Sam_g0_imec0/Noise4Sam_g0_t0.imec0.ap.bin - SpikeGLXConverter-Phy: SpikeGLXConverterPipe: folder_path: ephy_testing_data/spikeglx/Noise4Sam_g0 PhySortingInterface: folder_path: ephy_testing_data/phy/phy_example_0 - Suite2P_SinglePlane_SingleChannel: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: Suite2pSegmentationInterface: folder_path: ophys_testing_data/segmentation_datasets/suite2p - Caiman_SinglePlane_SingleChannel: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: CaimanSegmentationInterface: file_path: ophys_testing_data/segmentation_datasets/caiman/caiman_analysis.hdf5 - ScanImage_SinglePlane_SingleChannel: - metadata: Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: ScanImageImagingInterface: file_path: ophys_testing_data/imaging_datasets/ScanImage/sample_scanimage_version_3_8.tiff - TDT: metadata: - NWBFile: - session_start_time: 2024-05-09T00:00 + NWBFile: + session_start_time: 2024-05-09T00:00 interfaces: TdtRecordingInterface: folder_path: ephy_testing_data/tdt/aep_05 gain: 1 - CellExplorer: metadata: NWBFile: @@ -109,7 +95,6 @@ CellExplorer: CellExplorerRecordingInterface: folder_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed - CellExplorerUnits: metadata: NWBFile: @@ -121,7 +106,6 @@ CellExplorerUnits: CellExplorerSortingInterface: file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat - CellExplorerSortingOnly: metadata: NWBFile: @@ -131,24 +115,19 @@ CellExplorerSortingOnly: CellExplorerSortingInterface: file_path: ephy_testing_data/cellexplorer/dataset_4/Peter_MS22_180629_110319_concat_stubbed/Peter_MS22_180629_110319_concat_stubbed.spikes.cellinfo.mat - IntanSingleFileRecording: metadata: NWBFile: session_start_time: 2024-05-09T00:00 Ecephys: ElectrodeGroup: - - - location: Test location - - - location: Test location - - - location: Test location + - location: Test location + - location: Test location + - location: Test location interfaces: IntanRecordingInterface: file_path: ephy_testing_data/intan/intan_rhd_test_1.rhd - MEArec: metadata: NWBFile: @@ -158,22 +137,18 @@ MEArec: MEArecRecordingInterface: file_path: ephy_testing_data/mearec/mearec_test_10s.h5 - BrukerTiffMultiChannelSinglePlane: metadata: Ophys: ImagingPlane: - - - excitation_lambda: NaN + - excitation_lambda: NaN optical_channel: - - - emission_lambda: NaN + - emission_lambda: NaN interfaces: BrukerTiffSinglePlaneConverter: folder_path: ophys_testing_data/imaging_datasets/BrukerTif/NCCR32_2023_02_20_Into_the_void_t_series_baseline-000 - DeepLabCut: metadata: NWBFile: @@ -181,9 +156,8 @@ DeepLabCut: interfaces: DeepLabCutInterface: - file_path: behavior_testing_data/DLC/m3v1mp4DLC_resnet50_openfieldAug20shuffle1_30000.h5 - config_file_path: behavior_testing_data/DLC/config.yaml - + file_path: behavior_testing_data/DLC/open_field_with_video/m3v1mp4DLC_resnet50_openfieldOct30shuffle1_1000.h5 + config_file_path: behavior_testing_data/DLC/open_field_with_video/config.yaml SLEAP: metadata: diff --git a/tests/e2e/puppeteer.ts b/tests/e2e/puppeteer.ts index 2548b49e3..a51cda6c9 100644 --- a/tests/e2e/puppeteer.ts +++ b/tests/e2e/puppeteer.ts @@ -50,7 +50,8 @@ export const connect = () => { }); const browserURL = `http://localhost:${electronDebugPort}` - const browser = output.browser = await puppeteer.launch({ headless: 'new' }) + // See notes on no sandbox mode: https://pptr.dev/troubleshooting#setting-up-chrome-linux-sandbox + const browser = output.browser = await puppeteer.launch({ headless: 'new', args: ["--no-sandbox", "--disable-setuid-sandbox"] }) const page = output.page = await browser.newPage(); await page.goto(browserURL); const endpoint = await page.evaluate(() => fetch(`json/version`).then(res => res.json()).then(res => res.webSocketDebuggerUrl))