Update docs for memory rebalance #1347
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: neurodamus build and test | |
| on: | |
| schedule: | |
| # every evening at 20:00 UTC | |
| - cron: '0 20 * * *' | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| NEURON_BRANCH: | |
| description: 'NEURON branch to use' | |
| required: false | |
| LIBSONATA_REPORT_BRANCH: | |
| description: 'libsonatareport branch to use' | |
| required: false | |
| LIBSONATA_BRANCH: | |
| description: 'libsonata branch to use' | |
| required: false | |
| NEURODAMUS_MODELS_BRANCH: | |
| description: 'neurodamus-models branch to use' | |
| required: false | |
| env: | |
| NEURON_COMMIT_ID: 'c48d7d5' | |
| RDMAV_FORK_SAFE: '1' | |
| jobs: | |
| simulation: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-22.04] | |
| python-version: ['3.9.20', '3.10.15', '3.11.10', '3.12.6'] | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Checkout neurodamus repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 0 | |
| - name: Cache pip | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ matrix.os }}-pip-${{ matrix.python-version }}-v1 | |
| restore-keys: | | |
| ${{ matrix.os }}-pip- | |
| - name: Get latest tags | |
| run: | | |
| echo "NEURON_BRANCH=${{ inputs.NEURON_BRANCH || github.event.inputs.NEURON_BRANCH || 'master' }}" >> $GITHUB_ENV | |
| LIBSONATA_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonata/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")') | |
| echo "LIBSONATA_BRANCH=${{ inputs.LIBSONATA_BRANCH || github.event.inputs.LIBSONATA_BRANCH || '$LIBSONATA_LATEST' }}" >> $GITHUB_ENV | |
| LIBSONATA_REPORT_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/libsonatareport/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")') | |
| echo "LIBSONATA_REPORT_BRANCH=${{ inputs.LIBSONATA_REPORT_BRANCH || github.event.inputs.LIBSONATA_REPORT_BRANCH || '$LIBSONATA_REPORT_LATEST' }}" >> $GITHUB_ENV | |
| NEURODAMUS_MODELS_LATEST=$(curl -s https://api.github.com/repos/BlueBrain/neurodamus-models/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")') | |
| echo "NEURODAMUS_MODELS_BRANCH=${{ inputs.NEURODAMUS_MODELS_BRANCH || github.event.inputs.NEURODAMUS_MODELS_BRANCH || '$NEURODAMUS_MODELS_LATEST' }}" >> $GITHUB_ENV | |
| - name: Get HEAD commit message and look for branches | |
| run: | | |
| COMMIT_MESSAGE=$(git log -1 --pretty=%B) | |
| echo $COMMIT_MESSAGE | |
| NEURON_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'NEURON_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true) | |
| if [[ ! -z $NEURON_BRANCH ]]; then echo "NEURON_BRANCH=$NEURON_BRANCH" >> $GITHUB_ENV; fi | |
| LIBSONATA_REPORT_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'LIBSONATA_REPORT_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true) | |
| if [[ ! -z $LIBSONATA_REPORT_BRANCH ]]; then echo "LIBSONATA_REPORT_BRANCH=$LIBSONATA_REPORT_BRANCH" >> $GITHUB_ENV; fi | |
| LIBSONATA_BRANCH=$(echo $COMMIT_MESSAGE | grep -Po 'LIBSONATA_BRANCH=\K[0-9a-zA-Z/_.\-]*' || true) | |
| if [[ ! -z $LIBSONATA_BRANCH ]]; then echo "LIBSONATA_BRANCH=$LIBSONATA_BRANCH" >> $GITHUB_ENV; fi | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y mpich libmpich-dev libhdf5-mpich-dev hdf5-tools flex libfl-dev bison ninja-build | |
| - name: Cache python virtual env | |
| id: cache-venv | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-venv | |
| with: | |
| path: venv | |
| key: ${{ matrix.os }}-libsonata-${{ env.LIBSONATA_BRANCH }}-py${{ matrix.python-version }} | |
| - name: Upgrade pip and install base Python packages | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| run: | | |
| python -m venv venv | |
| . ./venv/bin/activate | |
| python -m pip install --upgrade pip setuptools | |
| pip install cython numpy wheel pkgconfig | |
| - name: Add virtual environment to PATH | |
| run: echo "${{ github.workspace }}/venv/bin" >> $GITHUB_PATH | |
| - name: Install libsonata | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| run: | | |
| CC=mpicc CXX=mpic++ pip install git+https://github.com/BlueBrain/libsonata@${{ env.LIBSONATA_BRANCH }} | |
| - name: Cache libsonatareport | |
| id: cache-libsonatareport | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-libsonatareport | |
| with: | |
| path: libsonatareport | |
| key: ${{ matrix.os }}-libsonatareport-${{ env.LIBSONATA_REPORT_BRANCH }}-py${{ matrix.python-version }} | |
| - name: Install libsonatareport | |
| if: steps.cache-libsonatareport.outputs.cache-hit != 'true' | |
| run: | | |
| git clone --branch="${{ env.LIBSONATA_REPORT_BRANCH}}" https://github.com/BlueBrain/libsonatareport.git --recursive --depth=1 | |
| cd libsonatareport | |
| mkdir build && cd build | |
| cmake -DCMAKE_INSTALL_PREFIX=$(pwd)/install -DCMAKE_BUILD_TYPE=Release -DSONATA_REPORT_ENABLE_SUBMODULES=ON -DSONATA_REPORT_ENABLE_MPI=ON .. | |
| cmake --build . --parallel | |
| cmake --build . --target install | |
| - name: Cache NEURON | |
| id: cache-neuron | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-neuron | |
| with: | |
| path: nrn | |
| key: ${{ matrix.os }}-neuron-${{ env.NEURON_BRANCH }}-${{ env.NEURON_COMMIT_ID }}-py${{ matrix.python-version }} | |
| - name: Install NEURON | |
| if: steps.cache-neuron.outputs.cache-hit != 'true' | |
| run: | | |
| export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install | |
| if [[ ${{ env.NEURON_BRANCH }} == 'master' ]]; then | |
| git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git | |
| cd nrn | |
| git checkout ${{ env.NEURON_COMMIT_ID }} | |
| cd .. | |
| else | |
| git clone --branch=${{ env.NEURON_BRANCH }} https://github.com/neuronsimulator/nrn.git --depth=1 | |
| fi | |
| python -m pip install --upgrade pip -r nrn/nrn_requirements.txt | |
| cmake -B nrn/build -S nrn -G Ninja \ | |
| -DPYTHON_EXECUTABLE=$(which python) \ | |
| -DCMAKE_INSTALL_PREFIX=$(pwd)/nrn/build/install \ | |
| -DNRN_ENABLE_MPI=ON \ | |
| -DNRN_ENABLE_INTERVIEWS=OFF \ | |
| -DNRN_ENABLE_CORENEURON=ON \ | |
| -DCMAKE_C_COMPILER=gcc \ | |
| -DCMAKE_CXX_COMPILER=g++ \ | |
| -DCORENRN_ENABLE_REPORTING=ON \ | |
| -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR | |
| cmake --build nrn/build --parallel | |
| cmake --build nrn/build --target install | |
| - name: Install mpi4py | |
| if: steps.cache-venv.outputs.cache-hit != 'true' | |
| run: | | |
| pip install mpi4py | |
| - name: Build h5py with the local hdf5 | |
| run: | | |
| CC="mpicc" HDF5_MPI="ON" HDF5_INCLUDEDIR=/usr/include/hdf5/mpich HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich \ | |
| pip install --no-cache-dir --no-binary=h5py h5py --no-build-isolation | |
| - name: Install neurodamus | |
| run: | | |
| pip install .[full] | |
| - name: Build neocortex models | |
| run: | | |
| export SONATAREPORT_DIR=$(pwd)/libsonatareport/build/install | |
| export PATH=$(pwd)/nrn/build/install/bin:$PATH | |
| # Clone neurodamus-models repository | |
| git clone --branch=${{ env.NEURODAMUS_MODELS_BRANCH }} https://github.com/BlueBrain/neurodamus-models.git | |
| # Build neocortex model | |
| DATADIR=$(python -c "import neurodamus; from pathlib import Path; print(Path(neurodamus.__file__).parent / 'data')") | |
| cmake -B neurodamus-models/build -S neurodamus-models/ \ | |
| -DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install \ | |
| -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ | |
| -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \ | |
| -DNEURODAMUS_CORE_DIR=${DATADIR} \ | |
| -DNEURODAMUS_MECHANISMS=neocortex \ | |
| -DNEURODAMUS_NCX_V5=ON | |
| cmake --build neurodamus-models/build | |
| cmake --install neurodamus-models/build | |
| echo "NEURODAMUS_NEOCORTEX_ROOT=$(pwd)/neurodamus-models/install" >> $GITHUB_ENV; | |
| # Build neocortex-multiscale model | |
| echo "build neurodamus-neocortex-multiscale model" | |
| cmake -B neurodamus-models/build_multiscale -S neurodamus-models/ \ | |
| -DCMAKE_INSTALL_PREFIX=$PWD/neurodamus-models/install_multiscale \ | |
| -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ | |
| -DCMAKE_PREFIX_PATH=$SONATAREPORT_DIR \ | |
| -DNEURODAMUS_CORE_DIR=${DATADIR} \ | |
| -DNEURODAMUS_MECHANISMS=neocortex \ | |
| -DNEURODAMUS_NCX_METABOLISM=ON \ | |
| -DNEURODAMUS_NCX_NGV=ON \ | |
| -DNEURODAMUS_ENABLE_CORENEURON=OFF | |
| cmake --build neurodamus-models/build_multiscale | |
| cmake --install neurodamus-models/build_multiscale | |
| echo "NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT=$(pwd)/neurodamus-models/install_multiscale" >> $GITHUB_ENV; | |
| - name: load neocortex model | |
| run: | | |
| export PYTHONPATH=$(pwd)/nrn/build/install/lib/python:$PYTHONPATH | |
| export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_ROOT/share/neurodamus_neocortex/hoc | |
| export NEURODAMUS_PYTHON=$(pwd)/neurodamus/data | |
| export CORENEURONLIB=$NEURODAMUS_NEOCORTEX_ROOT/lib/libcorenrnmech.so | |
| export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_ROOT/lib/libnrnmech.so | |
| export PATH=$NEURODAMUS_NEOCORTEX_ROOT/bin:$PATH | |
| echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV; | |
| echo "HOC_LIBRARY_PATH=$HOC_LIBRARY_PATH" >> $GITHUB_ENV; | |
| echo "CORENEURONLIB=$CORENEURONLIB" >> $GITHUB_ENV; | |
| echo "NRNMECH_LIB_PATH=$NRNMECH_LIB_PATH" >> $GITHUB_ENV; | |
| echo "NEURODAMUS_PYTHON=$NEURODAMUS_PYTHON" >> $GITHUB_ENV; | |
| which special | |
| # launch simulation with NEURON | |
| cd tests/simulations/usecase3/ | |
| mpirun -np 2 special -mpi -python $NEURODAMUS_PYTHON/init.py --configFile=simulation_sonata.json | |
| ls reporting/*.h5 | |
| # launch simulation with CORENEURON | |
| mpirun -np 2 special -mpi -python $NEURODAMUS_PYTHON/init.py --configFile=simulation_sonata_coreneuron.json | |
| ls reporting_coreneuron/*.h5 | |
| - name: Install test dependencies | |
| run: | | |
| pip install -r tests/requirements.txt | |
| - name: test-unit | |
| run: | | |
| pytest tests/unit | |
| - name: test-integration | |
| run: | | |
| pytest -x --forked tests/integration | |
| - name: test-integration-e2e | |
| run: | | |
| pytest -s -x --forked --durations=5 --durations-min=15 tests/integration-e2e | |
| - name: test-scientific | |
| run: | | |
| pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific | |
| - name: test-scientific-ngv | |
| run: | | |
| export HOC_LIBRARY_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/share/neurodamus_neocortex/hoc | |
| export NRNMECH_LIB_PATH=$NEURODAMUS_NEOCORTEX_MULTISCALE_ROOT/lib/libnrnmech.so | |
| unset CORENEURONLIB | |
| pytest -s -x --forked --durations=5 --durations-min=15 tests/scientific-ngv | |
| - name: live debug session, comment out | |
| if: failure() | |
| uses: mxschmitt/action-tmate@v3 | |
| with: | |
| timeout-minutes: 60 # or any other value you need |