Windows ssh support by ipcluster #1209
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| pull_request: | |
| push: | |
| branches-ignore: | |
| - "pre-commit-ci*" | |
| concurrency: | |
| group: >- | |
| ${{ github.workflow }}- | |
| ${{ github.ref_type }}- | |
| ${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| # UTF-8 content may be interpreted as ascii and causes errors without this. | |
| LANG: C.UTF-8 | |
| IPP_DISABLE_JS: "1" | |
| JUPYTER_PLATFORM_DIRS: "1" | |
| GITHUB_RUNNER: "1" # this will disable the ssh test on windows | |
| jobs: | |
| test: | |
| runs-on: ${{ matrix.runs_on || 'ubuntu-20.04' }} | |
| timeout-minutes: 40 | |
| strategy: | |
| # Keep running even if one variation of the job fail | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - python: "3.9" | |
| cluster_type: ssh | |
| - python: "3.8" | |
| cluster_type: mpi | |
| - python: "3.10" | |
| cluster_type: slurm | |
| container: slurmctld | |
| - python: "3.8" | |
| - python: "3.10" | |
| env: | |
| IPP_CONTROLLER_IP: "*" | |
| - python: "3.9" | |
| env: | |
| IPP_ENABLE_CURVE: "1" | |
| - python: "3.8" | |
| cluster_type: ssh | |
| runs_on: windows-2019 | |
| - python: "3.9" | |
| runs_on: macos-11 | |
| - python: "3.11" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Cache conda environment | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/conda | |
| key: conda | |
| - name: Cache node_modules | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| node_modules | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Set environment variables | |
| if: ${{ matrix.env }} | |
| env: | |
| MATRIX_ENV: ${{ toJSON(matrix.env) }} | |
| run: | | |
| python3 <<EOF | |
| import json | |
| import os | |
| matrix_env = json.loads(os.environ["MATRIX_ENV"]) | |
| with open(os.environ["GITHUB_ENV"], "a") as f: | |
| for key, value in matrix_env.items(): | |
| f.write(f"{key}={value}\n") | |
| EOF | |
| - name: Set up docker-compose for ssh linux launcher | |
| if: ${{ matrix.cluster_type == 'ssh' && !contains(matrix.runs_on, 'windows') }} | |
| run: | | |
| export DOCKER_BUILDKIT=1 | |
| export COMPOSE_DOCKER_CLI_BUILD=1 | |
| cd ci/ssh | |
| docker-compose -f linux_docker-compose.yaml up -d --build | |
| # retrieve id_rsa file for public key authentication | |
| mkdir ~/.ssh/ | |
| docker cp ssh_sshd_1:/home/ciuser/.ssh/id_rsa ~/.ssh/id_rsa | |
| cat ~/.ssh/id_rsa | |
| #check ssh connection and accept host key (with an arbitrary linux command) | |
| ssh -o "StrictHostKeyChecking no" [email protected] -p 2222 -v echo "ssh connection to container succeeded" | |
| - name: Set up docker-compose for ssh windows launcher | |
| if: ${{ matrix.cluster_type == 'ssh' && contains(matrix.runs_on, 'windows') }} | |
| env: | |
| SSH_HOST: [email protected] | |
| SSH_PORT: 2222 | |
| CODE_ROOT: c:\src\ipyparallel | |
| run: | | |
| cd ci/ssh | |
| # determine host ip and place it as 'static' env variables in corresponding docker compose file (win_Dockerfile_template -> win_Dockerfile) | |
| $env:docker_host_ip=(Get-NetIPConfiguration -InterfaceAlias "Ethernet*").IPv4Address.IPAddress.Trim() | |
| $content = Get-Content "win_Dockerfile_template" | |
| $content | ForEach-Object { | |
| $_ -replace '\${docker_host_ip}', $env:docker_host_ip -replace '\${docker_host_name}', $env:computername | |
| } | Set-Content "win_Dockerfile" | |
| docker-compose -f win_docker-compose.yaml up -d --build | |
| # retrieve id_rsa file for public key authentication | |
| mkdir $env:USERPROFILE/.ssh/ | |
| docker run ipyparallel-sshd powershell.exe -Command "type C:\Users\ciuser\.ssh\id_rsa" | out-file -encoding ascii $env:USERPROFILE/.ssh/id_rsa | |
| # install newer version of openssh (removes "GetConsoleMode on STD_INPUT_HANDLE failed with 6" error in pytest) | |
| choco install openssh --pre | |
| mv C:\Windows\System32\OpenSSH C:\Windows\System32\_OpenSSH | |
| #check ssh connection and accept host key (with arbitrary windows command) | |
| ssh -o "StrictHostKeyChecking no" $env:SSH_HOST -p $env:SSH_PORT -v echo "ssh connection to container succeeded" | |
| # copy ipyparallel code to docker container (use zip, scp and unzip) | |
| ssh $env:SSH_HOST -p $env:SSH_PORT mkdir $env:CODE_ROOT | |
| # zip ipyparallel files (excluding files probably not needed) | |
| cd ../.. | |
| $exclude = @("__pycache__","node_modules") | |
| $files = Get-ChildItem -Path "." -Exclude $exclude | |
| Compress-Archive -Path $files -DestinationPath ipyparallel.zip -CompressionLevel Fastest | |
| # copy file into docker (we need to do it over ssh since docker copy or mount doesn't work in Hyper-V) | |
| scp -P $env:SSH_PORT ipyparallel.zip ${env:SSH_HOST}:${env:CODE_ROOT} | |
| # deflate ipyparallel files | |
| ssh $env:SSH_HOST -p $env:SSH_PORT powershell.exe -Command "Expand-Archive -Path $env:CODE_ROOT\ipyparallel.zip -DestinationPath $env:CODE_ROOT" | |
| # pip install ipyparallel files | |
| #ssh $env:SSH_HOST -p $env:SSH_PORT "cd $env:CODE_ROOT && pip install -e ." | |
| ssh $env:SSH_HOST -p $env:SSH_PORT "pip install -e file://c:/src/ipyparallel#egg=ipyparallel[test]" | |
| # we need to disable the windows firewall for github runners otherwise the ipyparallel engines cannot connect to the controller. | |
| # obviously, a more precautious adaption of the firewall would be desirable. since an adaption of the firewall is NOT necessary | |
| # for a local standard windows environment, no further improvements were made. | |
| echo "Disable Firewall:" | |
| Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False | |
| # just see were pip is installed and what libraries are install | |
| echo "Check pip inside container" | |
| ssh $env:SSH_HOST -p $env:SSH_PORT "where pip" | |
| ssh $env:SSH_HOST -p $env:SSH_PORT "pip list" | |
| ssh $env:SSH_HOST -p $env:SSH_PORT "python -c 'import ipyparallel;print(""""ipyparallel is ok"""")'" | |
| echo "Check if container can ping the docker host (requires adapted hosts file and firewall disabled)" | |
| docker run ipyparallel-sshd ping -n 1 $env:computername | |
| - name: Set up slurm | |
| if: ${{ matrix.cluster_type == 'slurm' }} | |
| run: | | |
| export DOCKER_BUILDKIT=1 | |
| export COMPOSE_DOCKER_CLI_BUILD=1 | |
| cd ci/slurm | |
| docker-compose up -d --build | |
| - name: Install Python (conda) ${{ matrix.python }} | |
| if: ${{ matrix.cluster_type == 'mpi' }} | |
| run: | | |
| export MAMBA_ROOT_PREFIX=$HOME/conda | |
| test -d $MAMBA_ROOT_PREFIX || mkdir $MAMBA_ROOT_PREFIX | |
| wget -qO- https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba | |
| eval "$(./bin/micromamba shell hook -s posix)" | |
| micromamba activate | |
| micromamba install -y -c conda-forge mpich mpi4py python=${{ matrix.python }} | |
| echo "PATH=$MAMBA_ROOT_PREFIX/bin:$PATH" >> $GITHUB_ENV | |
| - name: Install Python ${{ matrix.python }} | |
| if: ${{ matrix.cluster_type != 'mpi' }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install ipyparallel itself | |
| run: | | |
| pip install --upgrade pip | |
| pip install --no-deps . | |
| - name: Install Python dependencies | |
| run: | | |
| pip install --pre --upgrade ipyparallel[test] | |
| - name: Install extra Python packages | |
| if: ${{ ! startsWith(matrix.python, '3.11') }} | |
| run: | | |
| pip install distributed joblib | |
| # pip install --only-binary :all: matplotlib || echo "no matplotlib"# | |
| # the || syntax doesn't work under windows powershell, but since the command (currently) works for all | |
| # python version (3.8-3.10), the "||" branch can be simply removed. Otherwise we would need a splitted | |
| # step for Windows and Linux (as for the docker-compose command) | |
| pip install --only-binary :all: matplotlib | |
| - name: Show environment | |
| run: pip freeze | |
| - name: Run ${{ matrix.cluster_type }} tests in container ${{ matrix.container }} | |
| if: ${{ matrix.cluster_type && matrix.container }} | |
| run: | | |
| EXEC='docker exec -i ${{ matrix.container }}' | |
| echo "EXEC=${EXEC}" >> $GITHUB_ENV | |
| ${EXEC} pytest -v --maxfail=2 --cov=ipyparallel ipyparallel/tests/test_${{ matrix.cluster_type }}.py | |
| - name: Run ${{ matrix.cluster_type }} tests | |
| if: ${{ matrix.cluster_type && !matrix.container }} | |
| run: | | |
| # ${EXEC:-} syntax doesn't work under windows. hence, the slurm test was move to the previous step | |
| pytest -v --maxfail=2 --cov=ipyparallel ipyparallel/tests/test_${{ matrix.cluster_type }}.py | |
| - name: Run tests | |
| if: ${{ ! matrix.cluster_type }} | |
| run: | | |
| pytest -v --maxfail=3 --cov=ipyparallel | |
| - name: Fixup coverage permissions ${{ matrix.container }} | |
| if: ${{ matrix.container }} | |
| run: | | |
| ls -l .coverage* | |
| ${EXEC} chmod -R a+rw .coverage* | |
| - name: Submit codecov report | |
| uses: codecov/codecov-action@v4 | |
| - name: Report on slurm | |
| if: ${{ matrix.cluster_type == 'slurm' && failure() }} | |
| run: | | |
| set -x | |
| docker ps -a | |
| docker logs slurmctld | |
| docker exec -i slurmctld squeue --states=all | |
| docker exec -i slurmctld sinfo | |
| docker logs c1 | |
| docker logs c2 |