FEAT: Slurm Deployment For Xorbits

.github/workflows/cluster.yaml

@@ -0,0 +1,60 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        jobqueue: ["slurm"]
+
+    steps:
+      - name: Cancel previous runs
+        uses: styfle/[email protected]
+        with:
+            access_token: ${{ github.token }}
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Setup Empty Conda Environment with Mamba
+        if: matrix.jobqueue == 'none'
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          channels: conda-forge
+          mamba-version: "*"
+          activate-environment: xorbits
+          auto-activate-base: false
+
+      - name: Setup xorbits conda environment 
+        if: matrix.jobqueue == 'none'
+        run: |
+          mamba env update -f CI/requirements-wheel.txt
+          mamba list
+
+      - name: Setup Job queuing system
+        if: matrix.jobqueue != 'none'
+        run: |
+          source CI/${{ matrix.jobqueue }}.sh
+          jobqueue_before_install
+
+      - name: Install xorbits
+        run: |
+          source CI/${{ matrix.jobqueue }}.sh
+          jobqueue_install
+
+      - name: Test
+        run: |
+          source CI/${{ matrix.jobqueue }}.sh
+          jobqueue_script
+
+      - name: Cleanup
+        if: always()
+        run: |
+          source CI/${{ matrix.jobqueue }}.sh
+          jobqueue_after_script

.gitignore

@@ -5,7 +5,7 @@ __pycache__/
 
 # C extensions
 *.so
-
+.pre-commit-config.yaml
 # Distribution / packaging
 .Python
 build/

.pre-commit-config.yaml

@@ -26,11 +26,6 @@ repos:
         additional_dependencies: [tokenize-rt==3.2.0]
         exclude: _mars
         args: [--ignore-missing-imports, --follow-imports, skip]
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0 # Use the sha or tag you want to point at
-    hooks:
-      - id: prettier
-        types_or: [html, javascript]
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.5
     hooks:

CI/slurm.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+function jobqueue_before_install {
+    docker version
+    docker-compose version
+
+    # start slurm cluster
+    cd ./CI/slurm
+    docker-compose pull
+    ./start-slurm.sh
+    cd -
+
+    #Set shared space permissions
+    docker exec slurmctld /bin/bash -c "chmod -R 777 /shared_space"
+
+    docker ps -a
+    docker images
+    show_network_interfaces
+}
+
+function show_network_interfaces {
+    for c in slurmctld c1 c2; do
+        echo '------------------------------------------------------------'
+        echo docker container: $c
+        docker exec $c python -c 'import psutil; print(psutil.net_if_addrs().keys())'
+        echo '------------------------------------------------------------'
+    done
+}
+
+function jobqueue_install {
+    docker exec slurmctld /bin/bash -c "cd xorbits/python/; pip install -e ."
+}
+
+function jobqueue_script {
+    docker exec c1 /bin/bash -c "pip install xorbits"
+    docker exec c2 /bin/bash -c "pip install xorbits"
+    docker exec slurmctld /bin/bash -c "python /xorbits/python/xorbits/deploy/cluster/Slurm.py"
+    docker exec slurmctld /bin/bash -c "cat /shared_space/output.out"
+}
+
+function jobqueue_after_script {
+    docker exec slurmctld bash -c 'sinfo'
+    docker exec slurmctld bash -c 'squeue'
+    docker exec slurmctld bash -c 'sacct -l'
+}

CI/slurm/Dockerfile

@@ -0,0 +1,2 @@
+FROM daskdev/dask-jobqueue:slurm
+RUN pip install xorbits

CI/slurm/docker-compose.yml

@@ -0,0 +1,120 @@
+version: "2.2"
+
+services:
+  mysql:
+    image: mysql:5.7.29
+    hostname: mysql
+    container_name: mysql
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: "yes"
+      MYSQL_DATABASE: slurm_acct_db
+      MYSQL_USER: slurm
+      MYSQL_PASSWORD: password
+    volumes:
+      - var_lib_mysql:/var/lib/mysql
+    networks:
+      common-network:
+
+  slurmdbd:
+    image: daskdev/dask-jobqueue:slurm
+    build: .
+    command: ["slurmdbd"]
+    container_name: slurmdbd
+    hostname: slurmdbd
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - var_log_slurm:/var/log/slurm
+    expose:
+      - "6819"
+    depends_on:
+      - mysql
+    networks:
+      common-network:
+
+  slurmctld:
+    image: daskdev/dask-jobqueue:slurm
+    build: .
+    command: ["slurmctld"]
+    container_name: slurmctld
+    hostname: slurmctld
+    environment:
+      - CI_SHARED_SPACE=/shared_space
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - slurm_jobdir:/data
+      - var_log_slurm:/var/log/slurm
+      - ../..:/xorbits
+      - shared_space:/shared_space
+    expose:
+      - "6817"
+    depends_on:
+      - "slurmdbd"
+    networks:
+      common-network:
+        ipv4_address: 10.1.1.10
+    cap_add:
+      - NET_ADMIN
+
+  c1:
+    image: daskdev/dask-jobqueue:slurm
+    build: .
+    command: ["slurmd"]
+    hostname: c1
+    container_name: c1
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - slurm_jobdir:/data
+      - var_log_slurm:/var/log/slurm
+      - ../..:/xorbits
+      - shared_space:/shared_space
+    expose:
+      - "6818"
+    depends_on:
+      - "slurmctld"
+    networks:
+      common-network:
+        ipv4_address: 10.1.1.11
+    cap_add:
+      - NET_ADMIN
+
+  c2:
+    image: daskdev/dask-jobqueue:slurm
+    build: .
+    command: ["slurmd"]
+    hostname: c2
+    container_name: c2
+    volumes:
+      - etc_munge:/etc/munge
+      - etc_slurm:/etc/slurm
+      - slurm_jobdir:/data
+      - var_log_slurm:/var/log/slurm
+      - ../..:/xorbits
+      - shared_space:/shared_space
+    expose:
+      - "6818"
+    depends_on:
+      - "slurmctld"
+    networks:
+      common-network:
+        ipv4_address: 10.1.1.12
+    cap_add:
+      - NET_ADMIN
+
+volumes:
+  etc_munge:
+  etc_slurm:
+  slurm_jobdir:
+  var_lib_mysql:
+  var_log_slurm:
+  shared_space:
+
+networks:
+  common-network:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 10.1.1.0/24

CI/slurm/register_cluster.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+
+docker exec slurmctld bash -c "/usr/bin/sacctmgr --immediate add cluster name=linux" && \
+docker-compose restart slurmdbd slurmctld

CI/slurm/slurm.conf

@@ -0,0 +1,98 @@
+# slurm.conf
+#
+# See the slurm.conf man page for more information.
+#
+ClusterName=linux
+ControlMachine=slurmctld
+ControlAddr=slurmctld
+#BackupController=
+#BackupAddr=
+#
+SlurmUser=slurm
+#SlurmdUser=root
+SlurmctldPort=6817
+SlurmdPort=6818
+AuthType=auth/munge
+#JobCredentialPrivateKey=
+#JobCredentialPublicCertificate=
+StateSaveLocation=/var/lib/slurmd
+SlurmdSpoolDir=/var/spool/slurmd
+SwitchType=switch/none
+MpiDefault=none
+SlurmctldPidFile=/var/run/slurmd/slurmctld.pid
+SlurmdPidFile=/var/run/slurmd/slurmd.pid
+ProctrackType=proctrack/linuxproc
+#PluginDir=
+CacheGroups=0
+#FirstJobId=
+ReturnToService=0
+#MaxJobCount=
+#PlugStackConfig=
+#PropagatePrioProcess=
+#PropagateResourceLimits=
+#PropagateResourceLimitsExcept=
+#Prolog=
+#Epilog=
+#SrunProlog=
+#SrunEpilog=
+#TaskProlog=
+#TaskEpilog=
+#TaskPlugin=
+#TrackWCKey=no
+#TreeWidth=50
+#TmpFS=
+#UsePAM=
+#
+# TIMERS
+SlurmctldTimeout=300
+SlurmdTimeout=300
+InactiveLimit=0
+MinJobAge=300
+KillWait=300
+Waittime=30
+#change this avoids low resource kill the process   
+#**log**
+#srun: Job step aborted: Waiting up to 32 seconds for job step to finish.
+#slurmstepd: error: *** STEP 27.0 ON c1 CANCELLED AT 2023-09-25T06:30:54 ***
+
+# SCHEDULING
+SchedulerType=sched/backfill
+#SchedulerAuth=
+#SchedulerPort=
+#SchedulerRootFilter=
+SelectType=select/cons_res
+SelectTypeParameters=CR_CPU_Memory
+FastSchedule=1
+#PriorityType=priority/multifactor
+#PriorityDecayHalfLife=14-0
+#PriorityUsageResetPeriod=14-0
+#PriorityWeightFairshare=100000
+#PriorityWeightAge=1000
+#PriorityWeightPartition=10000
+#PriorityWeightJobSize=1000
+#PriorityMaxAge=1-0
+#
+# LOGGING
+SlurmctldDebug=3
+SlurmctldLogFile=/var/log/slurm/slurmctld.log
+SlurmdDebug=3
+SlurmdLogFile=/var/log/slurm/slurmd.log
+JobCompType=jobcomp/filetxt
+JobCompLoc=/var/log/slurm/jobcomp.log
+#
+# ACCOUNTING
+JobAcctGatherType=jobacct_gather/linux
+JobAcctGatherFrequency=30
+#
+AccountingStorageType=accounting_storage/slurmdbd
+AccountingStorageHost=slurmdbd
+AccountingStoragePort=6819
+AccountingStorageLoc=slurm_acct_db
+#AccountingStoragePass=
+#AccountingStorageUser=
+#
+# COMPUTE NODES
+NodeName=c[1-2] RealMemory=4096 CPUs=2 State=UNKNOWN
+#
+# PARTITIONS
+PartitionName=normal Default=yes Nodes=c[1-2] Priority=50 DefMemPerCPU=2048 Shared=NO MaxNodes=2 MaxTime=5-00:00:00 DefaultTime=5-00:00:00 State=UP

CI/slurm/start-slurm.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+docker-compose up -d --no-build
+
+while [ `./register_cluster.sh 2>&1 | grep "sacctmgr: error" | wc -l` -ne 0 ]
+  do
+    echo "Waiting for SLURM cluster to become ready";
+    sleep 2
+  done
+echo "SLURM properly configured"
+
+# On some clusters the login node does not have the same interface as the
+# compute nodes. The next three lines allow to test this edge case by adding
+# separate interfaces on the worker and on the scheduler nodes.
+docker exec slurmctld ip addr add 10.1.1.20/24 dev eth0 label eth0:scheduler
+docker exec c1 ip addr add 10.1.1.21/24 dev eth0 label eth0:worker
+docker exec c2 ip addr add 10.1.1.22/24 dev eth0 label eth0:worker