Add script and service to automatically update computer resources (#6)

.gitignore

@@ -1,3 +1,5 @@
 *~
 *.swp
+.DS_Store
+.vscode/
 .galaxy_install_info

defaults/main.yml

@@ -1,6 +1,5 @@
 ---
 slurm_hostname: qmobile
-slurm_cpus: 2
 # Size of real memory on the node in megabytes
 slurm_memory: 1024
 slurm_user: slurm
@@ -12,3 +11,8 @@ slurm_test_folder: "/tmp/slurm-tests"
 # Adds a system service that forces the hostname on startup
 # Note: This is e.g. necessary on cloud platforms like AWS
 slurm_hostname_service: false
+slurm_set_hostname: false
+
+# Adds a system service that re-configures the slurm compute resources on startup
+# Note: This is necessary when preparing image that can start on a different hardware than it was built on
+slurm_resources_service: false

files/slurm-resources.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:          set-resources
+# Required-Start:    $remote_fs $syslog 
+# Required-Stop:     $remote_fs $syslog 
+# X-Start-Before:    slurmctld slurmd
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Set SLURM compute resources
+# Description:       Set SLURM compute resources
+### END INIT INFO
+
+run_update ()
+{
+# TODO ideally we would also output stderr to the log file
+# but for some reason 2>&1 makes $* in slurm-update-resources also read the log path as a variable
+export ANSIBLE_LOCAL_TEMP=$HOME/.ansible/tmp
+export ANSIBLE_REMOTE_TEMP=$HOME/.ansible/tmp
+/usr/bin/slurm-update-resources -v
+}
+
+case "$1" in
+  start)
+    run_update
+    ;;
+  stop)
+    echo "stopped"
+    ;;
+  restart)
+    run_update
+    ;;
+  try-restart|condrestart)
+    run_update
+    ;;
+  reload)
+    run_update
+    ;;
+  force-reload)
+    run_update
+    ;;
+  status)
+    echo "no status to report"
+    ;;
+  *)
+    COMMANDS="start|stop|restart|try-restart|reload|force-reload|status"
+    echo "Usage: `basename \"$0\"` ($COMMANDS)" >&2
+    exit 2                              # invalid or excess argument(s)
+    ;;
+esac

files/update-slurm-resources.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
+  echo "This script calls ansible-playbook to update the slurm configuration"
+  echo "You can add options such as:"
+  echo ""
+  echo "- Restart SLURM (for changes to take affect): -e restart_slurm=true"
+  echo "- Restart SLURM only on changes: -e restart_on_change=true"
+  echo "- Change the maximum cpus allowed: -e slurm_max_cpus=2"
+  echo "- Change the verbosity with: -vv"
+  exit 0
+fi
+
+/usr/bin/ansible-playbook /etc/slurm-llnl/update-playbook.yml $@

molecule/default/playbook.yml → molecule/default/converge.yml

@@ -10,6 +10,8 @@
   vars:
     - run_tests: true
     - cloud_platform: docker
+    - slurm_hostname_service: true
+    - slurm_resources_service: true
 
   roles:
     - role: marvel-nccr.slurm

tasks/hostname.yml

@@ -13,4 +13,4 @@
     line: "\\1 {{ slurm_hostname }}"
     backrefs: true
     unsafe_writes: true
-  when: cloud_platform is defined and cloud_platform == 'docker'
+  when: slurm_set_hostname or (cloud_platform is defined and cloud_platform == 'docker')

tasks/main.yml

@@ -1,4 +1,10 @@
 ---
+- name: add ansible repository
+  become: true
+  apt_repository:
+    repo: 'ppa:ansible/ansible'
+    state: present
+
 - name: Install apt packages
   become: true
   apt:
@@ -8,6 +14,7 @@
       - slurm-wlm-basic-plugins-dev
       - munge
       - sendmail
+      - ansible
     state: present
     update_cache: true
 
@@ -43,6 +50,23 @@
     src: slurm.conf
     owner: "{{ slurm_user }}"
     dest: /etc/slurm-llnl/slurm.conf
+  register: conf_template
+
+- name: create confinguration update playbook
+  become: true
+  template:
+    src: config-playbook.yml.j2
+    dest: /etc/slurm-llnl/update-playbook.yml
+
+- name: copy confinguration update executable
+  become: true
+  copy:
+    src: update-slurm-resources.sh
+    dest: /usr/bin/slurm-update-resources
+    mode: u=rwx,g=rx,o=rx
+
+- include_tasks: resources_service.yml
+  when: slurm_resources_service
 
 # munge key already created by apt-get install
 # will not overwrite key due to 'creates' flag

tasks/resources_service.yml

@@ -0,0 +1,22 @@
+---
+# Adds a system service that re-configures the slurm compute resources on startup
+# Note: This is necessary when preparing image that can start on a different hardware than it was built on
+
+- name: copy init.d template
+  become: true
+  copy:
+    src: "slurm-resources.sh"
+    dest: "/etc/init.d/slurm-resources"
+    mode: 0755
+
+- name: enable service to set the slurm compute resources
+  become: true
+  command: update-rc.d slurm-resources defaults
+  args:
+    creates: "/etc/rc5.d/S??slurm-resources"
+
+- name: start service to set the slurm compute resources
+  become: true
+  service:
+    name: slurm-resources
+    state: started

tasks/tests.yml

@@ -1,5 +1,10 @@
 ---
-- name: Creates directory
+- name: run configuration update script
+  become: true
+  command: /usr/bin/slurm-update-resources -e restart_on_change=true
+  changed_when: false
+
+- name: create test directory
   file:
     path: "{{ slurm_test_folder }}"
     state: directory

templates/config-playbook.yml.j2

@@ -0,0 +1,38 @@
+- name: Update SLURM configuration
+  hosts: localhost
+
+  vars:
+    slurm_hostname: "{{ slurm_hostname }}"
+    slurm_partition_name: "{{ slurm_partition_name }}"
+{% raw %}
+    slurm_conf_file: /etc/slurm-llnl/slurm.conf
+    slurm_max_cpus: "{{ ansible_processor_vcpus }}"
+    restart_on_change: false
+    restart_slurm: false
+
+  tasks:
+
+    - debug:
+        msg: "Run: {{ lookup('pipe', 'date +%Y-%m-%d-%H:%M:%S') }}"
+
+    - name: Update SLURM configuration
+      become: true
+      blockinfile:
+        path: "{{ slurm_conf_file }}"
+        marker: "# {mark} ANSIBLE MANAGED NODES"
+        block: |
+          NodeName={{ slurm_hostname }} Sockets={{ ansible_processor_count }} CoresPerSocket={{ ansible_processor_cores }} ThreadsPerCore={{ ansible_processor_threads_per_core }} State=UNKNOWN
+          PartitionName={{ slurm_partition_name }} Nodes={{ slurm_hostname }} Default=YES MaxTime=INFINITE State=UP MaxNodes=1 MaxCPUsPerNode={{ slurm_max_cpus }}
+        backup: yes
+      register: update
+
+    - name: Restart Slurm
+      when: (update.changed and restart_on_change | bool) or (restart_slurm | bool)
+      become: true
+      service:
+        name: "{{ item }}"
+        state: restarted
+      with_items:
+        - slurmctld
+        - slurmd
+{% endraw %}

templates/slurm.conf

@@ -37,7 +37,9 @@ SchedulerType=sched/backfill
 # cons_res: schedule individual cores
 SelectType=select/cons_res
 SelectTypeParameters=CR_Core
-# 
+# this ensures submissions fail if they ask for more resources than available on the partition 
+EnforcePartLimits=ALL
+#
 # 
 # LOGGING AND ACCOUNTING 
 AccountingStorageType=accounting_storage/none
@@ -50,6 +52,7 @@ SlurmctldLogFile=/var/log/slurm-llnl/slurmctld.log
 SlurmdLogFile=/var/log/slurm-llnl/slurmd.log
 # 
 # 
-# COMPUTE NODES 
-NodeName={{ slurm_hostname }} CPUs={{ slurm_cpus }} CoresPerSocket={{ slurm_cpus }} State=UNKNOWN
-PartitionName={{ slurm_partition_name }} Nodes={{ slurm_hostname }} Default=YES MaxTime=INFINITE State=UP
+# BEGIN ANSIBLE MANAGED NODES
+NodeName={{ slurm_hostname }} Sockets={{ ansible_processor_count }} CoresPerSocket={{ ansible_processor_cores }} ThreadsPerCore={{ ansible_processor_threads_per_core }} State=UNKNOWN
+PartitionName={{ slurm_partition_name }} Nodes={{ slurm_hostname }} Default=YES MaxTime=INFINITE State=UP MaxNodes=1 MaxCPUsPerNode={{ ansible_processor_vcpus }}
+# END ANSIBLE MANAGED NODES