Merge pull request #1657 from Azure/feature/infra_on_alma8

infrastructure on alma8

.github/workflows/all_bicep.yml

@@ -3,7 +3,7 @@ name : All Bicep End to End
 on:
   workflow_dispatch:
   schedule:
-    - cron: "10 2 * * 1" # At 02:10 UTC on each Sunday
+    - cron: "10 4 * * 0" # At 04:10 UTC on each Saturday
 
 jobs:
   deploy:
@@ -15,6 +15,7 @@ jobs:
         home_type: ['azurefiles', 'anf']
         user_auth: ['local', 'ad']
         monitoring: ['none', 'grafana']
+        infra_os: ['centos', 'almalinux']
     uses: ./.github/workflows/z_base_callable.yml
     with:
       resource_group: 'AUTO_GENERATED'
@@ -28,6 +29,7 @@ jobs:
       clean: 'true'
       ui_tests: 'true'
       monitoring: ${{ matrix.monitoring }}
+      infra_os: ${{ matrix.infra_os }}
     secrets:
       ARM_CLIENT_SECRET: ${{ secrets.ARM_CLIENT_SECRET }}
       ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}

.github/workflows/configs/almalinux.yml

@@ -1,4 +1,6 @@
-linux_base_image: "almalinux:almalinux-hpc:8_7-hpc-gen2:latest" # publisher:offer:sku:version or image_id
+linux_base_image: almalinux:almalinux-x86_64:8_7-gen2:latest # almalinux:almalinux-hpc:8_7-hpc-gen2:latest # publisher:offer:sku:version or image_id
+# linux image plan if required, format is publisher:product:name
+#linux_base_plan: almalinux:almalinux-hpc:8_7-hpc-gen2
 
 queues:
   - name: execute

bicep/azhop.bicep

@@ -57,8 +57,8 @@ var linuxBasePlan = contains(azhopConfig, 'linux_base_plan') ? azhopConfig.linux
 var windowsBaseImage = contains(azhopConfig, 'windows_base_image') ? azhopConfig.windows_base_image : 'MicrosoftWindowsServer:WindowsServer:2019-Datacenter-smalldisk:latest'
 var lustreBaseImage = contains(azhopConfig, 'lustre_base_image') ? azhopConfig.lustre_base_image : 'azhpc:azurehpc-lustre:azurehpc-lustre-2_12:latest'
 var lustreBasePlan = contains(azhopConfig, 'lustre_base_plan') ? azhopConfig.lustre_base_plan : 'azhpc:azurehpc-lustre:azurehpc-lustre-2_12'
-var cyclecloudBaseImage = contains(azhopConfig.cyclecloud, 'image') ? azhopConfig.cyclecloud.image : 'OpenLogic:CentOS:7_9-gen2:latest'
-var cyclecloudBasePlan = contains(azhopConfig.cyclecloud, 'plan') ? azhopConfig.cyclecloud.plan : ''
+var cyclecloudBaseImage = contains(azhopConfig.cyclecloud, 'image') ? azhopConfig.cyclecloud.image : linuxBaseImage
+var cyclecloudBasePlan = contains(azhopConfig.cyclecloud, 'plan') ? azhopConfig.cyclecloud.plan : linuxBasePlan
 
 var createDatabase = (config.queue_manager == 'slurm' && config.slurm.accounting_enabled ) || config.enable_remote_winviz
 

config.tpl.yml

@@ -177,6 +177,8 @@ locked_down_network:
   public_ip: true # Enable public IP creation for Jumpbox, OnDemand and create images. Default to true
 
 # Base image configuration. Can be either an image reference or an image_id from the image registry or a custom managed image
+# For AlmaLinux 8.7 use almalinux:almalinux-x86_64:8_7-gen2:latest
+# For CentOS 7.9 use OpenLogic:CentOS:7_9-gen2:latest
 linux_base_image: "OpenLogic:CentOS:7_9-gen2:latest" # publisher:offer:sku:version or image_id
 # linux image plan if required, format is publisher:product:name
 #linux_base_plan:

docs/deploy/index.md

@@ -537,6 +537,8 @@ locked_down_network:
   public_ip: true # Enable public IP creation for Jumpbox, OnDemand and create images. Default to true
 
 # Base image configuration. Can be either an image reference or an image_id from the image registry or a custom managed image
+# For AlmaLinux 8.7 use almalinux:almalinux-x86_64:8_7-gen2:latest
+# For CentOS 7.9 use OpenLogic:CentOS:7_9-gen2:latest
 linux_base_image: "OpenLogic:CentOS:7_9-gen2:latest" # publisher:offer:sku:version or image_id
 # linux image plan if required, format is publisher:product:name
 #linux_base_plan:

packer/scripts/almalinux/openpbs.sh

@@ -27,7 +27,7 @@ if [ "$BUILD_FROM_SOURCE" == "yes" ] ; then
     chmod 4755 /opt/pbs/sbin/pbs_iff /opt/pbs/sbin/pbs_rcp
 else
     wget https://github.com/openpbs/openpbs/releases/download/v20.0.1/openpbs_20.0.1.centos_8.zip
-    unzip -o pbspro_19.1.1.centos7.zip
+    unzip -o openpbs_20.0.1.centos_8.zip
     dnf install epel-release -y
     dnf install -y openpbs_20.0.1.centos_8/openpbs-execution-20.0.1-0.x86_64.rpm jq
     rm -rf openpbs_20.0.1.centos_8.zip

playbooks/create_local_user.yml

@@ -16,7 +16,6 @@
     comment: 'Local user'
     name: '{{user.name}}'
     groups: "{{ usergroups | selectattr('gid', 'in', (user.groups | default([], true))) | map(attribute='name')}}"
-    shell: /bin/bash
     uid: '{{user.uid}}'
     group: "{{ (usergroups | selectattr('gid', 'eq', 5000) | map(attribute='name'))[0] | default('azhop-users', true) }}"
     home: "{{ homedir_mountpoint }}/{{ user.name }}"

playbooks/ood.yml

@@ -33,18 +33,9 @@
       state: latest
       lock_timeout : 180
 
-  - name: Set up PAM authentication
-    shell: |
-      yum install -y mod_authnz_pam
-      mkdir -p /opt/rh/httpd24/root/usr/lib64/httpd/modules/
-      cp /usr/lib64/httpd/modules/mod_authnz_pam.so /opt/rh/httpd24/root/usr/lib64/httpd/modules/
-      mkdir -p /opt/rh/httpd24/root/etc/httpd/conf.modules.d
-      echo "LoadModule authnz_pam_module modules/mod_authnz_pam.so" > /opt/rh/httpd24/root/etc/httpd/conf.modules.d/55-authnz_pam.conf
-      cp /etc/pam.d/sshd /etc/pam.d/ood
-      chmod 640 /etc/shadow
-      chgrp apache /etc/shadow
-    args:
-      creates: /etc/pam.d/ood
+  - name: Set up PAM authentication for OOD
+    include_role:
+      name: ood_pam_auth
 
   - name: Retrieve OIDC secret
     block:
@@ -246,7 +237,7 @@
       apply:
         become: true
     vars:
-      cc_slurm_version: '{{slurm.slurm_version | default("20.11.9")}}-1'
+      slurm_version: '{{slurm.slurm_version | default("20.11.9")}}-1'
     when: ( queue_manager is defined and queue_manager == "slurm" )
 
   - name: setup cyclecloud proxy
@@ -269,9 +260,16 @@
           sed -i '$e cat cyclecloud_proxy' /opt/ood/ood-portal-generator/templates/ood-portal.conf.erb
           rm cyclecloud_proxy
           /opt/ood/ood-portal-generator/sbin/update_ood_portal
-          systemctl try-restart httpd24-httpd.service httpd24-htcacheclean.service
         fi
 
+  - name: Configure Lmod
+    block:
+      - name: enable powertools
+        command: dnf config-manager --enable powertools
+    when: 
+      - ansible_distribution == 'AlmaLinux'
+      - ansible_distribution_major_version == '8'
+
   - name: add modules for job submission
     yum: 
       name: Lmod, hwloc, numactl, jq
@@ -319,15 +317,6 @@
       path: /etc/ood/config/apps/dashboard/initializers
       state: directory
 
-  # - name: create ood lustre menu option
-  #   lineinfile:
-  #     path: /etc/ood/config/apps/dashboard/initializers/ood.rb
-  #     search_string: /lustre
-  #     line : 'OodFilesApp.candidate_favorite_paths << FavoritePath.new("/lustre", title: "Lustre")'
-  #     create: yes
-  #     state: present
-  #   when: ( lustre.create | default(false) )
-
   - name: Create "{{mounts[item].mountpoint}}" directory mountpoint
     file:
       path: '{{mounts[item].mountpoint}}'
@@ -352,16 +341,6 @@
       src: 'ood.rb.j2'
       dest: '/etc/ood/config/apps/dashboard/initializers/ood.rb'
 
-  # - name: create mount "{{mounts[item].mountpoint}}" menu option
-  #   lineinfile:
-  #     path: /etc/ood/config/apps/dashboard/initializers/ood.rb
-  #     search_string: '{{mounts[item].mountpoint}}'
-  #     line : 'OodFilesApp.candidate_favorite_paths << FavoritePath.new("{{mounts[item].mountpoint}}", title: "{{item}}" )'
-  #     create: yes
-  #     state: present
-  #   loop: "{{mounts | list}}"
-  #   when: item != 'home'
-
   # Update file upload staging area to be in /mnt/resource
   - name: Create passenger temp dir
     file: 
@@ -412,6 +391,9 @@
       sed -i "s/^LoadModule mpm_prefork_module/#LoadModule mpm_prefork_module/" /opt/rh/httpd24/root/etc/httpd/conf.modules.d/00-mpm.conf
       sed -i "s/^LoadModule mpm_worker_module/#LoadModule mpm_worker_module/" /opt/rh/httpd24/root/etc/httpd/conf.modules.d/00-mpm.conf
       sed -i "s/^#LoadModule mpm_event_module/LoadModule mpm_event_module/" /opt/rh/httpd24/root/etc/httpd/conf.modules.d/00-mpm.conf
+    when: 
+      - ansible_distribution == 'CentOS'
+      - ansible_distribution_major_version == '7'
 
   # - name: Tune Apache mpm_event_module
   #   copy:
@@ -462,6 +444,18 @@
 
   - name: restart ood
     shell: systemctl try-restart httpd24-httpd.service httpd24-htcacheclean.service
+    when: 
+      - ansible_distribution == 'CentOS'
+      - ansible_distribution_major_version == '7'
+
+  - name: Ensure apache is restarted
+    service: 
+      name: httpd
+      state: restarted
+      enabled: yes
+    when: 
+      - ansible_distribution == 'AlmaLinux'
+      - ansible_distribution_major_version == '8'
 
   - name: Add cvmsf-eessi support
     block:
@@ -470,6 +464,7 @@
           name: https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm 
           state: present
           lock_timeout : 180
+          disable_gpg_check: true
 
       - name: Install cvmfs package
         yum:
@@ -482,6 +477,7 @@
           name: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
           state: present
           lock_timeout : 180
+          disable_gpg_check: true
 
       - name: Create cvmfs config 
         copy:

playbooks/roles/cyclecloud/tasks/AlmaLinux.yml

@@ -0,0 +1,60 @@
+---
+- name: Disable SELinux
+  selinux:
+    state: disabled
+  register: selinux
+
+- name: reboot 
+  reboot:
+  when: selinux.reboot_required 
+
+- name: Update packages marked for security
+  yum:
+    state: latest
+    security: yes
+    lock_timeout : 180
+
+- name: install AZ CLI repo (CentOS)
+  shell: |
+    set -e
+    rpm --import https://packages.microsoft.com/keys/microsoft.asc
+    cat > /etc/yum.repos.d/azure-cli.repo <<EOF
+    [azure-cli]
+    name=Azure CLI
+    baseurl=https://packages.microsoft.com/yumrepos/azure-cli
+    enabled=1
+    gpgcheck=1
+    gpgkey=https://packages.microsoft.com/keys/microsoft.asc
+    EOF
+  args:
+    creates: /etc/yum.repos.d/azure-cli.repo
+
+- name: install CycleCloud repo
+  shell: |
+    cat > /etc/yum.repos.d/cyclecloud.repo <<EOF
+    [cyclecloud]
+    name=cyclecloud
+    baseurl=https://packages.microsoft.com/yumrepos/cyclecloud
+    gpgcheck=1
+    gpgkey=https://packages.microsoft.com/keys/microsoft.asc
+    EOF
+  args:
+    creates: /etc/yum.repos.d/cyclecloud.repo
+
+- name: Install pre-reqs packages
+  yum:
+    name: azure-cli, dnsmasq, unzip, java
+    state: present
+    lock_timeout : 180
+
+- name: Install CycleCloud
+  yum:
+    name: "cyclecloud8-{{cc_version}}"
+    state: present
+    lock_timeout : 180
+
+- name: Install Jetpack
+  yum:
+    name: "jetpack8-{{cc_version}}"
+    state: present
+    lock_timeout : 180

playbooks/roles/cyclecloud_cluster/projects/openpbs/cluster-init/files/almalinux/installpbs.sh

@@ -1,22 +1,49 @@
 #!/bin/bash
+cyclecloud_pbspro=$1
+openpbs_version=$2
 
-BUILD_FROM_SOURCE=yes
-[ -d /opt/pbs ] && exit 0
+major_installed_pbs_version=$(cat /var/spool/pbs/pbs_version | cut -d '=' -f2 | cut -d '.' -f1)
+major_wanted_pbs_version=$(echo $openpbs_version | cut -d '.' -f1)
+
+function install_or_build() {
+    local version=$1
+    case $version in
+        19)
+            build19
+            ;;
+        20|22)
+            install
+            ;;
+        *)
+            echo "Unsupported PBS version: $version"
+            exit 1
+            ;;
+    esac
+}
+
+function install() {
+    dnf install -y epel-release
+    dnf install -y https://github.com/Azure/cyclecloud-pbspro/releases/download/${cyclecloud_pbspro}/openpbs-execution-${openpbs_version}-0.x86_64.rpm jq
+}
+
+function build19() {
+    [ -d /opt/pbs ] && exit 0
 
-if [ "$BUILD_FROM_SOURCE" == "yes" ] ; then
     dnf install -y gcc make rpm-build libtool hwloc-devel \
         libX11-devel libXt-devel libedit-devel libical-devel \
         ncurses-devel perl postgresql-devel postgresql-contrib python2 python2-devel tcl-devel \
         tk-devel swig expat-devel openssl-devel libXext libXft \
         autoconf automake gcc-c++ git jq
 
     cd /mnt
+    rm -rf hwloc
     git clone https://github.com/open-mpi/hwloc.git -b v1.11
     cd hwloc
+    ./autogen.sh
     ./configure --enable-static --enable-embedded-mode
     make
     cd ..
-    
+
     wget -q https://github.com/openpbs/openpbs/releases/download/v19.1.1/pbspro-19.1.1.tar.gz -O pbspro-19.1.1.tar.gz
     tar -xzf pbspro-19.1.1.tar.gz
     cd pbspro-19.1.1/
@@ -26,11 +53,21 @@ if [ "$BUILD_FROM_SOURCE" == "yes" ] ; then
 
     /opt/pbs/libexec/pbs_postinstall execution
     chmod 4755 /opt/pbs/sbin/pbs_iff /opt/pbs/sbin/pbs_rcp
+}
+
+
+# If PBS is not installed, then install it
+if [ ! -f "/etc/pbs.conf" ]; then
+    install_or_build $major_wanted_pbs_version
 else
-    wget https://github.com/openpbs/openpbs/releases/download/v20.0.1/openpbs_20.0.1.centos_8.zip
-    unzip -o openpbs_20.0.1.centos_8.zip
-    dnf install epel-release -y
-    dnf install -y openpbs_20.0.1.centos_8/openpbs-execution-20.0.1-0.x86_64.rpm jq
-    rm -rf openpbs_20.0.1.centos_8.zip
-    rm -rf openpbs_20.0.1.centos_8
-fi
+    # If installed version is not the same as the version we want to install, then remove and install it
+    if [ "$major_installed_pbs_version" != "major_wanted_pbs_version" ]; then
+        echo "Removing old PBS version $major_installed_pbs_version"
+        set +e
+        systemctl stop pbs
+        rm -rf /opt/pbs
+        rm -rf /var/spool/pbs
+        set -e
+        install_or_build $major_wanted_pbs_version
+    fi
+fi

playbooks/roles/cyclecloud_cluster/projects/openpbs/cluster-init/scripts/2-installpbs.sh → playbooks/roles/cyclecloud_cluster/projects/openpbs/cluster-init/scripts/2-installpbs.sh.j2

@@ -4,18 +4,18 @@ source "$script_dir/../files/azhop-helpers.sh"
 read_os
 
 find $script_dir/../files -name "*.sh" -exec chmod +x {} \;
-
-# If PBS is not installed, then install it
-if [ ! -f "/etc/pbs.conf" ]; then
-    $script_dir/../files/$os_release/installpbs.sh
-fi
+$script_dir/../files/$os_release/installpbs.sh {{cyclecloud_openbps_release}} {{openpbs_version}}
 
 echo "Configuring PBS"
 
+# for version 19
 sed -i 's/CHANGE_THIS_TO_PBS_PRO_SERVER_HOSTNAME/scheduler/' /etc/pbs.conf
 sed -i 's/CHANGE_THIS_TO_PBS_PRO_SERVER_HOSTNAME/scheduler/' /var/spool/pbs/mom_priv/config
 sed -i "s/^if /#if /g" /opt/pbs/lib/init.d/limits.pbs_mom
 sed -i "s/^fi/#fi /g" /opt/pbs/lib/init.d/limits.pbs_mom
 
+# for version 20 and above
+sed -i 's/CHANGE_THIS_TO_PBS_SERVER_HOSTNAME/scheduler/' /etc/pbs.conf
+
 systemctl restart pbs || exit 1
 echo "PBS Restarted"