Initial POC for installing nvidia driver on EDPM nodes

TODO:
 - molecule tests
 - docs
 - more checks

galaxy.yml

@@ -8,7 +8,7 @@ namespace: cifmw
 name: general
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 1.0.0
+version: 1.0.0+15f75128
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

playbooks/nvidia-mdev.yml

@@ -0,0 +1,67 @@
+---
+# Copyright 2024 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Gather the list of EDPM computes
+  hosts: "{{ cifmw_target_hook_host | default('localhost') }}"
+  gather_facts: false
+  tasks:
+    - name: Fetch OSP BMO nodesets
+      environment:
+        KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+        PATH: "{{ cifmw_path }}"
+      ansible.builtin.command:
+        cmd: >-
+          oc get OpenStackBaremetalSet -n "{{ namespace|default('openstack') }}"  -o yaml
+      register: _osp_bmo_nodsets_oc_out
+
+    - name: Add OSP BMO nodesets to Ansible
+      ansible.builtin.add_host:
+        name: "{{ item.name }}"
+        groups: "{{ item.group }}"
+        ansible_ssh_user: "{{ item.user }}"
+        ansible_host: "{{ item.ip }}"
+        ansible_ssh_private_key_file: "{{ ansible_user_dir }}/.ssh/id_cifw"
+        ansible_ssh_extra_args: '-o StrictHostKeyChecking=no'
+      loop: >-
+        {% set hosts = [] -%}
+        {% set nodesets = (_osp_bmo_nodsets_oc_out.stdout | from_yaml)['items'] | default([]) -%}
+        {% for spec in nodesets | map(attribute='spec') -%}
+        {%   for host_key, host_val in spec.baremetalHosts.items() -%}
+        {%     set _ = hosts.append(
+          {
+          'name': host_key,
+          'ip': host_val['ctlPlaneIP'] | ansible.utils.ipaddr('address'),
+          'user': spec.cloudUserName,
+          'group': host_key | split('-') | first + 's'
+        }) -%}
+        {%   endfor -%}
+        {% endfor -%}
+        {{ hosts }}
+
+- name: Run the Nvidia role
+  hosts: computes
+  tasks:
+    - name: Run phase1
+      ansible.builtin.import_role:
+        name: edpm_nvidia_mdev_prepare
+        # As a reminder, at the end of phase1, the compute will reboot
+        tasks_from: phase1
+
+    - name: Run phase 2
+      ansible.builtin.import_role:
+        name: edpm_nvidia_mdev_prepare
+        tasks_from: phase2

roles/edpm_nvidia_mdev_prepare/README.md

@@ -0,0 +1,10 @@
+# edpm_nvidia_mdev_prepare
+Please explain the role purpose.
+
+## Privilege escalation
+If apply, please explain the privilege escalation done in this role.
+
+## Parameters
+* `param_1`: this is an example
+
+## Examples

roles/edpm_nvidia_mdev_prepare/defaults/main.yml

@@ -0,0 +1,28 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Does the OS needs to disable the nouveau driver ?
+cifmw_edpm_nvidia_mdev_prepare_disable_nouveau: true
+
+# What is the URL or path for the nvidia driver RPM ?
+cifmw_edpm_nvidia_mdev_prepare_driver_url: ''
+
+# What will be the name of the nvidia package ?
+cifmw_edpm_nvidia_mdev_prepare_package_name: "NVIDIA-vGPU-rhel"
+
+# Which SR-IOV GPU devices should be creating VFs ?
+cifmw_edpm_nvidia_mdev_prepare_sriov_devices:
+  - ALL

roles/edpm_nvidia_mdev_prepare/files/nvidia-sriov-manage@.service

@@ -0,0 +1,26 @@
+[Unit]
+After = nvidia-vgpu-mgr.service
+After = nvidia-vgpud.service
+Description = Enable Nvidia GPU virtual functions
+
+[Service]
+Type = oneshot
+User = root
+Group = root
+ExecStart = /usr/lib/nvidia/sriov-manage -e %i
+# Give a reasonable amount of time for the server to start up/shut down
+TimeoutSec = 120
+# This creates a specific slice which all services will operate from
+#  The accounting options give us the ability to see resource usage
+#  through the `systemd-cgtop` command.
+Slice = system.slice
+# Set Accounting
+CPUAccounting = True
+BlockIOAccounting = True
+MemoryAccounting = True
+TasksAccounting = True
+RemainAfterExit = True
+ExecStartPre = /usr/bin/sleep 30
+
+[Install]
+WantedBy = multi-user.target

roles/edpm_nvidia_mdev_prepare/handlers/main.yml

@@ -0,0 +1,15 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.

roles/edpm_nvidia_mdev_prepare/meta/main.yml

@@ -0,0 +1,30 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+galaxy_info:
+  author: CI Framework
+  description: CI Framework Role -- edpm_nvidia_mdev_prepare
+  company: Red Hat
+  license: Apache-2.0
+  min_ansible_version: "2.14"
+  namespace: cifmw
+  galaxy_tags:
+    - cifmw
+
+# List your role dependencies here, one per line. Be sure to remove the '[]' above,
+# if you add dependencies to this list.
+dependencies: []

roles/edpm_nvidia_mdev_prepare/molecule/default/converge.yml

@@ -0,0 +1,29 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Converge
+  hosts: all
+  tasks:
+    - name: Run phase1
+      ansible.builtin.import_role:
+        name: edpm_nvidia_mdev_prepare
+        tasks_from: phase1
+
+    - name: Run phase 2
+      ansible.builtin.import_role:
+        name: edpm_nvidia_mdev_prepare
+        tasks_from: phase2

roles/edpm_nvidia_mdev_prepare/molecule/default/molecule.yml

@@ -0,0 +1,11 @@
+---
+# Mainly used to override the defaults set in .config/molecule/
+# By default, it uses the "config_podman.yml" - in CI, it will use
+# "config_local.yml".
+log: true
+
+provisioner:
+  name: ansible
+  log: true
+  env:
+    ANSIBLE_STDOUT_CALLBACK: yaml

roles/edpm_nvidia_mdev_prepare/molecule/default/prepare.yml

@@ -0,0 +1,21 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+- name: Prepare
+  hosts: all
+  roles:
+    - role: test_deps

roles/edpm_nvidia_mdev_prepare/tasks/cleanup.yml

@@ -0,0 +1,19 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Cleaning the World
+  ansible.builtin.debug:
+    msg: "So here edpm_nvidia_mdev_prepare should clean things up!"

roles/edpm_nvidia_mdev_prepare/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+

roles/edpm_nvidia_mdev_prepare/tasks/phase1.yml

@@ -0,0 +1,92 @@
+---
+# Copyright 2024 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Source the secret file
+  ansible.builtin.include_vars:
+    file: "{{ lookup('env', 'HOME') }}/secret.txt"
+    name: cifmw_edpm_nvidia_mdev_prepare_secrets
+
+- name: Set url value from cifmw_edpm_nvidia_mdev_prepare_secrets
+  when:
+    - cifmw_edpm_nvidia_mdev_prepare_secrets['driver_url'] is defined
+  ansible.builtin.set_fact:
+    cifmw_edpm_nvidia_mdev_prepare_driver_url: >-
+      {{
+        cifmw_edpm_nvidia_mdev_prepare_secrets['driver_url']
+      }}
+
+- name: Blacklist nouveau
+  become: true
+  ansible.builtin.copy:
+    dest: "/etc/modprobe.d/blacklist-nouveau.conf"
+    mode: "0644"
+    content: |-
+      blacklist nouveau
+      options nouveau modeset=0
+    force: false
+  when:
+    - cifmw_edpm_nvidia_mdev_prepare_disable_nouveau | bool
+  register: _blacklist_nouveau
+
+- name: Make sure that we defined the driver URL
+  ansible.builtin.assert:
+    that:
+      - cifmw_edpm_nvidia_mdev_prepare_driver_url is defined
+      - cifmw_edpm_nvidia_mdev_prepare_driver_url | length > 0
+    msg: "You need to set cifmw_edpm_nvidia_mdev_prepare_driver_url"
+
+- name: Gather the package facts
+  ansible.builtin.package_facts:
+    manager: auto
+
+- name: Install nvidia driver RPM either from path or URL
+  become: true
+  ansible.builtin.dnf:
+    name: "{{ cifmw_edpm_nvidia_mdev_prepare_driver_url }}"
+    state: present
+    disable_gpg_check: true
+  when: cifmw_edpm_nvidia_mdev_prepare_package_name not in ansible_facts.packages
+  register: _nvidia_driver_install
+
+- name: Regenerate initramfs
+  become: true
+  ansible.builtin.command: "{{ item }}"
+  loop:
+    - 'dracut --force'
+    - 'grub2-mkconfig -o /boot/efi/EFI/redhat/grub.cfg'
+  when: _blacklist_nouveau.changed or _nvidia_driver_install.changed
+
+- name: Enforce a reboot to ensure that we have the driver loaded
+  block:
+    - name: Create directory required by edpm-reboot role
+      become: true
+      ansible.builtin.file:
+        path: /var/lib/openstack/reboot_required/
+        state: directory
+        mode: "0755"
+    - name: Create required file to enforce a reboot
+      become: true
+      ansible.builtin.file:
+        path: /var/lib/openstack/reboot_required/nvidia_mdev_reboot
+        state: touch
+        mode: "0600"
+    - name: Call edpm_reboot role
+      # Since the EDPM role isn't installed, we can't call it
+      # Removing it for now
+      # ansible.builtin.include_role:
+      #   name: edpm_reboot
+      # Instead, use a regular reboot
+      ansible.builtin.reboot: