Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions roles/edpm_accel_drivers/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,15 @@ edpm_accel_drivers:
url: ""
ssl_no_verify: false
disable_gpg_check: false
kernel:
pin: false
version: ""
remove_old: true
reboot:
post_driver: false
timeout: 1800
wait_for_connection_timeout: 900
verify_vgpu_mgr: true
features:
nvidia_mig:
manage: false
Expand Down
40 changes: 40 additions & 0 deletions roles/edpm_accel_drivers/meta/argument_specs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,46 @@ argument_specs:
type: bool
default: false
description: Whether to disable GPG check when installing RPM (not recommended for production)
kernel:
type: dict
description: Kernel version pinning configuration for driver compatibility
options:
pin:
type: bool
default: false
description: Whether to pin a specific kernel version before driver installation
version:
type: str
default: ""
description: >-
Target kernel version to install and pin (e.g. 5.14.0-570.26.1.el9_6).
Required when pin is true.
remove_old:
type: bool
default: true
description: Whether to remove non-target kernel packages and orphaned boot files
reboot:
type: dict
description: Post-driver-install reboot configuration
options:
post_driver:
type: bool
default: false
description: >-
Whether to reboot the node after driver installation and boot configuration.
Required for nvidia-vgpu-mgr to detect MIG instances on NVIDIA GPUs.
timeout:
type: int
default: 1800
description: Maximum seconds to wait for the reboot to complete
wait_for_connection_timeout:
type: int
default: 900
description: Maximum seconds to wait for the node to become reachable after reboot
verify_vgpu_mgr:
type: bool
default: true
description: Whether to verify nvidia-vgpu-mgr service started after reboot
features:
type: dict
description: GPU-specific feature configurations
Expand Down
23 changes: 23 additions & 0 deletions roles/edpm_accel_drivers/molecule/kernel_pin_reboot/converge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
---
- name: Converge
hosts: all
pre_tasks:
- name: Read kernel version saved by prepare
ansible.builtin.slurp:
src: /tmp/test-kernel-version
register: _converge_kernel_version

- name: Set kernel version fact
ansible.builtin.set_fact:
test_kernel_version: "{{ (_converge_kernel_version.content | b64decode).strip() }}"
roles:
- role: "edpm_accel_drivers"
vars:
edpm_accel_drivers:
kernel:
pin: true
version: "{{ test_kernel_version }}"
remove_old: true
reboot:
post_driver: false
verify_vgpu_mgr: false
24 changes: 24 additions & 0 deletions roles/edpm_accel_drivers/molecule/kernel_pin_reboot/molecule.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
---
dependency:
name: galaxy
driver:
name: delegated
options:
managed: false
ansible_connection_options:
ansible_connection: local
platforms:
- name: compute-1
groups:
- compute
provisioner:
name: ansible
verifier:
name: ansible
scenario:
test_sequence:
- destroy
- create
- prepare
- converge
- verify
118 changes: 118 additions & 0 deletions roles/edpm_accel_drivers/molecule/kernel_pin_reboot/prepare.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
---
- name: Prepare
hosts: all
roles:
- role: ../../../../molecule/common/test_deps
- role: env_data
tasks:
- name: Install mock nvidia-smi
become: true
ansible.builtin.copy:
src: /bin/true
dest: /usr/bin/nvidia-smi
mode: '0755'
force: true
remote_src: true

- name: Get installed kernel-core version
become: true
ansible.builtin.shell:
cmd: |
set -o pipefail
rpm -q kernel-core --qf '%{VERSION}-%{RELEASE}\n' 2>/dev/null | head -1 || \
rpm -q kernel --qf '%{VERSION}-%{RELEASE}\n' 2>/dev/null | head -1 || \
uname -r | sed 's/\.x86_64$//' | sed 's/\.aarch64$//'
executable: /bin/bash
register: _prepare_kernel_version
changed_when: false

- name: Store kernel version for converge
ansible.builtin.set_fact:
test_kernel_version: "{{ _prepare_kernel_version.stdout }}"

- name: Save kernel version to file for converge to read
ansible.builtin.copy:
content: "{{ test_kernel_version }}"
dest: /tmp/test-kernel-version
mode: '0644'

- name: Get system architecture
ansible.builtin.command:
cmd: uname -m
register: _prepare_arch
changed_when: false

- name: Create /boot directory
become: true
ansible.builtin.file:
path: /boot
state: directory
mode: '0755'

- name: Create fake vmlinuz for target kernel
become: true
ansible.builtin.file:
path: "/boot/vmlinuz-{{ test_kernel_version }}.{{ _prepare_arch.stdout }}"
state: touch
mode: '0644'

- name: Create fake vmlinuz for old kernel (to test cleanup)
become: true
ansible.builtin.file:
path: /boot/vmlinuz-4.18.0-old.el9.x86_64
state: touch
mode: '0644'

- name: Install mock grubby
become: true
ansible.builtin.copy:
content: |
#!/bin/bash
if [[ "$1" == "--set-default" ]]; then
echo "$2" > /var/tmp/mock-grubby-default
chmod 644 /var/tmp/mock-grubby-default
elif [[ "$1" == "--default-kernel" ]]; then
cat /var/tmp/mock-grubby-default 2>/dev/null || echo "none"
fi
exit 0
dest: /usr/sbin/grubby
mode: '0755'
force: true

- name: Symlink mock grubby to /usr/bin
become: true
ansible.builtin.file:
src: /usr/sbin/grubby
dest: /usr/bin/grubby
state: link
force: true

- name: Create mock dnf.conf
become: true
ansible.builtin.copy:
content: |
[main]
gpgcheck=1
installonly_limit=3
clean_requirements_on_remove=True
best=True
skip_if_unavailable=False
dest: /etc/dnf/dnf.conf
mode: '0644'
force: true

- name: Ensure /usr/lib/nvidia directory exists
become: true
ansible.builtin.file:
path: /usr/lib/nvidia
state: directory
mode: '0755'

- name: Install mock sriov-manage
become: true
ansible.builtin.copy:
src: /bin/true
dest: /usr/lib/nvidia/sriov-manage
mode: '0755'
force: true
remote_src: true
35 changes: 35 additions & 0 deletions roles/edpm_accel_drivers/molecule/kernel_pin_reboot/verify.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
---
- name: Verify
hosts: all
gather_facts: false
tasks:
- name: Read kernel version saved by prepare
ansible.builtin.slurp:
src: /tmp/test-kernel-version
register: _verify_kernel_version_raw

- name: Set kernel version fact
ansible.builtin.set_fact:
test_kernel_version: "{{ (_verify_kernel_version_raw.content | b64decode).strip() }}"

- name: Read dnf.conf
ansible.builtin.slurp:
src: /etc/dnf/dnf.conf
register: _verify_dnf_conf

- name: Verify kernel exclude line is present in dnf.conf
ansible.builtin.assert:
that: "'exclude=kernel,kernel-core,kernel-modules' in (_verify_dnf_conf.content | b64decode)"
fail_msg: "dnf.conf does not contain kernel package exclude line"

- name: Verify grubby default kernel was set
become: true
ansible.builtin.command:
cmd: grubby --default-kernel
register: _verify_default_kernel
changed_when: false

- name: Verify default kernel contains target version
ansible.builtin.assert:
that: "test_kernel_version in _verify_default_kernel.stdout"
fail_msg: "Default kernel is '{{ _verify_default_kernel.stdout }}', expected {{ test_kernel_version }}"
91 changes: 91 additions & 0 deletions roles/edpm_accel_drivers/tasks/kernel_pin.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
---
- name: Validate kernel version format
ansible.builtin.assert:
that:
- _edpm_accel_drivers_config.kernel.version | length > 10
- _edpm_accel_drivers_config.kernel.version is regex('^[0-9]+\.[0-9]+\.[0-9]+-')
fail_msg: >-
kernel.version '{{ _edpm_accel_drivers_config.kernel.version }}'
does not look like a valid kernel version (e.g. 5.14.0-570.26.1.el9_6)

- name: Detect system architecture
ansible.builtin.command:
cmd: uname -m
register: _edpm_accel_drivers_arch
changed_when: false

- name: Install target kernel packages
become: true
ansible.builtin.dnf:
name:
- "kernel-{{ _edpm_accel_drivers_config.kernel.version }}"
- "kernel-core-{{ _edpm_accel_drivers_config.kernel.version }}"
- "kernel-modules-{{ _edpm_accel_drivers_config.kernel.version }}"
state: present
register: _edpm_accel_drivers_kernel_install

- name: Set target kernel as default boot kernel
become: true
ansible.builtin.command:
cmd: >-
grubby --set-default
/boot/vmlinuz-{{ _edpm_accel_drivers_config.kernel.version }}.{{ _edpm_accel_drivers_arch.stdout }}
changed_when: true

- name: Verify default kernel is set correctly
become: true
ansible.builtin.command:
cmd: grubby --default-kernel
register: _edpm_accel_drivers_default_kernel
changed_when: false
failed_when: _edpm_accel_drivers_config.kernel.version not in _edpm_accel_drivers_default_kernel.stdout

- name: Remove old kernel packages
when: _edpm_accel_drivers_config.kernel.remove_old | bool
block:
- name: Get list of non-target kernel packages
ansible.builtin.shell:
cmd: |
set -o pipefail
rpm -qa --qf '%{NAME}-%{VERSION}-%{RELEASE}\n' \
'kernel' 'kernel-core' 'kernel-modules' \
| grep -v '{{ _edpm_accel_drivers_config.kernel.version }}' || true
executable: /bin/bash
register: _edpm_accel_drivers_old_kernels
become: true
changed_when: false

- name: Remove old kernel packages
become: true
ansible.builtin.dnf:
name: "{{ _edpm_accel_drivers_old_kernels.stdout_lines }}"
state: absent
when: _edpm_accel_drivers_old_kernels.stdout_lines | length > 0

- name: Remove orphaned boot files for non-target kernels
become: true
ansible.builtin.shell:
cmd: |
set -o pipefail
target="{{ _edpm_accel_drivers_config.kernel.version }}"
cd /boot
removed=0
for file in vmlinuz-* initramfs-* System.map-* config-* symvers-*; do
if [[ -f "$file" || -L "$file" ]] && \
[[ "$file" != *"${target}"* ]]; then
rm -f "$file"
removed=$((removed + 1))
fi
done
echo "Removed ${removed} orphaned boot files"
executable: /bin/bash
register: _edpm_accel_drivers_boot_cleanup
changed_when: "'Removed 0' not in _edpm_accel_drivers_boot_cleanup.stdout"

- name: Exclude kernel packages from future updates
become: true
ansible.builtin.lineinfile:
path: /etc/dnf/dnf.conf
regexp: '^exclude='
line: "exclude=kernel,kernel-core,kernel-modules"
create: false
10 changes: 10 additions & 0 deletions roles/edpm_accel_drivers/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@
- name: Setup configuration and build enabled vendors list
ansible.builtin.import_tasks: setup_config.yml

- name: Pin kernel version for driver compatibility
when:
- _edpm_accel_drivers_config.kernel.pin | bool
- _edpm_accel_drivers_config.kernel.version | length > 0
ansible.builtin.import_tasks: kernel_pin.yml

- name: Install GPU drivers for enabled vendors
ansible.builtin.include_tasks: driver_install.yml
loop: "{{ _edpm_accel_drivers_enabled_vendors }}"
Expand All @@ -26,6 +32,10 @@
when: _edpm_accel_drivers_enabled_vendors | length > 0
ansible.builtin.import_tasks: boot_config.yml

- name: Reboot node after driver installation
when: _edpm_accel_drivers_config.reboot.post_driver | bool
ansible.builtin.import_tasks: reboot.yml

- name: Configure MIG partitioning on NVIDIA GPU
when: _edpm_accel_drivers_config.features.nvidia_mig.manage | bool
ansible.builtin.import_tasks: nvidia_mig_partitioning.yml
Loading