From cd607a4bc02d4451af4188df20cc6c15ddd40bbc Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 27 Apr 2026 18:22:59 +0200 Subject: [PATCH 01/13] F #183: Add ability to rename net/pci/vfio devices - Normalize lspci_devices's structure - Add 'unlisted' option for hiding PCI devices from OpenNebula - Add 'set_name' option for naming PCI devices - Add 'match_address' test plugin to handle wildcards in PCI/MAC addresses - Add ability to match NICs by their MAC address - Add udev rules for net/pci/vfio device renaming/symlinking (set_name) - Rename 99-vfio.rules to 99-mode.rules in both helper/pci and openvswitch roles - Remove 'pci_passthrough_enabled' - Rename 'pci_passthrough_default_filter' to 'pci_default_filter' - Update precheck role - Update README.md files Signed-off-by: Michal Opala --- plugins/test/main.py | 32 ++++ plugins/test/match_address.yml | 43 ++++++ roles/helper/pci/README.md | 11 +- roles/helper/pci/defaults/main.yml | 1 - roles/helper/pci/tasks/devices.yml | 65 ++++----- roles/helper/pci/tasks/main.yml | 13 +- roles/helper/pci/tasks/query.yml | 145 +++++++------------ roles/helper/pci/tasks/udev.yml | 60 ++++++++ roles/openvswitch/tasks/main.yml | 37 ++--- roles/pci/frontend/README.md | 9 +- roles/pci/frontend/defaults/main.yml | 6 +- roles/pci/frontend/tasks/main.yml | 31 ++-- roles/pci/node/README.md | 11 +- roles/pci/node/defaults/main.yml | 1 - roles/pci/node/tasks/filters.yml | 4 +- roles/pci/node/tasks/main.yml | 14 +- roles/precheck/post_reboot/tasks/pci.yml | 16 +- roles/precheck/pre_reboot/tasks/features.yml | 2 - 18 files changed, 306 insertions(+), 195 deletions(-) create mode 100644 plugins/test/main.py create mode 100644 plugins/test/match_address.yml create mode 100644 roles/helper/pci/tasks/udev.yml diff --git a/plugins/test/main.py b/plugins/test/main.py new file mode 100644 index 00000000..b8b2d9a8 --- /dev/null +++ b/plugins/test/main.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import fnmatch +import re + +class TestModule(object): + def tests(self): + return { + 'match_address': self.match_address, + } + + # NOTE: It does not validate character classes or character count! + # EXAMPLES: + # pci -> match_address('0000:0*:00.*', sep='[:.]') + # mac -> match_address('52:54:*:*:*:0*', sep='[:]') + def match_address(self, address, pattern, sep=None): + """Tests if a split string matches a set of simple glob patterns.""" + + # In case no separator is provided simply match the whole string. + if sep is None: + return fnmatch.fnmatch(address, pattern) + + # Make sure both address and pattern contain identical separators. + if re.findall(sep, address) != re.findall(sep, pattern): + return False + + # Try matching (glob) each segment separately. + for x, y in zip(re.split(sep, address), re.split(sep, pattern)): + if not fnmatch.fnmatch(x, y): + return False + + return True diff --git a/plugins/test/match_address.yml b/plugins/test/match_address.yml new file mode 100644 index 00000000..56cb543e --- /dev/null +++ b/plugins/test/match_address.yml @@ -0,0 +1,43 @@ +--- +# Copyright: OpenNebula Project, OpenNebula Systems +# Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +DOCUMENTATION: + name: match_address + short_description: Apply glob patterns to a generic address. + description: + - Tests if a split string matches a set of simple glob patterns. + options: + _input: + description: + - An address to match. + type: string + required: true + sep: + description: + - A regex expression used to split the input string (optional). + type: string + required: false + author: + - Michal Opala (@sk4zuzu) + +EXAMPLES: | + - name: Match MAC address + ansible.builtin.debug: + msg: >- + {{ '12:34:56:78:90:ab' is opennebula.deploy.match_address('12:34:56:78:90:*', sep='[:]') }} + + - name: Select matching PCI addresses + ansible.builtin.debug: + msg: >- + {{ _addresses | select('opennebula.deploy.match_address', '0000:03:00.*', sep='[:.]') }} + vars: + _addresses: + - '12:34:56:78:90:ab' # MAC (skipped) + - '0000:03:00.4' # PCI (matched) + - '0000:04:00.0' # PCI (skipped) + +RETURN: + _value: + description: Returns True if the check passes. + type: boolean diff --git a/roles/helper/pci/README.md b/roles/helper/pci/README.md index c48a6d69..1a58888b 100644 --- a/roles/helper/pci/README.md +++ b/roles/helper/pci/README.md @@ -11,10 +11,9 @@ N/A Role Variables -------------- -| Name | Type | Default | Example | Description | -|---------------------------|--------|---------|---------------|---------------------------------| -| `pci_passthrough_enabled` | `bool` | `false` | | Enable/Disable PCI passthrough. | -| `pci_devices` | `list` | `[]` | (check below) | PCI devices configuration. | +| Name | Type | Default | Example | Description | +|---------------|--------|---------|---------------|---------------------------------| +| `pci_devices` | `list` | `[]` | (check below) | PCI devices configuration. | Dependencies ------------ @@ -26,10 +25,12 @@ Example Playbook - hosts: node vars: - pci_passthrough_enabled: true pci_devices: - address: "0000:02:00.0" excluded: true + - address: "0000:03:00.*" + unlisted: true + set_name: "asd{0[3]}" - vendor: "1af4" device: "*" class: "0200" diff --git a/roles/helper/pci/defaults/main.yml b/roles/helper/pci/defaults/main.yml index 8e696a88..ff929a8c 100644 --- a/roles/helper/pci/defaults/main.yml +++ b/roles/helper/pci/defaults/main.yml @@ -1,3 +1,2 @@ --- -pci_passthrough_enabled: false pci_devices: [] diff --git a/roles/helper/pci/tasks/devices.yml b/roles/helper/pci/tasks/devices.yml index 25de57b6..979c57c2 100644 --- a/roles/helper/pci/tasks/devices.yml +++ b/roles/helper/pci/tasks/devices.yml @@ -25,11 +25,7 @@ v.device | d('*'), v.class | d('*'), ) | lower, - "key_regex": "^{}:{}:{}$".format( - (v.vendor | d('*') == '*') | ternary('[^:]*', v.vendor), - (v.device | d('*') == '*') | ternary('[^:]*', v.device), - (v.class | d('*') == '*') | ternary('[^:]*', v.class), - ) | lower, + "key_type": "vdc", })) -}} {%- endfor -%} @@ -37,12 +33,22 @@ pci_devices_by_address: >- {%- set output = [] -%} {%- for v in pci_devices | selectattr('address', 'defined') -%} - {{- - output.append(v | combine({ - "key": v.address | lower, - "key_regex": "^{}$".format(v.address) | lower, - })) - -}} + {%- if v.address is opennebula.deploy.match_address('*:*:*.*', sep='[:.]') -%} + {{- + output.append(v | combine({ + "key": v.address | lower, + "key_type": "pci", + })) + -}} + {%- endif -%} + {%- if v.address is opennebula.deploy.match_address('*:*:*:*:*:*', sep='[:]') -%} + {{- + output.append(v | combine({ + "key": v.address | lower, + "key_type": "mac", + })) + -}} + {%- endif -%} {%- endfor -%} {{- output -}} @@ -52,25 +58,6 @@ - when: lspci_devices | count > 0 block: - - name: Ensure udev rules for vfio - ansible.builtin.copy: - dest: /etc/udev/rules.d/99-vfio.rules - owner: 0 - group: 0 - mode: u=rw,go=r - content: | - SUBSYSTEM=="vfio", GROUP="kvm", MODE="0666" - register: copy_udev_vfio - - - name: Refresh udev rules - ansible.builtin.shell: - cmd: | - set -o errexit - udevadm control --reload-rules && udevadm trigger - executable: /bin/bash - changed_when: true - when: copy_udev_vfio is changed - - name: Render sriov-enable service unit ansible.builtin.copy: dest: "{{ item.dest }}" @@ -140,7 +127,7 @@ _to_revert: >- {%- set output = [] -%} {%- for v in lspci_devices -%} - {%- if (v.ECAP_SRIOV | bool is true) and (v.set_numvfs == 'max' or v.set_numvfs | int > 0) -%} + {%- if (v.Ecap_sriov == 'yes') and (v.Set_numvfs == 'max' or v.Set_numvfs | int > 0) -%} {{- output.append(v) -}} {%- endif -%} {%- endfor -%} @@ -162,10 +149,10 @@ set -x -o errexit -o pipefail CHANGED=false {% for v in _to_enable %} - {% if v.set_numvfs == 'max' %} + {% if v.Set_numvfs == 'max' %} SRIOV_NUMVFS="$(head -n1 '/sys/bus/pci/devices/{{ v.Slot }}/sriov_totalvfs')" - {% elif v.set_numvfs | int > 0 %} - SRIOV_NUMVFS='{{ v.set_numvfs }}' + {% elif v.Set_numvfs | int > 0 %} + SRIOV_NUMVFS='{{ v.Set_numvfs }}' {% else %} unset SRIOV_NUMVFS {% endif %} @@ -203,7 +190,7 @@ _to_enable: >- {%- set output = [] -%} {%- for v in lspci_devices -%} - {%- if (v.ECAP_SRIOV | bool is true) and (v.set_numvfs == 'max' or v.set_numvfs | int > 0) -%} + {%- if (v.Ecap_sriov == 'yes') and (v.Set_numvfs == 'max' or v.Set_numvfs | int > 0) -%} {{- output.append(v) -}} {%- endif -%} {%- endfor -%} @@ -225,14 +212,14 @@ set -x -o errexit -o pipefail BEFORE="$(driverctl list-overrides | sort)" ||: {% for v in _to_override %} - if ! grep -E -m1 '^{{ v.Slot }}\s+{{ v.set_driver }}$' <<< "$BEFORE"; then - {% if (v.ECAP_SRIOV | bool is true) %} + if ! grep -E -m1 '^{{ v.Slot }}\s+{{ v.Set_driver }}$' <<< "$BEFORE"; then + {% if v.Ecap_sriov == 'yes' %} TO_DISABLE="$(systemctl show --all -P Id 'sriov-enable@{{ v.Slot }}-*.service' | grep -E -v '^\s*$')" ||: if [[ -n "$TO_DISABLE" ]] && systemctl is-active --quiet $TO_DISABLE; then systemctl disable --now $TO_DISABLE # cleanup spurious sriov-enable@ services fi {% endif %} - driverctl set-override '{{ v.Slot }}' '{{ v.set_driver }}' # force the new driver (vfio-pci by default) + driverctl set-override '{{ v.Slot }}' '{{ v.Set_driver }}' # force the new driver (vfio-pci by default) fi {% endfor %} AFTER="$(driverctl list-overrides | sort)" ||: @@ -247,7 +234,7 @@ _to_override: >- {%- set output = [] -%} {%- for v in lspci_devices -%} - {%- if (v.set_driver != 'omit') and ((v.ECAP_SRIOV | bool is false) or (v.set_numvfs != 'max' and v.set_numvfs | int == 0)) -%} + {%- if (v.Set_driver != 'omit') and ((v.Ecap_sriov == 'no') or (v.Set_numvfs != 'max' and v.Set_numvfs | int == 0)) -%} {{- output.append(v) -}} {%- endif -%} {%- endfor -%} diff --git a/roles/helper/pci/tasks/main.yml b/roles/helper/pci/tasks/main.yml index bfdd646c..f28efabe 100644 --- a/roles/helper/pci/tasks/main.yml +++ b/roles/helper/pci/tasks/main.yml @@ -1,9 +1,10 @@ --- -- when: pci_passthrough_enabled | bool is true +- when: + - pci_devices is sequence + - pci_devices | count > 0 block: - - ansible.builtin.import_tasks: + - ansible.builtin.include_tasks: file: "{{ role_path }}/tasks/devices.yml" - when: - - pci_devices is defined - - pci_devices is sequence - - pci_devices | count > 0 + + - ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/udev.yml" diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 334018fe..47b6839d 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -3,7 +3,7 @@ ansible.builtin.shell: cmd: | set -o errexit -o pipefail - {% for v in pci_devices_by_vendor_device_class %} + {% for v in pci_devices_by_vendor_device_class | selectattr('key_type', '==', 'vdc') %} STDOUT="$(lspci -vmm -nkD -d '{{ v.key }}')" if [[ -n "$STDOUT" ]]; then echo "$STDOUT" @@ -15,15 +15,19 @@ setpci -v -d '{{ v.key }}' STATUS | while IFS=' ' read -r SLOT _; do echo -e "Slot:\t$SLOT" if setpci -s "$SLOT" ECAP_SRIOV.B 1>/dev/null; then - echo -e "ECAP_SRIOV:\tyes" - echo + echo -e "Ecap_sriov:\tyes" else - echo -e "ECAP_SRIOV:\tno" - echo + echo -e "Ecap_sriov:\tno" fi + echo -e 'Set_driver:\t{{ v.set_driver | d('vfio-pci') }}' + echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' + echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' + echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' + echo done {% endfor %} - {% for v in pci_devices_by_address %} + {% for v in pci_devices_by_address | selectattr('key_type', '==', 'pci') %} STDOUT="$(lspci -vmm -nkD -s '{{ v.key }}')" if [[ -n "$STDOUT" ]]; then echo "$STDOUT" @@ -32,14 +36,51 @@ echo "Could not find '{{ v.key }}'" >&2 exit 1 fi - echo -e "Slot:\t{{ v.key }}" - if setpci -s '{{ v.key }}' ECAP_SRIOV.B 1>/dev/null; then - echo -e "ECAP_SRIOV:\tyes" + setpci -v -s '{{ v.key }}' STATUS | while IFS=' ' read -r SLOT _; do + echo -e "Slot:\t$SLOT" + if setpci -s "$SLOT" ECAP_SRIOV.B 1>/dev/null; then + echo -e "Ecap_sriov:\tyes" + else + echo -e "Ecap_sriov:\tno" + fi + echo -e 'Set_driver:\t{{ v.set_driver | d('vfio-pci') }}' + echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' + echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' + echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo - else - echo -e "ECAP_SRIOV:\tno" + done + {% endfor %} + {% for v in pci_devices_by_address | selectattr('key_type', '==', 'mac') %} + udevadm trigger -nv -s net -a 'address={{ v.key }}' | while read -r DEVPATH; do + IFS='/' read -r _ SYS DEVICES ROOT _ SLOT _ <<< "$DEVPATH" + if [[ "$SYS" != sys || "$DEVICES" != devices || "$ROOT" != pci* ]]; then + continue + fi + STDOUT="$(lspci -vmm -nkD -s "$SLOT")" + if [[ -n "$STDOUT" ]]; then + echo "$STDOUT" + echo + else + echo "Could not find '$SLOT'" >&2 + exit 1 + fi + echo -e "Slot:\t$SLOT" + if setpci -s "$SLOT" ECAP_SRIOV.B 1>/dev/null; then + echo -e "Ecap_sriov:\tyes" + else + echo -e "Ecap_sriov:\tno" + fi + if [[ -e "$DEVPATH/address" ]]; then + echo -e "Address:\t$(cat $DEVPATH/address)" + fi + echo -e 'Set_driver:\t{{ v.set_driver | d('vfio-pci') }}' + echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' + echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' + echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo - fi + done {% endfor %} executable: /bin/bash register: shell_lspci @@ -48,7 +89,7 @@ - name: Parse lspci's output ansible.builtin.set_fact: lspci_devices: >- - {{ _lspci_devices_filtered_and_extended }} + {{ _lspci_devices | rejectattr('Excluded', '==', 'yes') }} vars: _lspci_devices: >- {%- set output = {} -%} @@ -56,7 +97,6 @@ {{- output.update(output | combine({v.Slot: v}, recursive=true)) -}} {%- endfor -%} {{- output.values() | list -}} - _lspci_devices_raw: >- {{ shell_lspci.stdout | split(_sep1) | select('truthy') @@ -67,83 +107,6 @@ _sep2: "\n" _sep3: ":\t" # noqa no-tabs - _by_vendor_device_class_excluded: >- - {%- set output = [] -%} - {%- for v in pci_devices_by_vendor_device_class -%} - {%- if v.excluded | d(false) | bool is true -%} - {{- output.append(v) -}} - {%- endif -%} - {%- endfor -%} - {{- output -}} - - _by_address_excluded: >- - {%- set output = [] -%} - {%- for v in pci_devices_by_address -%} - {%- if v.excluded | d(false) | bool is true -%} - {{- output.append(v) -}} - {%- endif -%} - {%- endfor -%} - {{- output -}} - - _to_exclude: >- - {%- set output = [] -%} - {%- for v in _lspci_devices -%} - {%- set k = "{}:{}:{}".format(v.Vendor, v.Device, v.Class) -%} - {%- for x in _by_vendor_device_class_excluded -%} - {%- if k | regex_search(x.key_regex) -%} - {{- output.append(v.Slot) -}} - {%- endif -%} - {%- endfor -%} - {%- set k = v.Slot -%} - {%- for x in _by_address_excluded -%} - {%- if k | regex_search(x.key_regex) -%} - {{- output.append(v.Slot) -}} - {%- endif -%} - {%- endfor -%} - {%- endfor -%} - {{- output | unique -}} - - _lspci_devices_filtered: >- - {%- set output = [] -%} - {%- for v in _lspci_devices -%} - {%- if _to_exclude is not contains(v.Slot) -%} - {{- output.append(v) -}} - {%- endif -%} - {%- endfor -%} - {{- output -}} - - _lspci_devices_filtered_and_extended: >- - {%- set output = [] -%} - {%- for v in _lspci_devices_filtered -%} - {%- set to_append = [] %} - {%- for x in pci_devices_by_vendor_device_class -%} - {%- set k = "{}:{}:{}".format(v.Vendor, v.Device, v.Class) -%} - {%- if k | regex_search(x.key_regex) -%} - {{- - to_append.append(v | combine({ - "set_driver": x.set_driver | d('vfio-pci'), - "set_numvfs": x.set_numvfs | d(0), - })) - -}} - {%- endif -%} - {%- endfor -%} - {%- for x in pci_devices_by_address -%} - {%- set k = v.Slot -%} - {%- if k | regex_search(x.key_regex) -%} - {{- - to_append.append(v | combine({ - "set_driver": x.set_driver | d('vfio-pci'), - "set_numvfs": x.set_numvfs | d(0), - })) - -}} - {%- endif -%} - {%- endfor -%} - {%- if to_append | count > 0 -%} - {{- output.append(to_append | last) -}} - {%- endif -%} - {%- endfor -%} - {{- output -}} - - when: - pci_forbidden_addresses is undefined - _default is defined diff --git a/roles/helper/pci/tasks/udev.yml b/roles/helper/pci/tasks/udev.yml new file mode 100644 index 00000000..39950e7a --- /dev/null +++ b/roles/helper/pci/tasks/udev.yml @@ -0,0 +1,60 @@ +--- +- when: lspci_devices | count > 0 + vars: + _all: >- + {{ lspci_devices | selectattr('Set_name', 'defined') + | rejectattr('Set_name', '==', 'omit') + | selectattr('Class', 'defined') + | selectattr('Driver', 'defined') + | selectattr('Slot', 'defined') }} + _net: >- + {{ _all | selectattr('Class', 'in', ['0200']) + | rejectattr('Driver', 'in', ['vfio-pci']) }} + _pci: >- + {{ _all | rejectattr('Class', 'in', ['0200']) + | rejectattr('Driver', 'in', ['vfio-pci']) }} + _vfio: >- + {{ _all | selectattr('Driver', 'in', ['vfio-pci']) + | selectattr('IOMMUGroup', 'defined') }} + block: + - name: Render extra udev rules (/etc/udev/rules.d) + ansible.builtin.copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + owner: 0 + group: 0 + mode: u=rw,go=r + loop_control: { label: "{{ item.dest }}" } + loop: + - dest: /etc/udev/rules.d/99-rename.rules + content: | + # managed by one-deploy + # --- NET + {% for v in _net %} + {% set name = v.Set_name.format(v.Slot | regex_findall('[0-9a-fA-F]+')) %} + # {{ v.Slot }} <- {{ name }} + SUBSYSTEM=="net", ACTION=="add", ENV{ID_PATH}=="pci-{{ v.Slot }}", NAME="{{ name }}" + {% endfor %} + # --- PCI + {% for v in _pci %} + {% set name = v.Set_name.format(v.Slot | regex_findall('[0-9a-fA-F]+')) %} + # {{ v.Slot }} <- {{ name }} + SUBSYSTEM=="pci", ACTION=="add", ENV{ID_PATH}=="pci-{{ v.Slot }}", SYMLINK+="pci/by-tag/{{ name }}" + {% endfor %} + # --- VFIO + {% for v in _vfio %} + {% set name = v.Set_name.format(v.Slot | regex_findall('[0-9a-fA-F]+')) %} + # {{ v.Slot }} <- {{ name }} + SUBSYSTEM=="vfio", ACTION=="add", KERNEL=="{{ v.IOMMUGroup }}", SYMLINK+="vfio/by-tag/{{ name }}" + {% endfor %} + - dest: /etc/udev/rules.d/99-mode.rules + content: | + # managed by one-deploy + SUBSYSTEM=="vfio", ACTION=="add", GROUP="kvm", MODE="0666" + register: copy_udev_rules + + - name: Refresh udev rules + ansible.builtin.shell: + cmd: udevadm control --reload-rules && udevadm trigger --action=add + changed_when: true + when: copy_udev_rules is changed diff --git a/roles/openvswitch/tasks/main.yml b/roles/openvswitch/tasks/main.yml index 9c21bb7b..d2b242e6 100644 --- a/roles/openvswitch/tasks/main.yml +++ b/roles/openvswitch/tasks/main.yml @@ -248,24 +248,27 @@ | flatten | selectattr('key', 'in', ['addrs', 'gw', 'dns']) }} - - name: Ensure udev rules for vfio - ansible.builtin.copy: - dest: /etc/udev/rules.d/99-vfio.rules - owner: 0 - group: 0 - mode: u=rw,go=r - content: | - SUBSYSTEM=="vfio", GROUP="kvm", MODE="0666" - register: copy_udev_vfio + - block: + - name: Render extra udev rules (/etc/udev/rules.d) + ansible.builtin.copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + owner: 0 + group: 0 + mode: u=rw,go=r + loop_control: { label: "{{ item.dest }}" } + loop: + - dest: /etc/udev/rules.d/99-mode.rules + content: | + # managed by one-deploy + SUBSYSTEM=="vfio", ACTION=="add", GROUP="kvm", MODE="0666" + register: copy_udev_rules - - name: Refresh udev rules - ansible.builtin.shell: - cmd: | - set -o errexit - udevadm control --reload-rules && udevadm trigger - executable: /bin/bash - changed_when: true - when: copy_udev_vfio is changed + - name: Refresh udev rules + ansible.builtin.shell: + cmd: udevadm control --reload-rules && udevadm trigger --action=add --subsystem-match=vfio + changed_when: true + when: copy_udev_rules is changed - name: Install OVS packages ansible.builtin.package: diff --git a/roles/pci/frontend/README.md b/roles/pci/frontend/README.md index 83aca49f..106abb83 100644 --- a/roles/pci/frontend/README.md +++ b/roles/pci/frontend/README.md @@ -11,15 +11,14 @@ N/A Role Variables -------------- -| Name | Type | Default | Example | Description | -|----------------------------------|--------|---------|---------|------------------------------------------| -| `pci_passthrough_enabled` | `bool` | `false` | | Enable/Disable PCI passthrough. | -| `pci_passthrough_default_filter` | `bool` | `*:*` | | Default (global) PCI passthrough filter. | +| Name | Type | Default | Example | Description | +|----------------------|--------|---------|---------|------------------------------------------| +| `pci_default_filter` | `bool` | `*:*` | | Default (global) PCI passthrough filter. | Dependencies ------------ -- opennebula.deploy.opennebula.leader +N/A Example Playbook ---------------- diff --git a/roles/pci/frontend/defaults/main.yml b/roles/pci/frontend/defaults/main.yml index 21fdd75a..3aa11fe7 100644 --- a/roles/pci/frontend/defaults/main.yml +++ b/roles/pci/frontend/defaults/main.yml @@ -1,6 +1,2 @@ --- -pci_passthrough_enabled: >- - {{ (groups[node_group | d('node')] | map('extract', hostvars) - | map(attribute='pci_passthrough_enabled', default=false) - | map('bool')) is any }} -pci_passthrough_default_filter: "*:*" # NOTE: OpenNebula's default is "0:0" +pci_default_filter: "*:*" # NOTE: OpenNebula's default is "0:0" diff --git a/roles/pci/frontend/tasks/main.yml b/roles/pci/frontend/tasks/main.yml index d42f0e8f..635ce7d1 100644 --- a/roles/pci/frontend/tasks/main.yml +++ b/roles/pci/frontend/tasks/main.yml @@ -1,12 +1,21 @@ --- -- name: Apply global PCI filter - opennebula.deploy.cfgtool: - dest: /var/lib/one/remotes/etc/im/kvm-probes.d/pci.conf - parser: Yaml - actions: - - put: - path: [":filter"] - value: "{{ pci_passthrough_default_filter }}" - notify: - - Sync Remotes - when: pci_passthrough_enabled | bool is true +- when: _enabled is true + vars: + _enabled: >- + {{ _hosts | map('extract', hostvars) + | map(attribute='pci_devices', default=[]) + | select + | count > 0 }} + _hosts: >- + {{ groups[node_group | d('node')] }} + block: + - name: Apply global PCI filter + opennebula.deploy.cfgtool: + dest: /var/lib/one/remotes/etc/im/kvm-probes.d/pci.conf + parser: Yaml + actions: + - put: + path: [":filter"] + value: "{{ pci_default_filter }}" + notify: + - Sync Remotes diff --git a/roles/pci/node/README.md b/roles/pci/node/README.md index 48446dfb..21e3f6ee 100644 --- a/roles/pci/node/README.md +++ b/roles/pci/node/README.md @@ -11,10 +11,9 @@ N/A Role Variables -------------- -| Name | Type | Default | Example | Description | -|---------------------------|--------|---------|---------------|---------------------------------| -| `pci_passthrough_enabled` | `bool` | `false` | | Enable/Disable PCI passthrough. | -| `pci_devices` | `list` | `[]` | (check below) | PCI devices configuration. | +| Name | Type | Default | Example | Description | +|---------------|--------|---------|---------------|---------------------------------| +| `pci_devices` | `list` | `[]` | (check below) | PCI devices configuration. | Dependencies ------------ @@ -26,10 +25,12 @@ Example Playbook - hosts: node vars: - pci_passthrough_enabled: true pci_devices: - address: "0000:02:00.0" excluded: true + - address: "0000:03:00.*" + unlisted: true + set_name: "asd{0[3]}" - vendor: "1af4" device: "*" class: "0200" diff --git a/roles/pci/node/defaults/main.yml b/roles/pci/node/defaults/main.yml index 8e696a88..ff929a8c 100644 --- a/roles/pci/node/defaults/main.yml +++ b/roles/pci/node/defaults/main.yml @@ -1,3 +1,2 @@ --- -pci_passthrough_enabled: false pci_devices: [] diff --git a/roles/pci/node/tasks/filters.yml b/roles/pci/node/tasks/filters.yml index 48004541..4312f202 100644 --- a/roles/pci/node/tasks/filters.yml +++ b/roles/pci/node/tasks/filters.yml @@ -44,8 +44,8 @@ _after: PCI_SHORT_ADDRESS: >- {%- set output = [] -%} - {%- for v in lspci_devices -%} - {%- if (v.ECAP_SRIOV | bool is false) or (v.set_numvfs != 'max' and v.set_numvfs | int == 0) -%} + {%- for v in lspci_devices | rejectattr('Unlisted', '==', 'yes') -%} + {%- if (v.Ecap_sriov == 'no') or (v.Set_numvfs != 'max' and v.Set_numvfs | int == 0) -%} {{- output.append(v.Slot | regex_replace('^[^:]{4}:(.+)$', '\g<1>')) -}} {%- endif -%} {%- endfor -%} diff --git a/roles/pci/node/tasks/main.yml b/roles/pci/node/tasks/main.yml index 6a79eb6a..bdc26537 100644 --- a/roles/pci/node/tasks/main.yml +++ b/roles/pci/node/tasks/main.yml @@ -1,14 +1,20 @@ --- -- when: pci_passthrough_enabled | bool is true +- when: _enabled is true + vars: + _enabled: >- + {{ _hosts | map('extract', hostvars) + | map(attribute='pci_devices', default=[]) + | select + | count > 0 }} + _hosts: >- + {{ groups[node_group | d('node')] }} block: - ansible.builtin.include_role: name: opennebula.deploy.helper.pci - when: - - lspci_devices is undefined + when: lspci_devices is undefined - ansible.builtin.import_tasks: file: "{{ role_path }}/tasks/filters.yml" when: - lspci_devices is defined - - lspci_devices is sequence - lspci_devices | count > 0 diff --git a/roles/precheck/post_reboot/tasks/pci.yml b/roles/precheck/post_reboot/tasks/pci.yml index 24f51a32..f5a9732e 100644 --- a/roles/precheck/post_reboot/tasks/pci.yml +++ b/roles/precheck/post_reboot/tasks/pci.yml @@ -1,7 +1,21 @@ --- +- name: Ensure user renames / removes obsolete variables + ansible.builtin.assert: + that: + - pci_passthrough_enabled is undefined + - pci_passthrough_default_filter is undefined + fail_msg: | + Detected obsolete pci_passthrough_* variables. + Please remove 'pci_passthrough_enabled' as it no longer has any effect. + Please rename 'pci_passthrough_default_filter' to 'pci_default_filter'. + - name: Run PCI passthrough pre-flight checks - when: pci_passthrough_enabled | d(false) | bool + when: + - _pci_devices is sequence + - _pci_devices | count > 0 vars: + _pci_devices: >- + {{ pci_devices | d([]) }} _virtualization_flag: >- {{ 'vmx' if 'GenuineIntel' in ansible_facts.processor else 'svm' if 'AuthenticAMD' in ansible_facts.processor else '' }} diff --git a/roles/precheck/pre_reboot/tasks/features.yml b/roles/precheck/pre_reboot/tasks/features.yml index 8136e9f9..bbf201d3 100644 --- a/roles/precheck/pre_reboot/tasks/features.yml +++ b/roles/precheck/pre_reboot/tasks/features.yml @@ -94,8 +94,6 @@ - name: Check if PCI / SR-IOV management is supported for the current distro ansible.builtin.assert: that: - (pci_passthrough_enabled | d(false) | bool is false) - or (pci_devices | d([]) | count == 0) or (not (ansible_distribution == 'SLES' and ansible_distribution_major_version == '15')) From b69cffd508c7cbf56b25c1fc4ee51f034222254f Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Wed, 6 May 2026 11:34:05 +0200 Subject: [PATCH 02/13] F #183: Maintain device order during processing (fix) Imperative top-to-bottom processing should be less confusing to users and allow mixing of address and vendor/device/class queries in predictable ways. Signed-off-by: Michal Opala --- roles/helper/pci/tasks/devices.yml | 56 ++++++++++++++---------------- roles/helper/pci/tasks/query.yml | 10 +++--- 2 files changed, 32 insertions(+), 34 deletions(-) diff --git a/roles/helper/pci/tasks/devices.yml b/roles/helper/pci/tasks/devices.yml index 979c57c2..9725a5cb 100644 --- a/roles/helper/pci/tasks/devices.yml +++ b/roles/helper/pci/tasks/devices.yml @@ -13,41 +13,39 @@ retries: 12 delay: 5 -- name: Parse and split pci_devices +- name: Parse pci_devices (user input) ansible.builtin.set_fact: - pci_devices_by_vendor_device_class: >- + pci_devices: >- {%- set output = [] -%} - {%- for v in pci_devices | selectattr('address', 'undefined') -%} - {{- - output.append(v | combine({ - "key": "{}:{}:{}".format( - v.vendor | d('*'), - v.device | d('*'), - v.class | d('*'), - ) | lower, - "key_type": "vdc", - })) - -}} - {%- endfor -%} - {{- output -}} - pci_devices_by_address: >- - {%- set output = [] -%} - {%- for v in pci_devices | selectattr('address', 'defined') -%} - {%- if v.address is opennebula.deploy.match_address('*:*:*.*', sep='[:.]') -%} - {{- - output.append(v | combine({ - "key": v.address | lower, - "key_type": "pci", - })) - -}} - {%- endif -%} - {%- if v.address is opennebula.deploy.match_address('*:*:*:*:*:*', sep='[:]') -%} + {%- for v in pci_devices -%} + {%- if v.address is undefined -%} {{- output.append(v | combine({ - "key": v.address | lower, - "key_type": "mac", + "key": "{}:{}:{}".format( + v.vendor | d('*'), + v.device | d('*'), + v.class | d('*'), + ) | lower, + "key_type": "vdc", })) -}} + {%- else -%} + {%- if v.address is opennebula.deploy.match_address('*:*:*.*', sep='[:.]') -%} + {{- + output.append(v | combine({ + "key": v.address | lower, + "key_type": "pci", + })) + -}} + {%- endif -%} + {%- if v.address is opennebula.deploy.match_address('*:*:*:*:*:*', sep='[:]') -%} + {{- + output.append(v | combine({ + "key": v.address | lower, + "key_type": "mac", + })) + -}} + {%- endif -%} {%- endif -%} {%- endfor -%} {{- output -}} diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 47b6839d..dce6d958 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -3,7 +3,8 @@ ansible.builtin.shell: cmd: | set -o errexit -o pipefail - {% for v in pci_devices_by_vendor_device_class | selectattr('key_type', '==', 'vdc') %} + {% for v in pci_devices %} + {% if v.key_type == 'vdc' %} STDOUT="$(lspci -vmm -nkD -d '{{ v.key }}')" if [[ -n "$STDOUT" ]]; then echo "$STDOUT" @@ -26,8 +27,7 @@ echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo done - {% endfor %} - {% for v in pci_devices_by_address | selectattr('key_type', '==', 'pci') %} + {% elif v.key_type == 'pci' %} STDOUT="$(lspci -vmm -nkD -s '{{ v.key }}')" if [[ -n "$STDOUT" ]]; then echo "$STDOUT" @@ -50,8 +50,7 @@ echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo done - {% endfor %} - {% for v in pci_devices_by_address | selectattr('key_type', '==', 'mac') %} + {% elif v.key_type == 'mac' %} udevadm trigger -nv -s net -a 'address={{ v.key }}' | while read -r DEVPATH; do IFS='/' read -r _ SYS DEVICES ROOT _ SLOT _ <<< "$DEVPATH" if [[ "$SYS" != sys || "$DEVICES" != devices || "$ROOT" != pci* ]]; then @@ -81,6 +80,7 @@ echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo done + {% endif %} {% endfor %} executable: /bin/bash register: shell_lspci From 93592daf91a81c9de113648328974e6555087608 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Wed, 6 May 2026 12:39:02 +0200 Subject: [PATCH 03/13] F #183: Never protect devices claimed by openvswitch role (fix) Signed-off-by: Michal Opala --- roles/helper/pci/tasks/query.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index dce6d958..d91177ad 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -123,29 +123,33 @@ {{- output.append(_facts[v].slaves | d([_facts[v].device | d(none)] | select)) -}} {%- endfor -%} {{- output | flatten -}} - # Do also a quick crosscheck with OVS/DPDK config (when available). + # Extract info from OVS/DPDK config (when available). + _ovs_interfaces: >- + {{ (ovs.iface | d({})).keys() | intersect(_facts.interfaces) }} _dpdk_pci_addrs: >- {{ (ovs.iface | d({})).values() | map(attribute='set', default=[]) | map('selectattr', 'options:dpdk-devargs', 'defined') | select | flatten | map(attribute='options:dpdk-devargs') }} - _dpdk_interfaces: >- - {{ (ovs.iface | d({})).keys() | intersect(_facts.interfaces) }} block: - name: Query udev for device info ansible.builtin.command: cmd: "udevadm info --query=property --property=ID_PATH --value {{ _paths | join(' ') }}" vars: + # NOTE: OVS interfaces are excluded here, as openvswitch role is allowed to handle primary NICs. + # Please make sure to use 'set_driver: omit' so openvswitch role entirely decides what driver should be used. _paths: >- - {{ (_interfaces + _dpdk_interfaces) | unique - | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} + {{ _interfaces | difference(_ovs_interfaces) + | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} register: command_udevadm_info changed_when: false - name: Gather forbidden PCI addresses ansible.builtin.set_fact: - pci_forbidden_addresses: "{{ (_pci_addrs + _dpdk_pci_addrs) | unique }}" + # NOTE: DPDK PCI addresses are excluded here, as openvswitch role is allowed to handle primary NICs. + # Please make sure to use 'set_driver: omit' so openvswitch role entirely decides what driver should be used. + pci_forbidden_addresses: "{{ _pci_addrs | difference(_dpdk_pci_addrs) }}" vars: _pci_addrs: >- {{ command_udevadm_info.stdout_lines | select @@ -161,7 +165,6 @@ fail_msg: >- Forbidden PCI addresses {{ _detected }} detected, aborting! Please adjust 'pci_devices' to exclude forbidden PCI addresses. - You might also want to look for conflicts with OVS/DPDK config. vars: _detected: >- {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses) }} From 1b6416a714f50771948d1a7f85bcf268d5eb34a2 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Wed, 6 May 2026 16:19:05 +0200 Subject: [PATCH 04/13] F #183: Do not modify drivers by default Require users to define driver explicitly to prevent later confusion. Signed-off-by: Michal Opala --- roles/helper/pci/tasks/query.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index d91177ad..69bc2783 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -20,7 +20,7 @@ else echo -e "Ecap_sriov:\tno" fi - echo -e 'Set_driver:\t{{ v.set_driver | d('vfio-pci') }}' + echo -e 'Set_driver:\t{{ v.set_driver | d('omit') }}' echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' @@ -43,7 +43,7 @@ else echo -e "Ecap_sriov:\tno" fi - echo -e 'Set_driver:\t{{ v.set_driver | d('vfio-pci') }}' + echo -e 'Set_driver:\t{{ v.set_driver | d('omit') }}' echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' @@ -73,7 +73,7 @@ if [[ -e "$DEVPATH/address" ]]; then echo -e "Address:\t$(cat $DEVPATH/address)" fi - echo -e 'Set_driver:\t{{ v.set_driver | d('vfio-pci') }}' + echo -e 'Set_driver:\t{{ v.set_driver | d('omit') }}' echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' From c008d76f669252ab9b428b0a4fd8449eafcf407c Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Wed, 6 May 2026 18:30:43 +0200 Subject: [PATCH 05/13] F #183: Soften PCI vs DPDK conflict condition (fix) Ignore devices for which helper/pci role does not manage driver changes. Signed-off-by: Michal Opala --- roles/openvswitch/tasks/main.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/roles/openvswitch/tasks/main.yml b/roles/openvswitch/tasks/main.yml index d2b242e6..ce810fda 100644 --- a/roles/openvswitch/tasks/main.yml +++ b/roles/openvswitch/tasks/main.yml @@ -116,15 +116,18 @@ that: _dpdk_pci_addrs_raw == _dpdk_pci_addrs_items # no invalid, no duplicated fail_msg: Please remove invalid / duplicated DPDK PCI addresses from OVS config. - - name: Assert DPDK PCI devices are not already claimed by PCI/SR-IOV management + - name: Assert DPDK PCI device drivers are not already claimed by PCI/SR-IOV management ansible.builtin.assert: that: _intersection | count == 0 - fail_msg: DPDK vs PCI/SR-IOV conflict detected. Please ensure {{ _intersection }} are not being claimed by both implementations. + fail_msg: >- + DPDK vs PCI/SR-IOV conflict detected. + Please ensure drivers of {{ _intersection }} are not managed by helper/pci role already. vars: _intersection: >- {{ _dpdk_pci_addrs.keys() | intersect(_slots) }} _slots: >- {{ lspci_devices | selectattr('Slot', 'defined') + | rejectattr('Set_driver', 'in', ['omit']) | map(attribute='Slot') }} when: # NOTE: The 'lspci_devices' fact is produced by the helper/pci role. From dc84ea1c2814cfc3279e77b137c14895a6d288ba Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Wed, 6 May 2026 18:38:33 +0200 Subject: [PATCH 06/13] F #183: Run after-reboot SR-IOV management earlier (fix) Allow opennebula-ovs.service to pick up renamed devices after reboot. This makes it possible to mix PCI/SR-IOV and OVS/DPDK devices. Signed-off-by: Michal Opala --- roles/helper/pci/tasks/devices.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/helper/pci/tasks/devices.yml b/roles/helper/pci/tasks/devices.yml index 9725a5cb..2389290a 100644 --- a/roles/helper/pci/tasks/devices.yml +++ b/roles/helper/pci/tasks/devices.yml @@ -86,7 +86,7 @@ cmd: | [Unit] Description=Enable SR-IOV VFs on %I - After=network.target + After=network-pre.target [Service] Type=oneshot From 68528ca213f57d038607c4d33df691b776dc24ac Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Wed, 6 May 2026 20:14:19 +0200 Subject: [PATCH 07/13] F #183: Update README.md files for helper/pci and openvswitch roles Signed-off-by: Michal Opala --- roles/helper/pci/README.md | 81 +++++++++++++++++++++++++++++++++---- roles/openvswitch/README.md | 14 ++++++- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/roles/helper/pci/README.md b/roles/helper/pci/README.md index 1a58888b..30944fdc 100644 --- a/roles/helper/pci/README.md +++ b/roles/helper/pci/README.md @@ -11,9 +11,18 @@ N/A Role Variables -------------- -| Name | Type | Default | Example | Description | -|---------------|--------|---------|---------------|---------------------------------| -| `pci_devices` | `list` | `[]` | (check below) | PCI devices configuration. | +| Name | Type | Default | Example | Description | +|-----------------------------|-------|-----------|---------------|------------------------------------------------------------------------------| +| `pci_devices` | `list`| `[]` | (check below) | PCI devices configuration. | +| `pci_devices[*].excluded` | `bool`| `false` | (check below) | Do not process matching PCI devices. | +| `pci_devices[*].unlisted` | `bool`| `false` | (check below) | Do not pass matching PCI devices to OpenNebula. | +| `pci_devices[*].address` | `str` | undefined | (check below) | Glob PCI devices by PCI or MAC address. | +| `pci_devices[*].vendor` | `str` | `*` | (check below) | Glob PCI devices by PCI Vendor (if address is undefined). | +| `pci_devices[*].device` | `str` | `*` | (check below) | Glob PCI devices by PCI Device (if address is undefined). | +| `pci_devices[*].class` | `str` | `*` | (check below) | Glob PCI devices by PCI Class (if address is undefined). | +| `pci_devices[*].set_driver` | `str` | `omit` | (check below) | Use driverctl to override driver (unless "omit"). | +| `pci_devices[*].set_name` | `str` | `omit` | (check below) | Rename device in udev (unless "omit"). | +| `pci_devices[*].set_numvfs` | `str` | `0` | (check below) | Enable Virtual Functions for SR-IOV capable devices (integer >= 0 or "max"). | Dependencies ------------ @@ -23,19 +32,77 @@ N/A Example Playbook ---------------- + # NOTE: Dicts defined in pci_devices are processed top-to-bottom without merging. + - hosts: node vars: pci_devices: + # Rename virtio-net-pci devices unless configured otherwise later (below). + - vendor: "1af4" + device: "*" + class: "0200" + # NOTE: 0[0] -> PCI Domain + # 0[1] -> PCI Bus + # 0[2] -> PCI Device + # 0[3] -> PCI Function + set_name: "asd{0[1]}v{0[3]}" + + # Do not rename "0000:04:00.0" + If it's SR-IOV capable, enable all available VFs. + - address: "0000:04:00.0" + set_numvfs: max + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.pci + + - hosts: node + vars: + pci_devices: + # Exclude primary NIC from processing. - address: "0000:02:00.0" excluded: true + + # Enable 2 VFs and rename them to asd3v1, asd3v2 + Make sure OpenNebula doesn't use them (unlisted <- true). - address: "0000:03:00.*" + set_name: "asd3v{0[3]}" unlisted: true - set_name: "asd{0[3]}" - - vendor: "1af4" - device: "*" + - address: "0000:03:00.0" + set_numvfs: 2 + + # Enable 2 VFs and rename them to asd4v1, asd4v2 + Make sure OpenNebula does use them (unlisted <- false). + - address: "0000:04:00.*" + set_driver: vfio-pci # usual requirement for OpenNebula VMs + set_name: "asd4v{0[3]}" + - address: "0000:04:00.0" + set_numvfs: 2 + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.pci + + - hosts: node + vars: + pci_devices: + # Enable all available VFs for all existing Mellanox PFs. + - vendor: "15b3" + device: "1015" class: "0200" - set_driver: omit # NOTE: 'vfio-pci' is the default, 'omit' skips override altogether set_numvfs: max + + # Rename all existing Mellanox VFs. + - vendor: "15b3" + device: "1016" + class: "0200" + set_name: "vf{0[2]}{0[3]}" + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.pci + + - hosts: node + vars: + pci_devices: + # Rename and unlist all NICs matching MAC address wildcard. + - address: "52:54:00:12:3*:*" + set_name: "unlist{0[1]}v{0[3]}" + unlisted: true roles: - role: opennebula.deploy.helper.facts - role: opennebula.deploy.helper.pci diff --git a/roles/openvswitch/README.md b/roles/openvswitch/README.md index 97bdee20..d0600461 100644 --- a/roles/openvswitch/README.md +++ b/roles/openvswitch/README.md @@ -48,6 +48,13 @@ Example Playbook - codeready-builder-for-rhel-9-x86_64-rpms - rhel-9-for-x86_64-highavailability-rpms - fast-datapath-for-rhel-9-x86_64-rpms + pci_devices: + - address: '0000:04:00.0' + set_name: asd123 + unlisted: true + - address: '52:54:00:12:34:56' + set_name: asd321 + unlisted: true ovs: set: - other_config:dpdk-init: 'true' @@ -77,7 +84,9 @@ Example Playbook - mtu_request: 9000 - options:dpdk-devargs: '0000:03:00.0' driver: omit # skip forcing the driver - eth3: {} # non-DPDK device + asd123: {} # non-DPDK device (renamed) + asd321: {} # non-DPDK device (renamed) + eth4: {} # non-DPDK device bond: bond0: ifaces: [dpdk-p1, dpdk-p2] @@ -94,10 +103,11 @@ Example Playbook gw: "{{ ansible_default_ipv4.gateway }}" dns: ["{{ ansible_default_ipv4.gateway }}"] ovsbr1: # non-DPDK bridge - ports: [eth3] + ports: [asd123, asd321, eth4] roles: - role: opennebula.deploy.helper.facts - role: opennebula.deploy.helper.kernel + - role: opennebula.deploy.helper.pci - role: opennebula.deploy.repository - role: opennebula.deploy.openvswitch From 068f6cf94ed1d71881f36f4691821b385fab2c1a Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Fri, 8 May 2026 16:43:23 +0200 Subject: [PATCH 08/13] F #183: Do not execute udevadm info on an empty device set (fix) Signed-off-by: Michal Opala --- roles/helper/pci/tasks/query.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 69bc2783..150e586e 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -110,7 +110,7 @@ - when: - pci_forbidden_addresses is undefined - _default is defined - - _interfaces | count > 0 + - _paths | count > 0 vars: _facts: >- {{ ansible_facts }} @@ -132,16 +132,15 @@ | select | flatten | map(attribute='options:dpdk-devargs') }} + # NOTE: OVS interfaces are excluded here, as openvswitch role is allowed to handle primary NICs. + # Please make sure to use 'set_driver: omit' so openvswitch role entirely decides what driver should be used. + _paths: >- + {{ _interfaces | difference(_ovs_interfaces) + | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} block: - name: Query udev for device info ansible.builtin.command: cmd: "udevadm info --query=property --property=ID_PATH --value {{ _paths | join(' ') }}" - vars: - # NOTE: OVS interfaces are excluded here, as openvswitch role is allowed to handle primary NICs. - # Please make sure to use 'set_driver: omit' so openvswitch role entirely decides what driver should be used. - _paths: >- - {{ _interfaces | difference(_ovs_interfaces) - | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} register: command_udevadm_info changed_when: false From 86c187b41f7ca082b7f350bf39f08f88631d795f Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Sun, 10 May 2026 20:46:17 +0200 Subject: [PATCH 09/13] F #183: Provide default value for pci_forbidden_addresses (fix) Signed-off-by: Michal Opala --- roles/helper/pci/tasks/query.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 150e586e..7628161b 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -166,4 +166,4 @@ Please adjust 'pci_devices' to exclude forbidden PCI addresses. vars: _detected: >- - {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses) }} + {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses | d([])) }} From 1a001583d0edfcc4c6b4fae3470ca752649b6783 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 11 May 2026 14:52:28 +0200 Subject: [PATCH 10/13] Revert "F #183: Provide default value for pci_forbidden_addresses (fix)" This reverts commit 86c187b41f7ca082b7f350bf39f08f88631d795f. --- roles/helper/pci/tasks/query.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 7628161b..150e586e 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -166,4 +166,4 @@ Please adjust 'pci_devices' to exclude forbidden PCI addresses. vars: _detected: >- - {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses | d([])) }} + {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses) }} From 51c9fd25579d910f80e0a07d17bb9e46d35db7f9 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 11 May 2026 14:52:30 +0200 Subject: [PATCH 11/13] Revert "F #183: Do not execute udevadm info on an empty device set (fix)" This reverts commit 068f6cf94ed1d71881f36f4691821b385fab2c1a. --- roles/helper/pci/tasks/query.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 150e586e..69bc2783 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -110,7 +110,7 @@ - when: - pci_forbidden_addresses is undefined - _default is defined - - _paths | count > 0 + - _interfaces | count > 0 vars: _facts: >- {{ ansible_facts }} @@ -132,15 +132,16 @@ | select | flatten | map(attribute='options:dpdk-devargs') }} - # NOTE: OVS interfaces are excluded here, as openvswitch role is allowed to handle primary NICs. - # Please make sure to use 'set_driver: omit' so openvswitch role entirely decides what driver should be used. - _paths: >- - {{ _interfaces | difference(_ovs_interfaces) - | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} block: - name: Query udev for device info ansible.builtin.command: cmd: "udevadm info --query=property --property=ID_PATH --value {{ _paths | join(' ') }}" + vars: + # NOTE: OVS interfaces are excluded here, as openvswitch role is allowed to handle primary NICs. + # Please make sure to use 'set_driver: omit' so openvswitch role entirely decides what driver should be used. + _paths: >- + {{ _interfaces | difference(_ovs_interfaces) + | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} register: command_udevadm_info changed_when: false From 257d0cd1514d50dbdc8d4a245b9e947ee92ce459 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 11 May 2026 15:01:03 +0200 Subject: [PATCH 12/13] Revert "F #183: Never protect devices claimed by openvswitch role (fix)" This reverts commit 93592daf91a81c9de113648328974e6555087608. --- roles/helper/pci/tasks/query.yml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 69bc2783..8fc77ab3 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -123,33 +123,29 @@ {{- output.append(_facts[v].slaves | d([_facts[v].device | d(none)] | select)) -}} {%- endfor -%} {{- output | flatten -}} - # Extract info from OVS/DPDK config (when available). - _ovs_interfaces: >- - {{ (ovs.iface | d({})).keys() | intersect(_facts.interfaces) }} + # Do also a quick crosscheck with OVS/DPDK config (when available). _dpdk_pci_addrs: >- {{ (ovs.iface | d({})).values() | map(attribute='set', default=[]) | map('selectattr', 'options:dpdk-devargs', 'defined') | select | flatten | map(attribute='options:dpdk-devargs') }} + _dpdk_interfaces: >- + {{ (ovs.iface | d({})).keys() | intersect(_facts.interfaces) }} block: - name: Query udev for device info ansible.builtin.command: cmd: "udevadm info --query=property --property=ID_PATH --value {{ _paths | join(' ') }}" vars: - # NOTE: OVS interfaces are excluded here, as openvswitch role is allowed to handle primary NICs. - # Please make sure to use 'set_driver: omit' so openvswitch role entirely decides what driver should be used. _paths: >- - {{ _interfaces | difference(_ovs_interfaces) - | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} + {{ (_interfaces + _dpdk_interfaces) | unique + | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} register: command_udevadm_info changed_when: false - name: Gather forbidden PCI addresses ansible.builtin.set_fact: - # NOTE: DPDK PCI addresses are excluded here, as openvswitch role is allowed to handle primary NICs. - # Please make sure to use 'set_driver: omit' so openvswitch role entirely decides what driver should be used. - pci_forbidden_addresses: "{{ _pci_addrs | difference(_dpdk_pci_addrs) }}" + pci_forbidden_addresses: "{{ (_pci_addrs + _dpdk_pci_addrs) | unique }}" vars: _pci_addrs: >- {{ command_udevadm_info.stdout_lines | select @@ -165,6 +161,7 @@ fail_msg: >- Forbidden PCI addresses {{ _detected }} detected, aborting! Please adjust 'pci_devices' to exclude forbidden PCI addresses. + You might also want to look for conflicts with OVS/DPDK config. vars: _detected: >- {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses) }} From 11b05b278a794b5a226e1954ac61f674760e9ff2 Mon Sep 17 00:00:00 2001 From: Michal Opala Date: Mon, 11 May 2026 15:44:46 +0200 Subject: [PATCH 13/13] F #183: Introduce 'unguarded' attribute (fix) - Add generic way to skip primary NIC checks (unguarded: true) - Decouple helper/pci from openvswitch role - Update README.md Signed-off-by: Michal Opala --- roles/helper/pci/README.md | 37 +++++++++++++++++++++----------- roles/helper/pci/tasks/query.yml | 23 +++++++++----------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/roles/helper/pci/README.md b/roles/helper/pci/README.md index 30944fdc..ac2a8e94 100644 --- a/roles/helper/pci/README.md +++ b/roles/helper/pci/README.md @@ -11,18 +11,19 @@ N/A Role Variables -------------- -| Name | Type | Default | Example | Description | -|-----------------------------|-------|-----------|---------------|------------------------------------------------------------------------------| -| `pci_devices` | `list`| `[]` | (check below) | PCI devices configuration. | -| `pci_devices[*].excluded` | `bool`| `false` | (check below) | Do not process matching PCI devices. | -| `pci_devices[*].unlisted` | `bool`| `false` | (check below) | Do not pass matching PCI devices to OpenNebula. | -| `pci_devices[*].address` | `str` | undefined | (check below) | Glob PCI devices by PCI or MAC address. | -| `pci_devices[*].vendor` | `str` | `*` | (check below) | Glob PCI devices by PCI Vendor (if address is undefined). | -| `pci_devices[*].device` | `str` | `*` | (check below) | Glob PCI devices by PCI Device (if address is undefined). | -| `pci_devices[*].class` | `str` | `*` | (check below) | Glob PCI devices by PCI Class (if address is undefined). | -| `pci_devices[*].set_driver` | `str` | `omit` | (check below) | Use driverctl to override driver (unless "omit"). | -| `pci_devices[*].set_name` | `str` | `omit` | (check below) | Rename device in udev (unless "omit"). | -| `pci_devices[*].set_numvfs` | `str` | `0` | (check below) | Enable Virtual Functions for SR-IOV capable devices (integer >= 0 or "max"). | +| Name | Type | Default | Example | Description | +|-----------------------------|-------|-----------|---------------|-------------------------------------------------------------------------------------| +| `pci_devices` | `list`| `[]` | (check below) | PCI devices configuration. | +| `pci_devices[*].excluded` | `bool`| `false` | (check below) | Do not process matching PCI devices. | +| `pci_devices[*].unguarded` | `bool`| `false` | (check below) | Do not protect matching PCI devices (this may cause primary NIC connectivity loss). | +| `pci_devices[*].unlisted` | `bool`| `false` | (check below) | Do not pass matching PCI devices to OpenNebula. | +| `pci_devices[*].address` | `str` | undefined | (check below) | Glob PCI devices by PCI or MAC address. | +| `pci_devices[*].vendor` | `str` | `*` | (check below) | Glob PCI devices by PCI Vendor (if address is undefined). | +| `pci_devices[*].device` | `str` | `*` | (check below) | Glob PCI devices by PCI Device (if address is undefined). | +| `pci_devices[*].class` | `str` | `*` | (check below) | Glob PCI devices by PCI Class (if address is undefined). | +| `pci_devices[*].set_driver` | `str` | `omit` | (check below) | Use driverctl to override driver (unless "omit"). | +| `pci_devices[*].set_name` | `str` | `omit` | (check below) | Rename device in udev (unless "omit"). | +| `pci_devices[*].set_numvfs` | `str` | `0` | (check below) | Enable Virtual Functions for SR-IOV capable devices (integer >= 0 or "max"). | Dependencies ------------ @@ -78,6 +79,18 @@ Example Playbook - role: opennebula.deploy.helper.facts - role: opennebula.deploy.helper.pci + - hosts: node + vars: + pci_devices: + # Process primary NIC by disabling its protection (NOTE: in general, this may cause connectivity loss!). + - address: "0000:02:00.0" + unguarded: true + unlisted: true + set_name: asd0 + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.pci + - hosts: node vars: pci_devices: diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 8fc77ab3..fd1ab9e2 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -24,6 +24,7 @@ echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Unguarded:\t{{ v.unguarded | d(false) | bool | ternary('yes', 'no') }}' echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo done @@ -47,6 +48,7 @@ echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Unguarded:\t{{ v.unguarded | d(false) | bool | ternary('yes', 'no') }}' echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo done @@ -77,6 +79,7 @@ echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Unguarded:\t{{ v.unguarded | d(false) | bool | ternary('yes', 'no') }}' echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo done @@ -123,30 +126,23 @@ {{- output.append(_facts[v].slaves | d([_facts[v].device | d(none)] | select)) -}} {%- endfor -%} {{- output | flatten -}} - # Do also a quick crosscheck with OVS/DPDK config (when available). - _dpdk_pci_addrs: >- - {{ (ovs.iface | d({})).values() | map(attribute='set', default=[]) - | map('selectattr', 'options:dpdk-devargs', 'defined') - | select - | flatten - | map(attribute='options:dpdk-devargs') }} - _dpdk_interfaces: >- - {{ (ovs.iface | d({})).keys() | intersect(_facts.interfaces) }} block: - name: Query udev for device info ansible.builtin.command: cmd: "udevadm info --query=property --property=ID_PATH --value {{ _paths | join(' ') }}" vars: _paths: >- - {{ (_interfaces + _dpdk_interfaces) | unique - | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} + {{ _interfaces | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} register: command_udevadm_info changed_when: false - name: Gather forbidden PCI addresses ansible.builtin.set_fact: - pci_forbidden_addresses: "{{ (_pci_addrs + _dpdk_pci_addrs) | unique }}" + pci_forbidden_addresses: "{{ _pci_addrs | difference(_pci_addrs_unguarded) }}" vars: + _pci_addrs_unguarded: >- + {{ lspci_devices | selectattr('Unguarded', '==', 'yes') + | map(attribute='Slot') }} _pci_addrs: >- {{ command_udevadm_info.stdout_lines | select | map('regex_replace', '^pci-', '') }} @@ -164,4 +160,5 @@ You might also want to look for conflicts with OVS/DPDK config. vars: _detected: >- - {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses) }} + {{ lspci_devices | map(attribute='Slot') + | intersect(pci_forbidden_addresses | d([])) }}