diff --git a/plugins/test/main.py b/plugins/test/main.py new file mode 100644 index 00000000..b8b2d9a8 --- /dev/null +++ b/plugins/test/main.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import fnmatch +import re + +class TestModule(object): + def tests(self): + return { + 'match_address': self.match_address, + } + + # NOTE: It does not validate character classes or character count! + # EXAMPLES: + # pci -> match_address('0000:0*:00.*', sep='[:.]') + # mac -> match_address('52:54:*:*:*:0*', sep='[:]') + def match_address(self, address, pattern, sep=None): + """Tests if a split string matches a set of simple glob patterns.""" + + # In case no separator is provided simply match the whole string. + if sep is None: + return fnmatch.fnmatch(address, pattern) + + # Make sure both address and pattern contain identical separators. + if re.findall(sep, address) != re.findall(sep, pattern): + return False + + # Try matching (glob) each segment separately. + for x, y in zip(re.split(sep, address), re.split(sep, pattern)): + if not fnmatch.fnmatch(x, y): + return False + + return True diff --git a/plugins/test/match_address.yml b/plugins/test/match_address.yml new file mode 100644 index 00000000..56cb543e --- /dev/null +++ b/plugins/test/match_address.yml @@ -0,0 +1,43 @@ +--- +# Copyright: OpenNebula Project, OpenNebula Systems +# Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +DOCUMENTATION: + name: match_address + short_description: Apply glob patterns to a generic address. + description: + - Tests if a split string matches a set of simple glob patterns. + options: + _input: + description: + - An address to match. + type: string + required: true + sep: + description: + - A regex expression used to split the input string (optional). + type: string + required: false + author: + - Michal Opala (@sk4zuzu) + +EXAMPLES: | + - name: Match MAC address + ansible.builtin.debug: + msg: >- + {{ '12:34:56:78:90:ab' is opennebula.deploy.match_address('12:34:56:78:90:*', sep='[:]') }} + + - name: Select matching PCI addresses + ansible.builtin.debug: + msg: >- + {{ _addresses | select('opennebula.deploy.match_address', '0000:03:00.*', sep='[:.]') }} + vars: + _addresses: + - '12:34:56:78:90:ab' # MAC (skipped) + - '0000:03:00.4' # PCI (matched) + - '0000:04:00.0' # PCI (skipped) + +RETURN: + _value: + description: Returns True if the check passes. + type: boolean diff --git a/roles/helper/pci/README.md b/roles/helper/pci/README.md index c48a6d69..b96f6dc6 100644 --- a/roles/helper/pci/README.md +++ b/roles/helper/pci/README.md @@ -11,10 +11,19 @@ N/A Role Variables -------------- -| Name | Type | Default | Example | Description | -|---------------------------|--------|---------|---------------|---------------------------------| -| `pci_passthrough_enabled` | `bool` | `false` | | Enable/Disable PCI passthrough. | -| `pci_devices` | `list` | `[]` | (check below) | PCI devices configuration. | +| Name | Type | Default | Example | Description | +|-----------------------------|-------|-----------|---------------|-------------------------------------------------------------------------------------| +| `pci_devices` | `list`| `[]` | (check below) | PCI devices configuration. | +| `pci_devices[*].excluded` | `bool`| `false` | (check below) | Do not process matching PCI devices. | +| `pci_devices[*].unguarded` | `bool`| `false` | (check below) | Do not protect matching PCI devices (this may cause primary NIC connectivity loss). | +| `pci_devices[*].unlisted` | `bool`| `false` | (check below) | Do not pass matching PCI devices to OpenNebula. | +| `pci_devices[*].address` | `str` | undefined | (check below) | Glob PCI devices by PCI or MAC address. | +| `pci_devices[*].vendor` | `str` | `*` | (check below) | Glob PCI devices by PCI Vendor (if address is undefined). | +| `pci_devices[*].device` | `str` | `*` | (check below) | Glob PCI devices by PCI Device (if address is undefined). | +| `pci_devices[*].class` | `str` | `*` | (check below) | Glob PCI devices by PCI Class (if address is undefined). | +| `pci_devices[*].set_driver` | `str` | `omit` | (check below) | Use driverctl to override driver (unless "omit"). | +| `pci_devices[*].set_name` | `str` | `omit` | (check below) | Rename device in udev (unless "omit"). | +| `pci_devices[*].set_numvfs` | `str` | `0` | (check below) | Enable Virtual Functions for SR-IOV capable devices (integer >= 0 or "max"). | Dependencies ------------ @@ -24,17 +33,96 @@ N/A Example Playbook ---------------- + # NOTE: Dicts defined in pci_devices are processed top-to-bottom without merging. + - hosts: node vars: - pci_passthrough_enabled: true pci_devices: + # Rename virtio-net-pci devices unless configured otherwise later (below). + - vendor: "1af4" + device: "*" + class: "0200" + # NOTE: 0[0] -> PCI Domain + # 0[1] -> PCI Bus + # 0[2] -> PCI Device + # 0[3] -> PCI Function + # NOTE: 1[0] -> PCI Domain (SR-IOV PF) + # 1[1] -> PCI Bus (SR-IOV PF) + # 1[2] -> PCI Device (SR-IOV PF) + # 1[3] -> PCI Function (SR-IOV PF) + # NOTE: 2 -> index (SR-IOV VF) + set_name: "pf{1[1]}{1[2]}vf{2}" + + # Do not rename "0000:04:00.0" + If it's SR-IOV capable, enable all available VFs. + - address: "0000:04:00.0" + set_numvfs: max + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.pci + + - hosts: node + vars: + pci_devices: + # Exclude primary NIC from processing. - address: "0000:02:00.0" excluded: true - - vendor: "1af4" + + # Enable 2 VFs and rename them to asd3v0, asd3v1 + Make sure OpenNebula doesn't use them (unlisted <- true). + - address: "0000:03:00.*" + set_name: "asd3vf{2}" + unlisted: true + - address: "0000:03:00.0" + set_numvfs: 2 + + # Enable 2 VFs and rename them to asd4v0, asd4v1 + Make sure OpenNebula does use them (unlisted <- false). + - address: "0000:04:00.*" + set_driver: vfio-pci # usual requirement for OpenNebula VMs + set_name: "asd4vf{2}" + - address: "0000:04:00.0" + set_numvfs: 2 + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.pci + + - hosts: node + vars: + pci_devices: + # Process primary NIC by disabling its protection (NOTE: in general, this may cause connectivity loss!). + - address: "0000:02:00.0" + unguarded: true + unlisted: true + set_name: asd0 + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.pci + + - hosts: node + vars: + pci_devices: + # Rename all existing Mellanox VFs. + - vendor: "15b3" device: "*" class: "0200" - set_driver: omit # NOTE: 'vfio-pci' is the default, 'omit' skips override altogether + set_name: "pf{1[1]}{1[2]}{1[3]}vf{2}" + + # Enable all available VFs for all existing Mellanox PFs + rename PFs. + - vendor: "15b3" + device: "1015" + class: "0200" set_numvfs: max + set_name: "pf{0[1]}{0[2]}{0[3]}" + + roles: + - role: opennebula.deploy.helper.facts + - role: opennebula.deploy.helper.pci + + - hosts: node + vars: + pci_devices: + # Rename and unlist all NICs matching MAC address wildcard. + - address: "52:54:00:12:3*:*" + set_name: "unlist{0[1]}{0{2}}{0[3]}" + unlisted: true roles: - role: opennebula.deploy.helper.facts - role: opennebula.deploy.helper.pci diff --git a/roles/helper/pci/defaults/main.yml b/roles/helper/pci/defaults/main.yml index 8e696a88..ff929a8c 100644 --- a/roles/helper/pci/defaults/main.yml +++ b/roles/helper/pci/defaults/main.yml @@ -1,3 +1,2 @@ --- -pci_passthrough_enabled: false pci_devices: [] diff --git a/roles/helper/pci/tasks/devices.yml b/roles/helper/pci/tasks/devices.yml index 25de57b6..fe133fbb 100644 --- a/roles/helper/pci/tasks/devices.yml +++ b/roles/helper/pci/tasks/devices.yml @@ -3,7 +3,7 @@ ansible.builtin.package: name: "{{ _common + _specific[ansible_os_family] }}" vars: - _common: [bash, coreutils, driverctl, grep, pciutils] + _common: [bash, coreutils, driverctl, findutils, grep, pciutils] _specific: Debian: [] RedHat: [] @@ -13,36 +13,40 @@ retries: 12 delay: 5 -- name: Parse and split pci_devices +- name: Parse pci_devices (user input) ansible.builtin.set_fact: - pci_devices_by_vendor_device_class: >- + pci_devices: >- {%- set output = [] -%} - {%- for v in pci_devices | selectattr('address', 'undefined') -%} - {{- - output.append(v | combine({ - "key": "{}:{}:{}".format( - v.vendor | d('*'), - v.device | d('*'), - v.class | d('*'), - ) | lower, - "key_regex": "^{}:{}:{}$".format( - (v.vendor | d('*') == '*') | ternary('[^:]*', v.vendor), - (v.device | d('*') == '*') | ternary('[^:]*', v.device), - (v.class | d('*') == '*') | ternary('[^:]*', v.class), - ) | lower, - })) - -}} - {%- endfor -%} - {{- output -}} - pci_devices_by_address: >- - {%- set output = [] -%} - {%- for v in pci_devices | selectattr('address', 'defined') -%} - {{- - output.append(v | combine({ - "key": v.address | lower, - "key_regex": "^{}$".format(v.address) | lower, - })) - -}} + {%- for v in pci_devices -%} + {%- if v.address is undefined -%} + {{- + output.append(v | combine({ + "key": "{}:{}:{}".format( + v.vendor | d('*'), + v.device | d('*'), + v.class | d('*'), + ) | lower, + "key_type": "vdc", + })) + -}} + {%- else -%} + {%- if v.address is opennebula.deploy.match_address('*:*:*.*', sep='[:.]') -%} + {{- + output.append(v | combine({ + "key": v.address | lower, + "key_type": "pci", + })) + -}} + {%- endif -%} + {%- if v.address is opennebula.deploy.match_address('*:*:*:*:*:*', sep='[:]') -%} + {{- + output.append(v | combine({ + "key": v.address | lower, + "key_type": "mac", + })) + -}} + {%- endif -%} + {%- endif -%} {%- endfor -%} {{- output -}} @@ -52,25 +56,6 @@ - when: lspci_devices | count > 0 block: - - name: Ensure udev rules for vfio - ansible.builtin.copy: - dest: /etc/udev/rules.d/99-vfio.rules - owner: 0 - group: 0 - mode: u=rw,go=r - content: | - SUBSYSTEM=="vfio", GROUP="kvm", MODE="0666" - register: copy_udev_vfio - - - name: Refresh udev rules - ansible.builtin.shell: - cmd: | - set -o errexit - udevadm control --reload-rules && udevadm trigger - executable: /bin/bash - changed_when: true - when: copy_udev_vfio is changed - - name: Render sriov-enable service unit ansible.builtin.copy: dest: "{{ item.dest }}" @@ -101,7 +86,7 @@ cmd: | [Unit] Description=Enable SR-IOV VFs on %I - After=network.target + After=network-pre.target [Service] Type=oneshot @@ -140,7 +125,7 @@ _to_revert: >- {%- set output = [] -%} {%- for v in lspci_devices -%} - {%- if (v.ECAP_SRIOV | bool is true) and (v.set_numvfs == 'max' or v.set_numvfs | int > 0) -%} + {%- if (v.Ecap_sriov == 'yes') and (v.Set_numvfs == 'max' or v.Set_numvfs | int > 0) -%} {{- output.append(v) -}} {%- endif -%} {%- endfor -%} @@ -162,10 +147,10 @@ set -x -o errexit -o pipefail CHANGED=false {% for v in _to_enable %} - {% if v.set_numvfs == 'max' %} + {% if v.Set_numvfs == 'max' %} SRIOV_NUMVFS="$(head -n1 '/sys/bus/pci/devices/{{ v.Slot }}/sriov_totalvfs')" - {% elif v.set_numvfs | int > 0 %} - SRIOV_NUMVFS='{{ v.set_numvfs }}' + {% elif v.Set_numvfs | int > 0 %} + SRIOV_NUMVFS='{{ v.Set_numvfs }}' {% else %} unset SRIOV_NUMVFS {% endif %} @@ -203,7 +188,7 @@ _to_enable: >- {%- set output = [] -%} {%- for v in lspci_devices -%} - {%- if (v.ECAP_SRIOV | bool is true) and (v.set_numvfs == 'max' or v.set_numvfs | int > 0) -%} + {%- if (v.Ecap_sriov == 'yes') and (v.Set_numvfs == 'max' or v.Set_numvfs | int > 0) -%} {{- output.append(v) -}} {%- endif -%} {%- endfor -%} @@ -225,14 +210,14 @@ set -x -o errexit -o pipefail BEFORE="$(driverctl list-overrides | sort)" ||: {% for v in _to_override %} - if ! grep -E -m1 '^{{ v.Slot }}\s+{{ v.set_driver }}$' <<< "$BEFORE"; then - {% if (v.ECAP_SRIOV | bool is true) %} + if ! grep -E -m1 '^{{ v.Slot }}\s+{{ v.Set_driver }}$' <<< "$BEFORE"; then + {% if v.Ecap_sriov == 'yes' %} TO_DISABLE="$(systemctl show --all -P Id 'sriov-enable@{{ v.Slot }}-*.service' | grep -E -v '^\s*$')" ||: if [[ -n "$TO_DISABLE" ]] && systemctl is-active --quiet $TO_DISABLE; then systemctl disable --now $TO_DISABLE # cleanup spurious sriov-enable@ services fi {% endif %} - driverctl set-override '{{ v.Slot }}' '{{ v.set_driver }}' # force the new driver (vfio-pci by default) + driverctl set-override '{{ v.Slot }}' '{{ v.Set_driver }}' # force the new driver (vfio-pci by default) fi {% endfor %} AFTER="$(driverctl list-overrides | sort)" ||: @@ -247,7 +232,7 @@ _to_override: >- {%- set output = [] -%} {%- for v in lspci_devices -%} - {%- if (v.set_driver != 'omit') and ((v.ECAP_SRIOV | bool is false) or (v.set_numvfs != 'max' and v.set_numvfs | int == 0)) -%} + {%- if (v.Set_driver != 'omit') and ((v.Ecap_sriov == 'no') or (v.Set_numvfs != 'max' and v.Set_numvfs | int == 0)) -%} {{- output.append(v) -}} {%- endif -%} {%- endfor -%} diff --git a/roles/helper/pci/tasks/main.yml b/roles/helper/pci/tasks/main.yml index bfdd646c..f28efabe 100644 --- a/roles/helper/pci/tasks/main.yml +++ b/roles/helper/pci/tasks/main.yml @@ -1,9 +1,10 @@ --- -- when: pci_passthrough_enabled | bool is true +- when: + - pci_devices is sequence + - pci_devices | count > 0 block: - - ansible.builtin.import_tasks: + - ansible.builtin.include_tasks: file: "{{ role_path }}/tasks/devices.yml" - when: - - pci_devices is defined - - pci_devices is sequence - - pci_devices | count > 0 + + - ansible.builtin.include_tasks: + file: "{{ role_path }}/tasks/udev.yml" diff --git a/roles/helper/pci/tasks/query.yml b/roles/helper/pci/tasks/query.yml index 334018fe..fd1ab9e2 100644 --- a/roles/helper/pci/tasks/query.yml +++ b/roles/helper/pci/tasks/query.yml @@ -3,7 +3,8 @@ ansible.builtin.shell: cmd: | set -o errexit -o pipefail - {% for v in pci_devices_by_vendor_device_class %} + {% for v in pci_devices %} + {% if v.key_type == 'vdc' %} STDOUT="$(lspci -vmm -nkD -d '{{ v.key }}')" if [[ -n "$STDOUT" ]]; then echo "$STDOUT" @@ -15,15 +16,19 @@ setpci -v -d '{{ v.key }}' STATUS | while IFS=' ' read -r SLOT _; do echo -e "Slot:\t$SLOT" if setpci -s "$SLOT" ECAP_SRIOV.B 1>/dev/null; then - echo -e "ECAP_SRIOV:\tyes" - echo + echo -e "Ecap_sriov:\tyes" else - echo -e "ECAP_SRIOV:\tno" - echo + echo -e "Ecap_sriov:\tno" fi + echo -e 'Set_driver:\t{{ v.set_driver | d('omit') }}' + echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' + echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' + echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Unguarded:\t{{ v.unguarded | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' + echo done - {% endfor %} - {% for v in pci_devices_by_address %} + {% elif v.key_type == 'pci' %} STDOUT="$(lspci -vmm -nkD -s '{{ v.key }}')" if [[ -n "$STDOUT" ]]; then echo "$STDOUT" @@ -32,14 +37,53 @@ echo "Could not find '{{ v.key }}'" >&2 exit 1 fi - echo -e "Slot:\t{{ v.key }}" - if setpci -s '{{ v.key }}' ECAP_SRIOV.B 1>/dev/null; then - echo -e "ECAP_SRIOV:\tyes" + setpci -v -s '{{ v.key }}' STATUS | while IFS=' ' read -r SLOT _; do + echo -e "Slot:\t$SLOT" + if setpci -s "$SLOT" ECAP_SRIOV.B 1>/dev/null; then + echo -e "Ecap_sriov:\tyes" + else + echo -e "Ecap_sriov:\tno" + fi + echo -e 'Set_driver:\t{{ v.set_driver | d('omit') }}' + echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' + echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' + echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Unguarded:\t{{ v.unguarded | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo - else - echo -e "ECAP_SRIOV:\tno" + done + {% elif v.key_type == 'mac' %} + udevadm trigger -nv -s net -a 'address={{ v.key }}' | while read -r DEVPATH; do + IFS='/' read -r _ SYS DEVICES ROOT _ SLOT _ <<< "$DEVPATH" + if [[ "$SYS" != sys || "$DEVICES" != devices || "$ROOT" != pci* ]]; then + continue + fi + STDOUT="$(lspci -vmm -nkD -s "$SLOT")" + if [[ -n "$STDOUT" ]]; then + echo "$STDOUT" + echo + else + echo "Could not find '$SLOT'" >&2 + exit 1 + fi + echo -e "Slot:\t$SLOT" + if setpci -s "$SLOT" ECAP_SRIOV.B 1>/dev/null; then + echo -e "Ecap_sriov:\tyes" + else + echo -e "Ecap_sriov:\tno" + fi + if [[ -e "$DEVPATH/address" ]]; then + echo -e "Address:\t$(cat $DEVPATH/address)" + fi + echo -e 'Set_driver:\t{{ v.set_driver | d('omit') }}' + echo -e 'Set_name:\t{{ v.set_name | d('omit') }}' + echo -e 'Set_numvfs:\t{{ v.set_numvfs | d(0) }}' + echo -e 'Unlisted:\t{{ v.unlisted | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Unguarded:\t{{ v.unguarded | d(false) | bool | ternary('yes', 'no') }}' + echo -e 'Excluded:\t{{ v.excluded | d(false) | bool | ternary('yes', 'no') }}' echo - fi + done + {% endif %} {% endfor %} executable: /bin/bash register: shell_lspci @@ -48,7 +92,7 @@ - name: Parse lspci's output ansible.builtin.set_fact: lspci_devices: >- - {{ _lspci_devices_filtered_and_extended }} + {{ _lspci_devices | rejectattr('Excluded', '==', 'yes') }} vars: _lspci_devices: >- {%- set output = {} -%} @@ -56,7 +100,6 @@ {{- output.update(output | combine({v.Slot: v}, recursive=true)) -}} {%- endfor -%} {{- output.values() | list -}} - _lspci_devices_raw: >- {{ shell_lspci.stdout | split(_sep1) | select('truthy') @@ -67,83 +110,6 @@ _sep2: "\n" _sep3: ":\t" # noqa no-tabs - _by_vendor_device_class_excluded: >- - {%- set output = [] -%} - {%- for v in pci_devices_by_vendor_device_class -%} - {%- if v.excluded | d(false) | bool is true -%} - {{- output.append(v) -}} - {%- endif -%} - {%- endfor -%} - {{- output -}} - - _by_address_excluded: >- - {%- set output = [] -%} - {%- for v in pci_devices_by_address -%} - {%- if v.excluded | d(false) | bool is true -%} - {{- output.append(v) -}} - {%- endif -%} - {%- endfor -%} - {{- output -}} - - _to_exclude: >- - {%- set output = [] -%} - {%- for v in _lspci_devices -%} - {%- set k = "{}:{}:{}".format(v.Vendor, v.Device, v.Class) -%} - {%- for x in _by_vendor_device_class_excluded -%} - {%- if k | regex_search(x.key_regex) -%} - {{- output.append(v.Slot) -}} - {%- endif -%} - {%- endfor -%} - {%- set k = v.Slot -%} - {%- for x in _by_address_excluded -%} - {%- if k | regex_search(x.key_regex) -%} - {{- output.append(v.Slot) -}} - {%- endif -%} - {%- endfor -%} - {%- endfor -%} - {{- output | unique -}} - - _lspci_devices_filtered: >- - {%- set output = [] -%} - {%- for v in _lspci_devices -%} - {%- if _to_exclude is not contains(v.Slot) -%} - {{- output.append(v) -}} - {%- endif -%} - {%- endfor -%} - {{- output -}} - - _lspci_devices_filtered_and_extended: >- - {%- set output = [] -%} - {%- for v in _lspci_devices_filtered -%} - {%- set to_append = [] %} - {%- for x in pci_devices_by_vendor_device_class -%} - {%- set k = "{}:{}:{}".format(v.Vendor, v.Device, v.Class) -%} - {%- if k | regex_search(x.key_regex) -%} - {{- - to_append.append(v | combine({ - "set_driver": x.set_driver | d('vfio-pci'), - "set_numvfs": x.set_numvfs | d(0), - })) - -}} - {%- endif -%} - {%- endfor -%} - {%- for x in pci_devices_by_address -%} - {%- set k = v.Slot -%} - {%- if k | regex_search(x.key_regex) -%} - {{- - to_append.append(v | combine({ - "set_driver": x.set_driver | d('vfio-pci'), - "set_numvfs": x.set_numvfs | d(0), - })) - -}} - {%- endif -%} - {%- endfor -%} - {%- if to_append | count > 0 -%} - {{- output.append(to_append | last) -}} - {%- endif -%} - {%- endfor -%} - {{- output -}} - - when: - pci_forbidden_addresses is undefined - _default is defined @@ -160,30 +126,23 @@ {{- output.append(_facts[v].slaves | d([_facts[v].device | d(none)] | select)) -}} {%- endfor -%} {{- output | flatten -}} - # Do also a quick crosscheck with OVS/DPDK config (when available). - _dpdk_pci_addrs: >- - {{ (ovs.iface | d({})).values() | map(attribute='set', default=[]) - | map('selectattr', 'options:dpdk-devargs', 'defined') - | select - | flatten - | map(attribute='options:dpdk-devargs') }} - _dpdk_interfaces: >- - {{ (ovs.iface | d({})).keys() | intersect(_facts.interfaces) }} block: - name: Query udev for device info ansible.builtin.command: cmd: "udevadm info --query=property --property=ID_PATH --value {{ _paths | join(' ') }}" vars: _paths: >- - {{ (_interfaces + _dpdk_interfaces) | unique - | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} + {{ _interfaces | map('regex_replace', '^(.*)$', "-p '/sys/class/net/\g<1>'") }} register: command_udevadm_info changed_when: false - name: Gather forbidden PCI addresses ansible.builtin.set_fact: - pci_forbidden_addresses: "{{ (_pci_addrs + _dpdk_pci_addrs) | unique }}" + pci_forbidden_addresses: "{{ _pci_addrs | difference(_pci_addrs_unguarded) }}" vars: + _pci_addrs_unguarded: >- + {{ lspci_devices | selectattr('Unguarded', '==', 'yes') + | map(attribute='Slot') }} _pci_addrs: >- {{ command_udevadm_info.stdout_lines | select | map('regex_replace', '^pci-', '') }} @@ -201,4 +160,5 @@ You might also want to look for conflicts with OVS/DPDK config. vars: _detected: >- - {{ lspci_devices | map(attribute='Slot') | intersect(pci_forbidden_addresses) }} + {{ lspci_devices | map(attribute='Slot') + | intersect(pci_forbidden_addresses | d([])) }} diff --git a/roles/helper/pci/tasks/udev.yml b/roles/helper/pci/tasks/udev.yml new file mode 100644 index 00000000..a8efd6cb --- /dev/null +++ b/roles/helper/pci/tasks/udev.yml @@ -0,0 +1,90 @@ +--- +- when: lspci_devices | count > 0 + vars: + _all: >- + {{ lspci_devices | selectattr('Set_name', 'defined') + | rejectattr('Set_name', '==', 'omit') + | selectattr('Class', 'defined') + | selectattr('Driver', 'defined') + | selectattr('Slot', 'defined') }} + + _net: >- + {{ _all | selectattr('Class', 'in', ['0200']) + | rejectattr('Driver', 'in', ['vfio-pci']) }} + + _pci: >- + {{ _all | rejectattr('Class', 'in', ['0200']) + | rejectattr('Driver', 'in', ['vfio-pci']) }} + + _vfio: >- + {{ _all | selectattr('Driver', 'in', ['vfio-pci']) + | selectattr('IOMMUGroup', 'defined') }} + + _vf_to_pf: >- + {{ shell_vf_to_pf_fn.stdout_lines | map('split', ';') + | items2dict(key_name=0, value_name=1) }} + + _vf_to_fn: >- + {{ shell_vf_to_pf_fn.stdout_lines | map('split', ';') + | items2dict(key_name=0, value_name=2) }} + block: + - name: Scan /sys/bus/pci/devices/*/virtfn* (SR-IOV) + ansible.builtin.shell: + cmd: | + set -o errexit -o pipefail + {% for v in lspci_devices | selectattr('Ecap_sriov', '==', 'yes') %} + find -P "/sys/bus/pci/devices/{{ v.Slot }}/" -maxdepth 1 -type l -name 'virtfn*' -printf '%l/%P\n' | while IFS='/' read -r _ VF FN; do + echo "$VF;{{ v.Slot }};${FN#virtfn}" + done + {% endfor %} + executable: /bin/bash + register: shell_vf_to_pf_fn + changed_when: false + + - name: Render extra udev rules (/etc/udev/rules.d) + ansible.builtin.copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + owner: 0 + group: 0 + mode: u=rw,go=r + loop_control: { label: "{{ item.dest }}" } + loop: + - dest: /etc/udev/rules.d/99-rename.rules + content: | + # managed by one-deploy + # --- NET + {% for v in _net %} + {% set name = v.Set_name.format(v.Slot | regex_findall('[0-9a-fA-F]+'), + _vf_to_pf.get(v.Slot, '') | regex_findall('[0-9a-fA-F]+'), + _vf_to_fn.get(v.Slot, '')) %} + # {{ v.Slot }} <- {{ name }} + SUBSYSTEM=="net", ACTION=="add", ENV{ID_PATH}=="pci-{{ v.Slot }}", NAME="{{ name }}" + {% endfor %} + # --- PCI + {% for v in _pci %} + {% set name = v.Set_name.format(v.Slot | regex_findall('[0-9a-fA-F]+'), + _vf_to_pf.get(v.Slot, '') | regex_findall('[0-9a-fA-F]+'), + _vf_to_fn.get(v.Slot, '')) %} + # {{ v.Slot }} <- {{ name }} + SUBSYSTEM=="pci", ACTION=="add", ENV{ID_PATH}=="pci-{{ v.Slot }}", SYMLINK+="pci/by-tag/{{ name }}" + {% endfor %} + # --- VFIO + {% for v in _vfio %} + {% set name = v.Set_name.format(v.Slot | regex_findall('[0-9a-fA-F]+'), + _vf_to_pf.get(v.Slot, '') | regex_findall('[0-9a-fA-F]+'), + _vf_to_fn.get(v.Slot, '')) %} + # {{ v.Slot }} <- {{ name }} + SUBSYSTEM=="vfio", ACTION=="add", KERNEL=="{{ v.IOMMUGroup }}", SYMLINK+="vfio/by-tag/{{ name }}" + {% endfor %} + - dest: /etc/udev/rules.d/99-mode.rules + content: | + # managed by one-deploy + SUBSYSTEM=="vfio", ACTION=="add", GROUP="kvm", MODE="0666" + register: copy_udev_rules + + - name: Refresh udev rules + ansible.builtin.shell: + cmd: udevadm control --reload-rules && udevadm trigger --action=add + changed_when: true + when: copy_udev_rules is changed diff --git a/roles/openvswitch/README.md b/roles/openvswitch/README.md index 97bdee20..d0600461 100644 --- a/roles/openvswitch/README.md +++ b/roles/openvswitch/README.md @@ -48,6 +48,13 @@ Example Playbook - codeready-builder-for-rhel-9-x86_64-rpms - rhel-9-for-x86_64-highavailability-rpms - fast-datapath-for-rhel-9-x86_64-rpms + pci_devices: + - address: '0000:04:00.0' + set_name: asd123 + unlisted: true + - address: '52:54:00:12:34:56' + set_name: asd321 + unlisted: true ovs: set: - other_config:dpdk-init: 'true' @@ -77,7 +84,9 @@ Example Playbook - mtu_request: 9000 - options:dpdk-devargs: '0000:03:00.0' driver: omit # skip forcing the driver - eth3: {} # non-DPDK device + asd123: {} # non-DPDK device (renamed) + asd321: {} # non-DPDK device (renamed) + eth4: {} # non-DPDK device bond: bond0: ifaces: [dpdk-p1, dpdk-p2] @@ -94,10 +103,11 @@ Example Playbook gw: "{{ ansible_default_ipv4.gateway }}" dns: ["{{ ansible_default_ipv4.gateway }}"] ovsbr1: # non-DPDK bridge - ports: [eth3] + ports: [asd123, asd321, eth4] roles: - role: opennebula.deploy.helper.facts - role: opennebula.deploy.helper.kernel + - role: opennebula.deploy.helper.pci - role: opennebula.deploy.repository - role: opennebula.deploy.openvswitch diff --git a/roles/openvswitch/tasks/main.yml b/roles/openvswitch/tasks/main.yml index 9c21bb7b..ce810fda 100644 --- a/roles/openvswitch/tasks/main.yml +++ b/roles/openvswitch/tasks/main.yml @@ -116,15 +116,18 @@ that: _dpdk_pci_addrs_raw == _dpdk_pci_addrs_items # no invalid, no duplicated fail_msg: Please remove invalid / duplicated DPDK PCI addresses from OVS config. - - name: Assert DPDK PCI devices are not already claimed by PCI/SR-IOV management + - name: Assert DPDK PCI device drivers are not already claimed by PCI/SR-IOV management ansible.builtin.assert: that: _intersection | count == 0 - fail_msg: DPDK vs PCI/SR-IOV conflict detected. Please ensure {{ _intersection }} are not being claimed by both implementations. + fail_msg: >- + DPDK vs PCI/SR-IOV conflict detected. + Please ensure drivers of {{ _intersection }} are not managed by helper/pci role already. vars: _intersection: >- {{ _dpdk_pci_addrs.keys() | intersect(_slots) }} _slots: >- {{ lspci_devices | selectattr('Slot', 'defined') + | rejectattr('Set_driver', 'in', ['omit']) | map(attribute='Slot') }} when: # NOTE: The 'lspci_devices' fact is produced by the helper/pci role. @@ -248,24 +251,27 @@ | flatten | selectattr('key', 'in', ['addrs', 'gw', 'dns']) }} - - name: Ensure udev rules for vfio - ansible.builtin.copy: - dest: /etc/udev/rules.d/99-vfio.rules - owner: 0 - group: 0 - mode: u=rw,go=r - content: | - SUBSYSTEM=="vfio", GROUP="kvm", MODE="0666" - register: copy_udev_vfio + - block: + - name: Render extra udev rules (/etc/udev/rules.d) + ansible.builtin.copy: + dest: "{{ item.dest }}" + content: "{{ item.content }}" + owner: 0 + group: 0 + mode: u=rw,go=r + loop_control: { label: "{{ item.dest }}" } + loop: + - dest: /etc/udev/rules.d/99-mode.rules + content: | + # managed by one-deploy + SUBSYSTEM=="vfio", ACTION=="add", GROUP="kvm", MODE="0666" + register: copy_udev_rules - - name: Refresh udev rules - ansible.builtin.shell: - cmd: | - set -o errexit - udevadm control --reload-rules && udevadm trigger - executable: /bin/bash - changed_when: true - when: copy_udev_vfio is changed + - name: Refresh udev rules + ansible.builtin.shell: + cmd: udevadm control --reload-rules && udevadm trigger --action=add --subsystem-match=vfio + changed_when: true + when: copy_udev_rules is changed - name: Install OVS packages ansible.builtin.package: diff --git a/roles/pci/frontend/README.md b/roles/pci/frontend/README.md index 83aca49f..106abb83 100644 --- a/roles/pci/frontend/README.md +++ b/roles/pci/frontend/README.md @@ -11,15 +11,14 @@ N/A Role Variables -------------- -| Name | Type | Default | Example | Description | -|----------------------------------|--------|---------|---------|------------------------------------------| -| `pci_passthrough_enabled` | `bool` | `false` | | Enable/Disable PCI passthrough. | -| `pci_passthrough_default_filter` | `bool` | `*:*` | | Default (global) PCI passthrough filter. | +| Name | Type | Default | Example | Description | +|----------------------|--------|---------|---------|------------------------------------------| +| `pci_default_filter` | `bool` | `*:*` | | Default (global) PCI passthrough filter. | Dependencies ------------ -- opennebula.deploy.opennebula.leader +N/A Example Playbook ---------------- diff --git a/roles/pci/frontend/defaults/main.yml b/roles/pci/frontend/defaults/main.yml index 21fdd75a..3aa11fe7 100644 --- a/roles/pci/frontend/defaults/main.yml +++ b/roles/pci/frontend/defaults/main.yml @@ -1,6 +1,2 @@ --- -pci_passthrough_enabled: >- - {{ (groups[node_group | d('node')] | map('extract', hostvars) - | map(attribute='pci_passthrough_enabled', default=false) - | map('bool')) is any }} -pci_passthrough_default_filter: "*:*" # NOTE: OpenNebula's default is "0:0" +pci_default_filter: "*:*" # NOTE: OpenNebula's default is "0:0" diff --git a/roles/pci/frontend/tasks/main.yml b/roles/pci/frontend/tasks/main.yml index d42f0e8f..635ce7d1 100644 --- a/roles/pci/frontend/tasks/main.yml +++ b/roles/pci/frontend/tasks/main.yml @@ -1,12 +1,21 @@ --- -- name: Apply global PCI filter - opennebula.deploy.cfgtool: - dest: /var/lib/one/remotes/etc/im/kvm-probes.d/pci.conf - parser: Yaml - actions: - - put: - path: [":filter"] - value: "{{ pci_passthrough_default_filter }}" - notify: - - Sync Remotes - when: pci_passthrough_enabled | bool is true +- when: _enabled is true + vars: + _enabled: >- + {{ _hosts | map('extract', hostvars) + | map(attribute='pci_devices', default=[]) + | select + | count > 0 }} + _hosts: >- + {{ groups[node_group | d('node')] }} + block: + - name: Apply global PCI filter + opennebula.deploy.cfgtool: + dest: /var/lib/one/remotes/etc/im/kvm-probes.d/pci.conf + parser: Yaml + actions: + - put: + path: [":filter"] + value: "{{ pci_default_filter }}" + notify: + - Sync Remotes diff --git a/roles/pci/node/README.md b/roles/pci/node/README.md index 48446dfb..21e3f6ee 100644 --- a/roles/pci/node/README.md +++ b/roles/pci/node/README.md @@ -11,10 +11,9 @@ N/A Role Variables -------------- -| Name | Type | Default | Example | Description | -|---------------------------|--------|---------|---------------|---------------------------------| -| `pci_passthrough_enabled` | `bool` | `false` | | Enable/Disable PCI passthrough. | -| `pci_devices` | `list` | `[]` | (check below) | PCI devices configuration. | +| Name | Type | Default | Example | Description | +|---------------|--------|---------|---------------|---------------------------------| +| `pci_devices` | `list` | `[]` | (check below) | PCI devices configuration. | Dependencies ------------ @@ -26,10 +25,12 @@ Example Playbook - hosts: node vars: - pci_passthrough_enabled: true pci_devices: - address: "0000:02:00.0" excluded: true + - address: "0000:03:00.*" + unlisted: true + set_name: "asd{0[3]}" - vendor: "1af4" device: "*" class: "0200" diff --git a/roles/pci/node/defaults/main.yml b/roles/pci/node/defaults/main.yml index 8e696a88..ff929a8c 100644 --- a/roles/pci/node/defaults/main.yml +++ b/roles/pci/node/defaults/main.yml @@ -1,3 +1,2 @@ --- -pci_passthrough_enabled: false pci_devices: [] diff --git a/roles/pci/node/tasks/filters.yml b/roles/pci/node/tasks/filters.yml index 48004541..4312f202 100644 --- a/roles/pci/node/tasks/filters.yml +++ b/roles/pci/node/tasks/filters.yml @@ -44,8 +44,8 @@ _after: PCI_SHORT_ADDRESS: >- {%- set output = [] -%} - {%- for v in lspci_devices -%} - {%- if (v.ECAP_SRIOV | bool is false) or (v.set_numvfs != 'max' and v.set_numvfs | int == 0) -%} + {%- for v in lspci_devices | rejectattr('Unlisted', '==', 'yes') -%} + {%- if (v.Ecap_sriov == 'no') or (v.Set_numvfs != 'max' and v.Set_numvfs | int == 0) -%} {{- output.append(v.Slot | regex_replace('^[^:]{4}:(.+)$', '\g<1>')) -}} {%- endif -%} {%- endfor -%} diff --git a/roles/pci/node/tasks/main.yml b/roles/pci/node/tasks/main.yml index 6a79eb6a..bdc26537 100644 --- a/roles/pci/node/tasks/main.yml +++ b/roles/pci/node/tasks/main.yml @@ -1,14 +1,20 @@ --- -- when: pci_passthrough_enabled | bool is true +- when: _enabled is true + vars: + _enabled: >- + {{ _hosts | map('extract', hostvars) + | map(attribute='pci_devices', default=[]) + | select + | count > 0 }} + _hosts: >- + {{ groups[node_group | d('node')] }} block: - ansible.builtin.include_role: name: opennebula.deploy.helper.pci - when: - - lspci_devices is undefined + when: lspci_devices is undefined - ansible.builtin.import_tasks: file: "{{ role_path }}/tasks/filters.yml" when: - lspci_devices is defined - - lspci_devices is sequence - lspci_devices | count > 0 diff --git a/roles/precheck/post_reboot/tasks/pci.yml b/roles/precheck/post_reboot/tasks/pci.yml index 24f51a32..f5a9732e 100644 --- a/roles/precheck/post_reboot/tasks/pci.yml +++ b/roles/precheck/post_reboot/tasks/pci.yml @@ -1,7 +1,21 @@ --- +- name: Ensure user renames / removes obsolete variables + ansible.builtin.assert: + that: + - pci_passthrough_enabled is undefined + - pci_passthrough_default_filter is undefined + fail_msg: | + Detected obsolete pci_passthrough_* variables. + Please remove 'pci_passthrough_enabled' as it no longer has any effect. + Please rename 'pci_passthrough_default_filter' to 'pci_default_filter'. + - name: Run PCI passthrough pre-flight checks - when: pci_passthrough_enabled | d(false) | bool + when: + - _pci_devices is sequence + - _pci_devices | count > 0 vars: + _pci_devices: >- + {{ pci_devices | d([]) }} _virtualization_flag: >- {{ 'vmx' if 'GenuineIntel' in ansible_facts.processor else 'svm' if 'AuthenticAMD' in ansible_facts.processor else '' }} diff --git a/roles/precheck/pre_reboot/tasks/features.yml b/roles/precheck/pre_reboot/tasks/features.yml index 8136e9f9..bbf201d3 100644 --- a/roles/precheck/pre_reboot/tasks/features.yml +++ b/roles/precheck/pre_reboot/tasks/features.yml @@ -94,8 +94,6 @@ - name: Check if PCI / SR-IOV management is supported for the current distro ansible.builtin.assert: that: - (pci_passthrough_enabled | d(false) | bool is false) - or (pci_devices | d([]) | count == 0) or (not (ansible_distribution == 'SLES' and ansible_distribution_major_version == '15'))