F #183: Add ability to rename net/pci/vfio devices

plugins/test/main.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+import fnmatch
+import re
+
+class TestModule(object):
+    def tests(self):
+        return {
+            'match_address': self.match_address,
+        }
+
+    # NOTE: It does not validate character classes or character count!
+    # EXAMPLES:
+    # pci -> match_address('0000:0*:00.*', sep='[:.]')
+    # mac -> match_address('52:54:*:*:*:0*', sep='[:]')
+    def match_address(self, address, pattern, sep=None):
+        """Tests if a split string matches a set of simple glob patterns."""
+
+        # In case no separator is provided simply match the whole string.
+        if sep is None:
+            return fnmatch.fnmatch(address, pattern)
+
+        # Make sure both address and pattern contain identical separators.
+        if re.findall(sep, address) != re.findall(sep, pattern):
+            return False
+
+        # Try matching (glob) each segment separately.
+        for x, y in zip(re.split(sep, address), re.split(sep, pattern)):
+            if not fnmatch.fnmatch(x, y):
+                return False
+
+        return True

plugins/test/match_address.yml

@@ -0,0 +1,43 @@
+---
+# Copyright: OpenNebula Project, OpenNebula Systems
+# Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+
+DOCUMENTATION:
+  name: match_address
+  short_description: Apply glob patterns to a generic address.
+  description:
+    - Tests if a split string matches a set of simple glob patterns.
+  options:
+    _input:
+      description:
+        - An address to match.
+      type: string
+      required: true
+    sep:
+      description:
+        - A regex expression used to split the input string (optional).
+      type: string
+      required: false
+  author:
+    - Michal Opala (@sk4zuzu)
+
+EXAMPLES: |
+  - name: Match MAC address
+    ansible.builtin.debug:
+      msg: >-
+        {{ '12:34:56:78:90:ab' is opennebula.deploy.match_address('12:34:56:78:90:*', sep='[:]') }}
+
+  - name: Select matching PCI addresses
+    ansible.builtin.debug:
+      msg: >-
+        {{ _addresses | select('opennebula.deploy.match_address', '0000:03:00.*', sep='[:.]') }}
+    vars:
+      _addresses:
+        - '12:34:56:78:90:ab' # MAC (skipped)
+        - '0000:03:00.4'      # PCI (matched)
+        - '0000:04:00.0'      # PCI (skipped)
+
+RETURN:
+  _value:
+    description: Returns True if the check passes.
+    type: boolean

roles/helper/pci/README.md

@@ -11,10 +11,19 @@ N/A
 Role Variables
 --------------
 
-| Name                      | Type   | Default | Example       | Description                     |
-|---------------------------|--------|---------|---------------|---------------------------------|
-| `pci_passthrough_enabled` | `bool` | `false` |               | Enable/Disable PCI passthrough. |
-| `pci_devices`             | `list` | `[]`    | (check below) | PCI devices configuration.      |
+| Name                        | Type  | Default   | Example       | Description                                                                         |
+|-----------------------------|-------|-----------|---------------|-------------------------------------------------------------------------------------|
+| `pci_devices`               | `list`| `[]`      | (check below) | PCI devices configuration.                                                          |
+| `pci_devices[*].excluded`   | `bool`| `false`   | (check below) | Do not process matching PCI devices.                                                |
+| `pci_devices[*].unguarded`  | `bool`| `false`   | (check below) | Do not protect matching PCI devices (this may cause primary NIC connectivity loss). |
+| `pci_devices[*].unlisted`   | `bool`| `false`   | (check below) | Do not pass matching PCI devices to OpenNebula.                                     |
+| `pci_devices[*].address`    | `str` | undefined | (check below) | Glob PCI devices by PCI or MAC address.                                             |
+| `pci_devices[*].vendor`     | `str` | `*`       | (check below) | Glob PCI devices by PCI Vendor (if address is undefined).                           |
+| `pci_devices[*].device`     | `str` | `*`       | (check below) | Glob PCI devices by PCI Device (if address is undefined).                           |
+| `pci_devices[*].class`      | `str` | `*`       | (check below) | Glob PCI devices by PCI Class (if address is undefined).                            |
+| `pci_devices[*].set_driver` | `str` | `omit`    | (check below) | Use driverctl to override driver (unless "omit").                                   |
+| `pci_devices[*].set_name`   | `str` | `omit`    | (check below) | Rename device in udev (unless "omit").                                              |
+| `pci_devices[*].set_numvfs` | `str` | `0`       | (check below) | Enable Virtual Functions for SR-IOV capable devices (integer >= 0 or "max").        |
 
 Dependencies
 ------------
@@ -24,21 +33,93 @@ N/A
 Example Playbook
 ----------------
 
+    # NOTE: Dicts defined in pci_devices are processed top-to-bottom without merging.
+
     - hosts: node
       vars:
-        pci_passthrough_enabled: true
         pci_devices:
-          - address: "0000:02:00.0"
-            excluded: true
+          # Rename virtio-net-pci devices unless configured otherwise later (below).
           - vendor: "1af4"
             device: "*"
             class: "0200"
-            set_driver: omit # NOTE: 'vfio-pci' is the default, 'omit' skips override altogether
+            # NOTE: 0[0] -> PCI Domain
+            #       0[1] -> PCI Bus
+            #       0[2] -> PCI Device
+            #       0[3] -> PCI Function
+            set_name: "asd{0[1]}v{0[3]}"
+
+          # Do not rename "0000:04:00.0" + If it's SR-IOV capable, enable all available VFs.
+          - address: "0000:04:00.0"
             set_numvfs: max
       roles:
         - role: opennebula.deploy.helper.facts
         - role: opennebula.deploy.helper.pci
 
+    - hosts: node
+      vars:
+        pci_devices:
+          # Exclude primary NIC from processing.
+          - address: "0000:02:00.0"
+            excluded: true
+
+          # Enable 2 VFs and rename them to asd3v1, asd3v2 + Make sure OpenNebula doesn't use them (unlisted <- true).
+          - address: "0000:03:00.*"
+            set_name: "asd3v{0[3]}"
+            unlisted: true
+          - address: "0000:03:00.0"
+            set_numvfs: 2
+
+          # Enable 2 VFs and rename them to asd4v1, asd4v2 + Make sure OpenNebula does use them (unlisted <- false).
+          - address: "0000:04:00.*"
+            set_driver: vfio-pci # usual requirement for OpenNebula VMs
+            set_name: "asd4v{0[3]}"
+          - address: "0000:04:00.0"
+            set_numvfs: 2
+      roles:
+        - role: opennebula.deploy.helper.facts
+        - role: opennebula.deploy.helper.pci
+
+    - hosts: node
+      vars:
+        pci_devices:
+          # Process primary NIC by disabling its protection (NOTE: in general, this may cause connectivity loss!).
+          - address: "0000:02:00.0"
+            unguarded: true
+            unlisted: true
+            set_name: asd0
+      roles:
+        - role: opennebula.deploy.helper.facts
+        - role: opennebula.deploy.helper.pci
+
+    - hosts: node
+      vars:
+        pci_devices:
+          # Enable all available VFs for all existing Mellanox PFs.
+          - vendor: "15b3"
+            device: "1015"
+            class: "0200"
+            set_numvfs: max
+
+          # Rename all existing Mellanox VFs.
+          - vendor: "15b3"
+            device: "1016"
+            class: "0200"
+            set_name: "vf{0[2]}{0[3]}"
+      roles:
+        - role: opennebula.deploy.helper.facts
+        - role: opennebula.deploy.helper.pci
+
+    - hosts: node
+      vars:
+        pci_devices:
+          # Rename and unlist all NICs matching MAC address wildcard.
+          - address: "52:54:00:12:3*:*"
+            set_name: "unlist{0[1]}v{0[3]}"
+            unlisted: true
+      roles:
+        - role: opennebula.deploy.helper.facts
+        - role: opennebula.deploy.helper.pci
+
 License
 -------
 

roles/helper/pci/defaults/main.yml

@@ -1,3 +1,2 @@
 ---
-pci_passthrough_enabled: false
 pci_devices: []

roles/helper/pci/tasks/devices.yml

@@ -13,36 +13,40 @@
   retries: 12
   delay: 5
 
-- name: Parse and split pci_devices
+- name: Parse pci_devices (user input)
   ansible.builtin.set_fact:
-    pci_devices_by_vendor_device_class: >-
+    pci_devices: >-
       {%- set output = [] -%}
-      {%- for v in pci_devices | selectattr('address', 'undefined') -%}
-        {{-
-          output.append(v | combine({
-            "key": "{}:{}:{}".format(
-              v.vendor | d('*'),
-              v.device | d('*'),
-              v.class | d('*'),
-            ) | lower,
-            "key_regex": "^{}:{}:{}$".format(
-              (v.vendor | d('*') == '*') | ternary('[^:]*', v.vendor),
-              (v.device | d('*') == '*') | ternary('[^:]*', v.device),
-              (v.class | d('*') == '*') | ternary('[^:]*', v.class),
-            ) | lower,
-          }))
-        -}}
-      {%- endfor -%}
-      {{- output -}}
-    pci_devices_by_address: >-
-      {%- set output = [] -%}
-      {%- for v in pci_devices | selectattr('address', 'defined') -%}
-        {{-
-          output.append(v | combine({
-            "key": v.address | lower,
-            "key_regex": "^{}$".format(v.address) | lower,
-          }))
-        -}}
+      {%- for v in pci_devices -%}
+        {%- if v.address is undefined -%}
+          {{-
+            output.append(v | combine({
+              "key": "{}:{}:{}".format(
+                v.vendor | d('*'),
+                v.device | d('*'),
+                v.class | d('*'),
+              ) | lower,
+              "key_type": "vdc",
+            }))
+          -}}
+        {%- else -%}
+          {%- if v.address is opennebula.deploy.match_address('*:*:*.*', sep='[:.]') -%}
+            {{-
+              output.append(v | combine({
+                "key": v.address | lower,
+                "key_type": "pci",
+              }))
+            -}}
+          {%- endif -%}
+          {%- if v.address is opennebula.deploy.match_address('*:*:*:*:*:*', sep='[:]') -%}
+            {{-
+              output.append(v | combine({
+                "key": v.address | lower,
+                "key_type": "mac",
+              }))
+            -}}
+          {%- endif -%}
+        {%- endif -%}
       {%- endfor -%}
       {{- output -}}
 
@@ -52,25 +56,6 @@
 
 - when: lspci_devices | count > 0
   block:
-    - name: Ensure udev rules for vfio
-      ansible.builtin.copy:
-        dest: /etc/udev/rules.d/99-vfio.rules
-        owner: 0
-        group: 0
-        mode: u=rw,go=r
-        content: |
-          SUBSYSTEM=="vfio", GROUP="kvm", MODE="0666"
-      register: copy_udev_vfio
-
-    - name: Refresh udev rules
-      ansible.builtin.shell:
-        cmd: |
-          set -o errexit
-          udevadm control --reload-rules && udevadm trigger
-        executable: /bin/bash
-      changed_when: true
-      when: copy_udev_vfio is changed
-
     - name: Render sriov-enable service unit
       ansible.builtin.copy:
         dest: "{{ item.dest }}"
@@ -101,7 +86,7 @@
           cmd: |
             [Unit]
             Description=Enable SR-IOV VFs on %I
-            After=network.target
+            After=network-pre.target
 
             [Service]
             Type=oneshot
@@ -140,7 +125,7 @@
         _to_revert: >-
           {%- set output = [] -%}
           {%- for v in lspci_devices -%}
-            {%- if (v.ECAP_SRIOV | bool is true) and (v.set_numvfs == 'max' or v.set_numvfs | int > 0) -%}
+            {%- if (v.Ecap_sriov == 'yes') and (v.Set_numvfs == 'max' or v.Set_numvfs | int > 0) -%}
               {{- output.append(v) -}}
             {%- endif -%}
           {%- endfor -%}
@@ -162,10 +147,10 @@
           set -x -o errexit -o pipefail
           CHANGED=false
           {% for v in _to_enable %}
-          {% if v.set_numvfs == 'max' %}
+          {% if v.Set_numvfs == 'max' %}
             SRIOV_NUMVFS="$(head -n1 '/sys/bus/pci/devices/{{ v.Slot }}/sriov_totalvfs')"
-          {% elif v.set_numvfs | int > 0 %}
-            SRIOV_NUMVFS='{{ v.set_numvfs }}'
+          {% elif v.Set_numvfs | int > 0 %}
+            SRIOV_NUMVFS='{{ v.Set_numvfs }}'
           {% else %}
             unset SRIOV_NUMVFS
           {% endif %}
@@ -203,7 +188,7 @@
         _to_enable: >-
           {%- set output = [] -%}
           {%- for v in lspci_devices -%}
-            {%- if (v.ECAP_SRIOV | bool is true) and (v.set_numvfs == 'max' or v.set_numvfs | int > 0) -%}
+            {%- if (v.Ecap_sriov == 'yes') and (v.Set_numvfs == 'max' or v.Set_numvfs | int > 0) -%}
               {{- output.append(v) -}}
             {%- endif -%}
           {%- endfor -%}
@@ -225,14 +210,14 @@
           set -x -o errexit -o pipefail
           BEFORE="$(driverctl list-overrides | sort)" ||:
           {% for v in _to_override %}
-          if ! grep -E -m1 '^{{ v.Slot }}\s+{{ v.set_driver }}$' <<< "$BEFORE"; then
-          {% if (v.ECAP_SRIOV | bool is true) %}
+          if ! grep -E -m1 '^{{ v.Slot }}\s+{{ v.Set_driver }}$' <<< "$BEFORE"; then
+          {% if v.Ecap_sriov == 'yes' %}
             TO_DISABLE="$(systemctl show --all -P Id 'sriov-enable@{{ v.Slot }}-*.service' | grep -E -v '^\s*$')" ||:
             if [[ -n "$TO_DISABLE" ]] && systemctl is-active --quiet $TO_DISABLE; then
               systemctl disable --now $TO_DISABLE # cleanup spurious sriov-enable@ services
             fi
           {% endif %}
-            driverctl set-override '{{ v.Slot }}' '{{ v.set_driver }}' # force the new driver (vfio-pci by default)
+            driverctl set-override '{{ v.Slot }}' '{{ v.Set_driver }}' # force the new driver (vfio-pci by default)
           fi
           {% endfor %}
           AFTER="$(driverctl list-overrides | sort)" ||:
@@ -247,7 +232,7 @@
         _to_override: >-
           {%- set output = [] -%}
           {%- for v in lspci_devices -%}
-            {%- if (v.set_driver != 'omit') and ((v.ECAP_SRIOV | bool is false) or (v.set_numvfs != 'max' and v.set_numvfs | int == 0)) -%}
+            {%- if (v.Set_driver != 'omit') and ((v.Ecap_sriov == 'no') or (v.Set_numvfs != 'max' and v.Set_numvfs | int == 0)) -%}
               {{- output.append(v) -}}
             {%- endif -%}
           {%- endfor -%}

roles/helper/pci/tasks/main.yml

@@ -1,9 +1,10 @@
 ---
-- when: pci_passthrough_enabled | bool is true
+- when:
+    - pci_devices is sequence
+    - pci_devices | count > 0
   block:
-    - ansible.builtin.import_tasks:
+    - ansible.builtin.include_tasks:
         file: "{{ role_path }}/tasks/devices.yml"
-      when:
-        - pci_devices is defined
-        - pci_devices is sequence
-        - pci_devices | count > 0
+
+    - ansible.builtin.include_tasks:
+        file: "{{ role_path }}/tasks/udev.yml"