diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 000000000..d837235ed Binary files /dev/null and b/.DS_Store differ diff --git a/docs/.DS_Store b/docs/.DS_Store new file mode 100644 index 000000000..562a3efb7 Binary files /dev/null and b/docs/.DS_Store differ diff --git a/docs/source/.DS_Store b/docs/source/.DS_Store new file mode 100644 index 000000000..ecc1aab6d Binary files /dev/null and b/docs/source/.DS_Store differ diff --git a/docs/source/playbooks/configure_network.rst b/docs/source/playbooks/configure_network.rst index 6f63672b1..13be08275 100644 --- a/docs/source/playbooks/configure_network.rst +++ b/docs/source/playbooks/configure_network.rst @@ -26,5 +26,14 @@ Uses value of the `edpm_network_config_tool` variable to determine which tool to The `'nmstate'` value will leave the process to the `systemroles.network` role, while the `'os-net-config'` will import custom tasks using os-net-config. +For SR-IOV on the nmstate path, set ``edpm_network_config_tool: nmstate``, +``edpm_network_config_tool_nmstate_override: true``, and use two explicit templates: +``edpm_network_config_nmstate_sriov_pf_template`` (phase 1, PF SR-IOV) and +``edpm_network_config_template`` (phase 2, VFs and other network settings). See +:doc:`../roles/role-edpm_network_config` (sections *nmstate tool* and +*SR-IOV with the nmstate tool*) and +``roles/edpm_network_config/examples/nmstate_sriov_phase1.yaml`` / +``roles/edpm_network_config/examples/nmstate_sriov_phase2.yaml``. + .. literalinclude:: ../../../playbooks/configure_network.yml :language: YAML diff --git a/docs/source/roles/role-edpm_network_config.rst b/docs/source/roles/role-edpm_network_config.rst index 81e2ebed3..6e502b9bc 100644 --- a/docs/source/roles/role-edpm_network_config.rst +++ b/docs/source/roles/role-edpm_network_config.rst @@ -60,3 +60,120 @@ An example of using ``remove_config`` is available in: .. literalinclude:: ../../../roles/edpm_network_config/molecule/default/converge.yml :language: YAML + +nmstate tool +~~~~~~~~~~~~ + +When ``edpm_network_config_tool`` is ``nmstate``, the role applies nmstate desired +state via the ``linux_system_roles.network`` role (see ``nmstate_tool.yml``). +NetworkManager is configured to manage ``/etc/resolv.conf`` on this path. + +The nmstate tool is experimental; set +``edpm_network_config_tool_nmstate_override: true`` to run it. +Set ``edpm_network_config_update: true`` (or rely on first-run / failed-run +logic) so the template is applied. + +**Single pass:** set ``edpm_network_config_template`` only (phase 2). + +**Two-step SR-IOV:** set ``edpm_network_config_nmstate_sriov_pf_template`` (phase 1: +PF + ``ethernet.sr-iov.total-vfs`` only — create VFs) and +``edpm_network_config_template`` (phase 2: per-VF ``vfs`` settings, bonds, IPs). +Phase 1 runs first; the role optionally waits until sysfs ``sriov_numvfs`` +matches ``total-vfs`` before phase 2. + +Example playbook (inline template): + +.. code-block:: YAML + + - name: Configure host network with nmstate + ansible.builtin.include_role: + name: osp.edpm.edpm_network_config + vars: + edpm_network_config_tool: nmstate + edpm_network_config_tool_nmstate_override: true + edpm_network_config_update: true + edpm_network_config_template: | + --- + interfaces: + - name: nic1 + type: ethernet + state: up + mtu: 1500 + +Two-step SR-IOV + bond (copy example files into playbook ``files/``): + +.. code-block:: YAML + + - name: Configure host network with nmstate (SR-IOV two-step) + ansible.builtin.include_role: + name: osp.edpm.edpm_network_config + vars: + edpm_network_config_tool: nmstate + edpm_network_config_tool_nmstate_override: true + edpm_network_config_update: true + edpm_network_config_nmstate_sriov_pf_template: >- + {{ lookup('file', playbook_dir + '/files/nmstate_sriov_phase1.yaml') }} + edpm_network_config_template: >- + {{ lookup('file', playbook_dir + '/files/nmstate_sriov_phase2.yaml') }} + +SR-IOV with the nmstate tool +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SR-IOV on the PF is configured in ``edpm_network_config_nmstate_sriov_pf_template`` +(phase 1) with ``ethernet.sr-iov.total-vfs`` only. Do **not** include the +per-VF ``vfs`` list in phase 1; VF creation and per-VF configuration cannot be +applied reliably in the same pass. + +Per-VF settings (``trust``, ``spoof-check``, MAC addresses, rates, VLAN, and so +on), bonds, bridges, and L3 addressing belong in ``edpm_network_config_template`` +(phase 2), applied after VFs exist. See the +`nmstate YAML API `_ SR-IOV section. + +Use ``sriov::`` port names or kernel VF netdev names in phase 2 +if resolution fails; see `Referring interface using SR-IOV PF name and VF ID +`_. + +When the ``vfs`` list is present in phase 2, nmstate expects configuration for +every VF up to ``total-vfs``. For OpenStack dataplane NICs, ``trust: true`` on +VFs is commonly required before Neutron SR-IOV agent use. + +Phase 1 example (PF, ``total-vfs`` only): + +.. literalinclude:: ../../../roles/edpm_network_config/examples/nmstate_sriov_phase1.yaml + :language: yaml + +Phase 2 example (per-VF ``vfs`` settings + Linux bond on two VFs): + +.. literalinclude:: ../../../roles/edpm_network_config/examples/nmstate_sriov_phase2.yaml + :language: yaml + +For SR-IOV-only hosts (no bond or phase-2 config), set +``edpm_network_config_nmstate_sriov_pf_template`` from phase 1 and leave +``edpm_network_config_template`` empty. + +Minimal SR-IOV (VF count only, default VF parameters): + +.. code-block:: yaml + + --- + interfaces: + - name: nic1 + type: ethernet + state: up + ethernet: + sr-iov: + drivers-autoprobe: true + total-vfs: 8 + +Optional wait tuning after phase 1: + +.. code-block:: yaml + + edpm_network_config_nmstate_sriov_vf_wait: true + edpm_network_config_nmstate_sriov_vf_wait_timeout: 60 + edpm_network_config_nmstate_sriov_vf_wait_delay: 2 + +Host SR-IOV in nmstate configures the PF and VFs on the node. Nova +``pci_passthrough:device_spec`` for SR-IOV instances is a separate step: use +the ``edpm_derive_pci_device_spec`` role and ``neutron_sriov`` playbook +(``physical_device_mappings`` on the agent) together with this network config. diff --git a/roles/edpm_network_config/defaults/main.yml b/roles/edpm_network_config/defaults/main.yml index 9cd4bc8ef..0e5ee8469 100644 --- a/roles/edpm_network_config/defaults/main.yml +++ b/roles/edpm_network_config/defaults/main.yml @@ -57,6 +57,13 @@ edpm_network_config_purge_provider: "" edpm_network_config_os_net_config_mappings: {} edpm_network_config_safe_defaults: false edpm_network_config_template: "" +# Phase 1 (optional): nmstate YAML to create SR-IOV VFs on PF(s) (total-vfs only). +# Per-VF vfs settings belong in edpm_network_config_template (phase 2). +edpm_network_config_nmstate_sriov_pf_template: "" +# After phase 1, poll sysfs until each PF's sriov_numvfs matches total-vfs from the SR-IOV template. +edpm_network_config_nmstate_sriov_vf_wait: true +edpm_network_config_nmstate_sriov_vf_wait_timeout: 60 +edpm_network_config_nmstate_sriov_vf_wait_delay: 2 edpm_bond_interface_ovs_options: "bond_mode=active-backup" edpm_dns_search_domains: [] edpm_network_config_nonconfigured_cleanup: false diff --git a/roles/edpm_network_config/examples/nmstate_sriov_phase1.yaml b/roles/edpm_network_config/examples/nmstate_sriov_phase1.yaml new file mode 100644 index 000000000..933d9ac91 --- /dev/null +++ b/roles/edpm_network_config/examples/nmstate_sriov_phase1.yaml @@ -0,0 +1,19 @@ +--- +# Phase 1: create SR-IOV VFs on the PF (total-vfs only). +# +# Per-VF settings (trust, spoof-check, MAC, etc.) belong in phase 2 — they cannot +# be applied reliably in the same pass as initial VF creation. +# +# Use with edpm_network_config_nmstate_sriov_pf_template (nmstate tool path). +# Phase 2: examples/nmstate_sriov_phase2.yaml via edpm_network_config_template. +# +# See https://nmstate.io/devel/yaml_api.html (Ethernet interface, SR-IOV). +interfaces: + - name: eno12399np0 + type: ethernet + state: up + mtu: 1500 + ethernet: + sr-iov: + drivers-autoprobe: true + total-vfs: 2 diff --git a/roles/edpm_network_config/examples/nmstate_sriov_phase2.yaml b/roles/edpm_network_config/examples/nmstate_sriov_phase2.yaml new file mode 100644 index 000000000..f1d22e937 --- /dev/null +++ b/roles/edpm_network_config/examples/nmstate_sriov_phase2.yaml @@ -0,0 +1,47 @@ +--- +# Phase 2: per-VF settings, bond, and L3 (after phase 1 created VFs). +# +# Use with edpm_network_config_template (nmstate tool path). +# VF ports may use sriov:: or kernel names (e.g. eno12399np0v0). +interfaces: + # Per-VF attributes on the PF (trust, spoof-check, rates, VLAN, etc.) + - name: eno12399np0 + type: ethernet + state: up + ethernet: + sr-iov: + total-vfs: 2 + vfs: + - id: 0 + spoof-check: false + trust: true + min-tx-rate: 0 + max-tx-rate: 0 + vlan-id: 0 + qos: 0 + - id: 1 + spoof-check: false + trust: true + min-tx-rate: 0 + max-tx-rate: 0 + vlan-id: 0 + qos: 0 + + - name: bond-sriov0 + type: bond + state: up + mtu: 1500 + mac-address: 52:54:00:6f:00:10 + link-aggregation: + mode: active-backup + options: + miimon: 100 + port: + - sriov:eno12399np0:0 + - sriov:eno12399np0:1 + ipv4: + enabled: true + dhcp: false + address: + - ip: 192.168.100.10 + prefix-length: 24 diff --git a/roles/edpm_network_config/meta/argument_specs.yml b/roles/edpm_network_config/meta/argument_specs.yml index 734f15d85..1479303b1 100644 --- a/roles/edpm_network_config/meta/argument_specs.yml +++ b/roles/edpm_network_config/meta/argument_specs.yml @@ -77,8 +77,30 @@ argument_specs: default: false edpm_network_config_template: type: str - description: "Which settings template should be rendered." + description: > + Nmstate desired-state YAML (phase 2). With edpm_network_config_tool=nmstate, applied after + edpm_network_config_nmstate_sriov_pf_template when that is set. For os-net-config, the os-net-config + network config template. + default: "" + edpm_network_config_nmstate_sriov_pf_template: + type: str + description: > + Nmstate desired-state YAML phase 1 (SR-IOV VF creation on PF). Set ethernet.sr-iov.total-vfs + only; do not include per-VF vfs settings here. Optional; when non-empty, applied before + edpm_network_config_template on the nmstate tool path. default: "" + edpm_network_config_nmstate_sriov_vf_wait: + type: bool + description: After phase 1, wait until sysfs sriov_numvfs matches total-vfs in the SR-IOV template. + default: true + edpm_network_config_nmstate_sriov_vf_wait_timeout: + type: int + description: Seconds to wait for SR-IOV VFs to appear after phase 1. + default: 60 + edpm_network_config_nmstate_sriov_vf_wait_delay: + type: int + description: Seconds between SR-IOV VF wait polls. + default: 2 edpm_bond_interface_ovs_options: type: str description: "Binding options to be rendered in a template" diff --git a/roles/edpm_network_config/tasks/nmstate_apply_state.yml b/roles/edpm_network_config/tasks/nmstate_apply_state.yml new file mode 100644 index 000000000..fa280e703 --- /dev/null +++ b/roles/edpm_network_config/tasks/nmstate_apply_state.yml @@ -0,0 +1,12 @@ +--- +# Apply one nmstate desired-state document via linux_system_roles.network. +# Caller sets edpm_nmstate_apply_template (YAML string) and edpm_nmstate_apply_phase (label for logs). + +- name: Render network_state variable [nmstate {{ edpm_nmstate_apply_phase }}] + ansible.builtin.set_fact: + network_state: "{{ edpm_nmstate_apply_template | from_yaml }}" + +- name: Load system-roles.network tasks [nmstate {{ edpm_nmstate_apply_phase }}] + ansible.builtin.include_role: + name: "{{ lookup('ansible.builtin.env', 'EDPM_SYSTEMROLES', default='fedora.linux_system_roles') + '.network' }}" + register: edpm_nmstate_apply_result diff --git a/roles/edpm_network_config/tasks/nmstate_tool.yml b/roles/edpm_network_config/tasks/nmstate_tool.yml index 4b148a128..58392c534 100644 --- a/roles/edpm_network_config/tasks/nmstate_tool.yml +++ b/roles/edpm_network_config/tasks/nmstate_tool.yml @@ -53,14 +53,14 @@ ((nmstate_returncode_slurp.content | b64decode | int) != 0)) or (not nmstate_returncode_stat.stat.exists) block: - - name: Render network_state variable - ansible.builtin.set_fact: - network_state: "{{ edpm_network_config_template | from_yaml }}" - - - name: Load system-roles.network tasks [nmstate] - ansible.builtin.include_role: - name: "{{ lookup('ansible.builtin.env', 'EDPM_SYSTEMROLES', default='fedora.linux_system_roles') + '.network' }}" - register: nmstate_result + - name: Fail when no nmstate template is defined for apply + ansible.builtin.fail: + msg: >- + Set edpm_network_config_template and/or + edpm_network_config_nmstate_sriov_pf_template (non-empty YAML strings). + when: + - edpm_network_config_nmstate_sriov_pf_template | trim | length == 0 + - edpm_network_config_template | trim | length == 0 - name: Ensure /var/lib/edpm-config exists ansible.builtin.file: @@ -68,8 +68,85 @@ state: directory mode: "0755" + - name: Apply nmstate phase 1 (SR-IOV on PF) + when: edpm_network_config_nmstate_sriov_pf_template | trim | length > 0 + block: + - name: Apply nmstate SR-IOV phase + ansible.builtin.include_tasks: nmstate_apply_state.yml + vars: + edpm_nmstate_apply_template: "{{ edpm_network_config_nmstate_sriov_pf_template }}" + edpm_nmstate_apply_phase: sriov + + - name: Record SR-IOV phase result + ansible.builtin.set_fact: + nmstate_sriov_result: "{{ edpm_nmstate_apply_result }}" + + - name: Fail when nmstate SR-IOV phase failed + ansible.builtin.fail: + msg: "nmstate SR-IOV phase (edpm_network_config_nmstate_sriov_pf_template) failed." + when: nmstate_sriov_result.failed | default(false) + + - name: Wait for SR-IOV VFs after phase 1 + when: edpm_network_config_nmstate_sriov_vf_wait | bool + block: + - name: Parse phase-1 state for VF wait + ansible.builtin.set_fact: + edpm_nmstate_sriov_phase_state: >- + {{ edpm_network_config_nmstate_sriov_pf_template | from_yaml }} + + - name: Wait until sysfs reports expected SR-IOV VF counts + ansible.builtin.include_tasks: nmstate_wait_sriov_vfs.yml + + - name: Apply nmstate phase 2 (main network template) + when: edpm_network_config_template | trim | length > 0 + block: + - name: Apply nmstate main phase + ansible.builtin.include_tasks: nmstate_apply_state.yml + vars: + edpm_nmstate_apply_template: "{{ edpm_network_config_template }}" + edpm_nmstate_apply_phase: main + + - name: Record main nmstate phase result + ansible.builtin.set_fact: + nmstate_result: "{{ edpm_nmstate_apply_result }}" + + - name: Fail when nmstate main phase failed + ansible.builtin.fail: + msg: "nmstate main phase (edpm_network_config_template) failed." + when: nmstate_result.failed | default(false) + + - name: Set nmstate_result when only SR-IOV phase ran + ansible.builtin.set_fact: + nmstate_result: "{{ nmstate_sriov_result }}" + when: + - nmstate_sriov_result is defined + - nmstate_result is not defined + - name: Write rc of nmstate run ansible.builtin.copy: - content: "0" + content: >- + {{ + '1' if ( + (nmstate_sriov_result.failed | default(false)) + or (nmstate_result.failed | default(false)) + ) else '0' + }} dest: /var/lib/edpm-config/nmstate.returncode mode: "0644" + + rescue: + - name: Ensure /var/lib/edpm-config exists after nmstate failure + ansible.builtin.file: + path: /var/lib/edpm-config + state: directory + mode: "0755" + + - name: Write failure rc of nmstate run + ansible.builtin.copy: + content: "1" + dest: /var/lib/edpm-config/nmstate.returncode + mode: "0644" + + - name: Re-raise nmstate apply failure + ansible.builtin.fail: + msg: "nmstate network configuration failed; see tasks above." diff --git a/roles/edpm_network_config/tasks/nmstate_wait_sriov_vfs.yml b/roles/edpm_network_config/tasks/nmstate_wait_sriov_vfs.yml new file mode 100644 index 000000000..cb7f04556 --- /dev/null +++ b/roles/edpm_network_config/tasks/nmstate_wait_sriov_vfs.yml @@ -0,0 +1,43 @@ +--- +# Wait until sysfs reports the desired VF count for each PF in phase-1 state. +# Caller sets edpm_nmstate_sriov_phase_state (parsed dict from phase-1 template). + +- name: Reset SR-IOV PF wait list + ansible.builtin.set_fact: + edpm_nmstate_sriov_pf_wait_list: [] + +- name: Build SR-IOV PF wait list from phase-1 nmstate state + ansible.builtin.set_fact: + edpm_nmstate_sriov_pf_wait_list: >- + {{ + edpm_nmstate_sriov_pf_wait_list | default([]) + [{ + 'name': item.name, + 'total_vfs': item.ethernet['sr-iov']['total-vfs'] | int + }] + }} + loop: "{{ edpm_nmstate_sriov_phase_state.interfaces | default([]) }}" + when: + - item.name is defined + - item.ethernet is defined + - item.ethernet['sr-iov'] is defined + - item.ethernet['sr-iov']['total-vfs'] is defined + +- name: Wait for SR-IOV VFs to appear on each PF + become: true + ansible.builtin.command: + argv: + - cat + - "/sys/class/net/{{ item.name }}/device/sriov_numvfs" + register: edpm_nmstate_sriov_numvfs_check + until: edpm_nmstate_sriov_numvfs_check.stdout | trim | int == item.total_vfs + retries: >- + {{ + ( + edpm_network_config_nmstate_sriov_vf_wait_timeout | int + / edpm_network_config_nmstate_sriov_vf_wait_delay | int + ) | int + }} + delay: "{{ edpm_network_config_nmstate_sriov_vf_wait_delay }}" + changed_when: false + loop: "{{ edpm_nmstate_sriov_pf_wait_list | default([]) }}" + when: edpm_nmstate_sriov_pf_wait_list | default([]) | length > 0