From d76e426454855369f74fbb77b2432bbc22dbd35f Mon Sep 17 00:00:00 2001 From: Joel Seguillon Date: Fri, 29 Apr 2022 20:12:48 +0200 Subject: [PATCH] Go stable (#39) All fields from VM spec are now supported. Default domain, network and volume can be supercharged in easy to understand way. Code clean + test cover supercharge mechanism And also doc --- .github/workflows/tox.yml | 52 +++- .pre-commit-config.yaml | 1 + README.rst | 143 +++++---- molecule/tests/converge.yml | 4 + molecule/tests/molecule.yml | 123 ++++++++ molecule/tests/verify.yml | 10 + molecule_kubevirt/driver.py | 41 +-- molecule_kubevirt/playbooks/create.yml | 284 ++++++++++-------- .../test/refs/instance-almost-default.yml | 39 +++ molecule_kubevirt/test/refs/instance-full.yml | 77 +++++ .../user_data-instance-almost-default.yml | 7 + .../test/refs/user_data-instance-full.yml | 26 ++ .../test/{test_func.py => test_init.py} | 10 - molecule_kubevirt/test/test_scenario_tests.py | 72 +++++ 14 files changed, 666 insertions(+), 223 deletions(-) create mode 100644 molecule/tests/converge.yml create mode 100644 molecule/tests/molecule.yml create mode 100644 molecule/tests/verify.yml create mode 100644 molecule_kubevirt/test/refs/instance-almost-default.yml create mode 100644 molecule_kubevirt/test/refs/instance-full.yml create mode 100644 molecule_kubevirt/test/refs/user_data-instance-almost-default.yml create mode 100644 molecule_kubevirt/test/refs/user_data-instance-full.yml rename molecule_kubevirt/test/{test_func.py => test_init.py} (82%) create mode 100644 molecule_kubevirt/test/test_scenario_tests.py diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index aa1cf72..702bf4d 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -26,27 +26,21 @@ jobs: - tox_env: lint # - tox_env: docs - tox_env: py36-ansible_2 - PREFIX: PYTEST_REQPASS=2 PYTHON_BASE_IMAGE: python:3.6 KUBERNETES_VERSION: v1.22.2 - tox_env: py36-ansible_2-devel - PREFIX: PYTEST_REQPASS=2 PYTHON_BASE_IMAGE: python:3.6 KUBERNETES_VERSION: v1.22.2 - tox_env: py37-ansible_3 - PREFIX: PYTEST_REQPASS=2 PYTHON_BASE_IMAGE: python:3.7 KUBERNETES_VERSION: v1.22.2 - tox_env: py38-ansible_4 - PREFIX: PYTEST_REQPASS=2 PYTHON_BASE_IMAGE: python:3.8 KUBERNETES_VERSION: v1.22.2 - tox_env: py39-ansible_2 - PREFIX: PYTEST_REQPASS=2 PYTHON_BASE_IMAGE: python:3.9 KUBERNETES_VERSION: v1.22.2 - tox_env: py39-ansible_2-devel - PREFIX: PYTEST_REQPASS=2 PYTHON_BASE_IMAGE: python:3.9 KUBERNETES_VERSION: v1.22.2 - tox_env: packaging @@ -149,7 +143,7 @@ jobs: if: ${{ contains(matrix.tox_env, 'py') }} - - name: Install virtcl + - name: Install KubeVirt virtcl uses: nick-invision/retry@v2 with: timeout_minutes: 5 @@ -161,6 +155,48 @@ jobs: sudo install virtctl /usr/local/bin if: ${{ contains(matrix.tox_env, 'py') }} + - name: Install KubeVirt's CDI + run: | + export VERSION=$(curl -s https://github.com/kubevirt/containerized-data-importer/releases/latest | grep -o "v[0-9]\.[0-9]*\.[0-9]*") + kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$VERSION/cdi-operator.yaml + kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$VERSION/cdi-cr.yaml + + - name: Install calico + run: | + kubectl apply -f https://projectcalico.docs.tigera.io/manifests/calico.yaml + echo "*******************************" + # FIXME activate wait for less flakyness + sleep 30 && kubectl -n kube-system set env daemonset/calico-node FELIX_IGNORELOOSERPF=true + + - name: Install Multus and configure one net-attach + run: | + curl -Ls https://raw.githubusercontent.com/k8snetworkplumbingwg/multus-cni/master/deployments/multus-daemonset.yml | kubectl apply -f - + + cat <- - {{ virtual_machine_instance | default({}) | combine ({name: instance}) }} + instance_conf_dict: + # if not running, ssh access will not be tested + running: "{{ item.running | default(true) }}" + instance: "{{ item.name }}" + address: "{{ ssh_service_address }}" + user: "{{ item.user_molecule.name | default(default_user_molecule.name) }}" + port: "22" + identity_file: "{{ molecule_ephemeral_directory }}/identity_{{ item.name }}" vars: - name: "{{ item.item.name }}" - instance: "{{ item.resources[0] | default({}) }}" - loop: "{{ virtual_machine_info.results }}" + vmi: "{{ virtual_machine_info.results | selectattr('item.name','==',item.name) | first }}" + # Also: should get a 'local node port' for nodePort kind usage for example + ssh_service_address: >- + {%- set svc_type = item.ssh_service.type | default(None) -%} + {%- if not svc_type -%} + {{ ((vmi['resources'] |first)['status']['interfaces'] | first)['ipAddress'] }} + {%- elif svc_type == 'NodePort' -%} + {{ item.ssh_service.nodePort_host | default('localhost') }}: + {{- ((node_port_services.results | selectattr('item.name','==',item.name) | first)['result']['spec']['ports'] | first)['nodePort'] }} + {%- elif svc_type == 'ClusterIP' -%} + {{ (cluster_ip_services.results | selectattr('item.name','==',item.name) | first)['result']['spec']['clusterIP'] }} + {%- endif -%} + register: instance_config_dict + loop: "{{ molecule_yml.platforms }}" loop_control: - label: "{{ item.item.name }}" - when: "item.ssh_service is not defined" - - - name: SSH check block - block: - - name: Populate instance config dict - set_fact: - instance_conf_dict: - instance: "{{ item.name }}" - address: "{{ ssh_service_address }}" - user: "{{ item.user_name | default('molecule') }}" - port: "22" - identity_file: "{{ molecule_ephemeral_directory }}/identity_{{ item.name }}" - vars: - vmi: "{{ virtual_machine_info.results | selectattr('item.name','==',item.name) | first }}" - ssh_service_address: >- - {%- set svc_type = item.ssh_service.type | default(None) -%} - {%- if not svc_type -%} - {{ ((vmi['resources'] |first)['status']['interfaces'] | first)['ipAddress'] }} - {%- elif svc_type == 'NodePort' -%} - {{ item.ssh_service.nodePort_host | default('localhost') }}: - {{- ((node_port_services.results | selectattr('item.name','==',item.name) | first)['result']['spec']['ports'] | first)['nodePort'] }} - {%- elif svc_type == 'ClusterIP' -%} - {{ (cluster_ip_services.results | selectattr('item.name','==',item.name) | first)['result']['spec']['clusterIP'] }} - {%- endif -%} - register: instance_config_dict - loop: "{{ molecule_yml.platforms }}" - loop_control: - label: "{{ item.name }}" - - - name: Convert instance config dict to a list - set_fact: - instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}" - - - name: Async start ssh access test - wait_for: - timeout: "{{ item.ssh_timeout | default(default_ssh_timeout) }}" - port: "{{ item.port | default('22') }}" - host: "{{ item.address }}" - delay: 1 - loop: "{{ instance_conf }}" - loop_control: - label: "{{ {item.instance: item.address} }}" - async: "{{ item.ssh_timeout | default(default_ssh_timeout) }}" - poll: 0 - register: ssh_reg + label: "{{ item.name }}" - - name: SSH test - async_status: - jid: "{{ async_result_item.ansible_job_id }}" - loop: "{{ ssh_reg.results }}" - loop_control: - loop_var: "async_result_item" - register: async_poll_results - until: async_poll_results.finished - delay: 1 - retries: "{{ item['ansible_facts']['instance_conf_dict']['ssh_timeout'] | default(default_ssh_timeout) }}" - rescue: - - name: Failed to get ssh - k8s_info: - kind: "{{ item }}" - api_version: kubevirt.io/v1alpha3 - namespace: "{{ item.namespace | default(default_namespace) }}" - with_items: - - Service - - VirtualMachine - - VirtualMachineInstance - register: vm_info + - name: Convert instance config dict to a list + set_fact: + instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}" - - name: Failed to get running VM - Dump Service, VM and VMI - debug: - var: dump_var - vars: - dump_var: "{{ item }}" - loop: "{{ vm_info.results }}" - failed_when: true + - name: Ssh access test - timeout={{ item.ssh_timeout | default(default_ssh_timeout) }} + wait_for: + timeout: "{{ item.ssh_timeout | default(default_ssh_timeout) }}" + port: "{{ item.port | default('22') }}" + host: "{{ item.address }}" + delay: "{{ item.ssh_delay | default(default_ssh_delay) }}" + loop: "{{ instance_conf | default([]) }}" + loop_control: + label: "{{ item.instance }} -> {{ item.address }} timeout={{ item.ssh_timeout | default(default_ssh_timeout) }}" + when: "item.running | default(true)" - name: Dump instance config copy: diff --git a/molecule_kubevirt/test/refs/instance-almost-default.yml b/molecule_kubevirt/test/refs/instance-almost-default.yml new file mode 100644 index 0000000..24c8e77 --- /dev/null +++ b/molecule_kubevirt/test/refs/instance-almost-default.yml @@ -0,0 +1,39 @@ +spec: + running: false + template: + metadata: + creationTimestamp: null + labels: + vm.cnv.io/name: instance-almost-default + spec: + domain: + devices: + disks: + - disk: + bus: virtio + name: boot + - disk: + bus: virtio + name: cloudinit + interfaces: + - bridge: {} + model: virtio + name: default + machine: + type: q35 + resources: + requests: + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 0 + volumes: + - containerDisk: + image: quay.io/kubevirt/fedora-cloud-container-disk-demo + imagePullPolicy: IfNotPresent + name: boot + - cloudInitNoCloud: + secretRef: + name: instance-almost-default + name: cloudinit diff --git a/molecule_kubevirt/test/refs/instance-full.yml b/molecule_kubevirt/test/refs/instance-full.yml new file mode 100644 index 0000000..80da40c --- /dev/null +++ b/molecule_kubevirt/test/refs/instance-full.yml @@ -0,0 +1,77 @@ +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: disk-dv-instance-full + spec: + preallocation: true + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + source: + http: + url: https://download.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.raw.xz + running: false + template: + metadata: + annotations: + cni.projectcalico.org/ipAddrs: '["10.244.25.25"]' + creationTimestamp: null + labels: + vm.cnv.io/name: instance-full + spec: + domain: + devices: + disks: + - disk: + bus: virtio + name: boot + - disk: + bus: virtio + name: emptydisk + - disk: + bus: virtio + name: cloudinit + interfaces: + - bridge: {} + model: e1000 + name: default + ports: + - port: 22 + - bridge: {} + model: virtio + name: multus + ports: + - port: 22 + machine: + type: q35 + resources: + limits: + cpu: "1" + memory: 3Gi + requests: + cpu: 200m + memory: 1Gi + hostname: myhost + networks: + - name: default + pod: {} + - multus: + networkName: macvlan-conf + name: multus + subdomain: my-domain + terminationGracePeriodSeconds: 30 + volumes: + - dataVolume: + name: disk-dv-instance-full + name: boot + - emptyDisk: + capacity: 2Gi + name: emptydisk + - cloudInitNoCloud: + secretRef: + name: instance-full + name: cloudinit diff --git a/molecule_kubevirt/test/refs/user_data-instance-almost-default.yml b/molecule_kubevirt/test/refs/user_data-instance-almost-default.yml new file mode 100644 index 0000000..196817b --- /dev/null +++ b/molecule_kubevirt/test/refs/user_data-instance-almost-default.yml @@ -0,0 +1,7 @@ +#cloud-config +users: +- name: molecule + ssh_authorized_keys: + - "" + sudo: + - ALL=(ALL) NOPASSWD:ALL diff --git a/molecule_kubevirt/test/refs/user_data-instance-full.yml b/molecule_kubevirt/test/refs/user_data-instance-full.yml new file mode 100644 index 0000000..f23f148 --- /dev/null +++ b/molecule_kubevirt/test/refs/user_data-instance-full.yml @@ -0,0 +1,26 @@ +#cloud-config +fs_setup: +- device: /dev/vdb + filesystem: ext4 + label: data_disk + overwrite: true +mounts: +- - /dev/vdb + - /var/lib/software + - auto + - defaults,nofail + - '0' + - '0' +users: +- gecos: dummy user + name: notmolecule + lock_passwd: false + plain_text_passwd: notmolecule + ssh_authorized_keys: + - "" + sudo: + - ALL=(ALL) NOPASSWD:/bin/mysql +- name: user2 + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDJRj9o4jhKW0Q6KnWa2jkThu/I070SJ+NBMDkP4ZXNu/t9Oq55Siz2dw6miwAjRVDfbB5HScM6XNJFWfPg10tY9ZUEizTirM5HeT8D+R5IvugfyqFeYs5d5V7X5O/TVJkNFUmqpA9TZYvoBUKsjnH4lH2/sPhtT13qUCLZNheUeQ== + sudo: false diff --git a/molecule_kubevirt/test/test_func.py b/molecule_kubevirt/test/test_init.py similarity index 82% rename from molecule_kubevirt/test/test_func.py rename to molecule_kubevirt/test/test_init.py index f6b3475..d8ce9e8 100644 --- a/molecule_kubevirt/test/test_func.py +++ b/molecule_kubevirt/test/test_init.py @@ -1,7 +1,6 @@ """Functional tests.""" import pathlib import shutil -import subprocess from molecule import logger from molecule.test.conftest import change_dir_to @@ -10,15 +9,6 @@ LOG = logger.get_logger(__name__) -def format_result(result: subprocess.CompletedProcess): - """Return friendly representation of completed process run.""" - return ( - f"RC: {result.returncode}\n" - + f"STDOUT: {result.stdout}\n" - + f"STDERR: {result.stderr}" - ) - - def test_command_init_and_test_scenario(tmp_path: pathlib.Path, DRIVER: str) -> None: """Verify that init scenario works.""" shutil.rmtree(tmp_path, ignore_errors=True) diff --git a/molecule_kubevirt/test/test_scenario_tests.py b/molecule_kubevirt/test/test_scenario_tests.py new file mode 100644 index 0000000..0a356cb --- /dev/null +++ b/molecule_kubevirt/test/test_scenario_tests.py @@ -0,0 +1,72 @@ +"""Functional tests.""" +from base64 import b64decode + +import pytest +import yaml +from molecule import logger +from molecule.util import run_command, safe_load_file + +LOG = logger.get_logger(__name__) + + +@pytest.mark.parametrize( + ("namespace", "name", "user"), + [ + ("kube-public", "instance-full", "notmolecule"), + ("default", "instance-almost-default", "molecule"), + ], +) +class TestClass: + """Test non running VMs and compare to references yaml files.""" + + @classmethod + def setup_class(cls): + cmd = ["molecule", "create", "-s", "tests"] + result = run_command(cmd) + assert result.returncode == 0 + + @classmethod + def teardown_class(cls): + cmd = ["molecule", "destroy", "-s", "tests"] + result = run_command(cmd) + assert result.returncode == 0 + + # Check spec result is same as tracked refs/ + def test_instance_spec(self, namespace, name, user): + cmd = ["kubectl", "get", "-n", namespace, "VirtualMachine", name, "-o", "yaml"] + result = run_command(cmd) + assert result.returncode == 0 + + result_yaml = yaml.safe_load(result.stdout) + spec_test_yaml = safe_load_file("molecule_kubevirt/test/refs/%s.yml" % (name)) + + assert result_yaml["spec"] == spec_test_yaml["spec"] + + # Check secret's user-data is same as tracked refs/ + def test_instance_secret(self, namespace, name, user): + cmd = [ + "kubectl", + "get", + "-n", + namespace, + "secret", + name, + "-o", + "jsonpath={.data.userdata}", + ] + result = run_command(cmd) + assert result.returncode == 0 + + cloud_config = b64decode(result.stdout) + cloud_config_yaml = yaml.safe_load(cloud_config) + + # ssh key is emptied before testing against tracked definition + for i in cloud_config_yaml["users"]: + if i["name"] == user: + i["ssh_authorized_keys"] = [""] + + spec_test_yaml = safe_load_file( + "molecule_kubevirt/test/refs/user_data-%s.yml" % (name) + ) + + assert spec_test_yaml == cloud_config_yaml