tests: Add some tests about NodePort services

Author: TeddyAndrieux

tests/post/features/network.feature

@@ -5,3 +5,23 @@ Feature: Network
         And we run on an untainted single node
         Then ports check succeed
         And we have only expected processes listening
+
+    Scenario: Access using NodePort on workload-plane IP
+        Given the Kubernetes API is available
+        When we create a 'test-svc-1' NodePort service that expose a simple pod
+        Then a request on the 'test-svc-1' NodePort on a workload-plane IP returns 200
+
+    Scenario: Access using NodePort on control-plane IP
+        Given the Kubernetes API is available
+        And the node control-plane IP is not equal to its workload-plane IP
+        When we create a 'test-svc-2' NodePort service that expose a simple pod
+        Then a request on the 'test-svc-2' NodePort on a control-plane IP should not return
+
+    Scenario: Expose NodePort on Control Plane
+        Given the Kubernetes API is available
+        And the node control-plane IP is not equal to its workload-plane IP
+        When we create a 'test-svc-3' NodePort service that expose a simple pod
+        And we set nodeport CIDRs to control-plane CIDR
+        And we wait for the rollout of 'daemonset/kube-proxy' in namespace 'kube-system' to complete
+        Then a request on the 'test-svc-3' NodePort on a control-plane IP returns 200
+        And a request on the 'test-svc-3' NodePort on a workload-plane IP should not return

tests/post/steps/conftest.py

@@ -123,6 +123,17 @@ def check_multi_node(k8s_client):
         pytest.skip("We skip single node cluster for this test")
 
 
+@given("the node control-plane IP is not equal to its workload-plane IP")
+def node_control_plane_ip_is_not_equal_to_its_workload_plane_ip(host):
+    data = utils.get_grain(host, "metalk8s")
+
+    assert "control_plane_ip" in data
+    assert "workload_plane_ip" in data
+
+    if data["control_plane_ip"] == data["workload_plane_ip"]:
+        pytest.skip("Node control-plane IP is equal to node workload-plane IP")
+
+
 # }}}
 
 # Then {{{

tests/post/steps/test_ingress.py

@@ -1,4 +1,3 @@
-import json
 import os
 import re
 import requests
@@ -109,17 +108,6 @@ def teardown(context, host, ssh_config, version, k8s_client):
         re_configure_portmap(host, version, ssh_config)
 
 
-@given("the node control-plane IP is not equal to its workload-plane IP")
-def node_control_plane_ip_is_not_equal_to_its_workload_plane_ip(host):
-    data = utils.get_grain(host, "metalk8s")
-
-    assert "control_plane_ip" in data
-    assert "workload_plane_ip" in data
-
-    if data["control_plane_ip"] == data["workload_plane_ip"]:
-        pytest.skip("Node control-plane IP is equal to node workload-plane IP")
-
-
 @given("a VIP for Control Plane Ingress is available")
 def we_have_a_vip(context):
     cp_ingress_vip = os.environ.get("CONTROL_PLANE_INGRESS_VIP")
@@ -147,7 +135,7 @@ def disable_metallb(host, context, ssh_config, version):
             "networks": {"controlPlane": {"metalLB": {"enabled": False}, "ingress": {}}}
         }
 
-        patch_bootstrap_config(context, host, bootstrap_patch)
+        utils.patch_bootstrap_config(context, host, bootstrap_patch)
         re_configure_cp_ingress(host, version, ssh_config, context=context)
 
 
@@ -259,7 +247,7 @@ def update_cp_ingress_ip(host, context, ssh_config, version, node_name):
 
     bootstrap_patch = {"networks": {"controlPlane": {"ingress": {"ip": ip}}}}
 
-    patch_bootstrap_config(context, host, bootstrap_patch)
+    utils.patch_bootstrap_config(context, host, bootstrap_patch)
     re_configure_cp_ingress(host, version, ssh_config, context=context)
 
 
@@ -273,7 +261,7 @@ def update_control_plane_ingress_ip(host, context, ssh_config, version, ip):
         }
     }
 
-    patch_bootstrap_config(context, host, bootstrap_patch)
+    utils.patch_bootstrap_config(context, host, bootstrap_patch)
     re_configure_cp_ingress(host, version, ssh_config, context=context)
 
 
@@ -288,7 +276,7 @@ def update_portmap_cidr(host, context, ssh_config, version, plane):
 
     bootstrap_patch = {"networks": {"portmap": {"cidr": new_cidrs}}}
 
-    patch_bootstrap_config(context, host, bootstrap_patch)
+    utils.patch_bootstrap_config(context, host, bootstrap_patch)
     re_configure_portmap(host, version, ssh_config, context=context)
 
 
@@ -404,27 +392,6 @@ def get_node_hosting_cp_ingress_vip(k8s_client):
     return match.group("node")
 
 
-def patch_bootstrap_config(context, host, patch):
-    if "bootstrap_to_restore" not in context:
-        with host.sudo():
-            cmd_ret = host.check_output("salt-call --out json --local temp.dir")
-
-        tmp_dir = json.loads(cmd_ret)["local"]
-
-        with host.sudo():
-            host.check_output("cp /etc/metalk8s/bootstrap.yaml {}".format(tmp_dir))
-
-        context["bootstrap_to_restore"] = os.path.join(tmp_dir, "bootstrap.yaml")
-
-    with host.sudo():
-        host.check_output(
-            "salt-call --local --retcode-passthrough state.single "
-            "file.serialize /etc/metalk8s/bootstrap.yaml "
-            "dataset='{}' "
-            "merge_if_exists=True".format(json.dumps(patch))
-        )
-
-
 def re_configure_cp_ingress(host, version, ssh_config, context=None):
     with host.sudo():
         host.check_output(

tests/post/steps/test_network.py

@@ -1,16 +1,60 @@
 import json
+import requests
 
 import pytest
-from pytest_bdd import given, scenario, then
+from pytest_bdd import given, scenario, when, then, parsers
 
-from tests import utils
+from tests import utils, kube_utils
 
 
 @scenario("../features/network.feature", "All expected listening processes")
 def test_all_listening_processes(host):
     pass
 
 
+@scenario("../features/network.feature", "Access using NodePort on workload-plane IP")
+def test_access_nodeport_wp(host, teardown):
+    pass
+
+
+@scenario("../features/network.feature", "Access using NodePort on control-plane IP")
+def test_access_nodeport_cp(host, teardown):
+    pass
+
+
+@scenario("../features/network.feature", "Expose NodePort on Control Plane")
+def test_change_nodeport_cidrs(host, teardown):
+    pass
+
+
+@pytest.fixture(scope="function")
+def context():
+    return {}
+
+
+@pytest.fixture
+def teardown(context, host, ssh_config, version, k8s_client):
+    yield
+    for svc_name in context.get("svc_to_delete", []):
+        k8s_client.resources.get(api_version="v1", kind="Pod").delete(
+            name=f"{svc_name}-pod", namespace="default"
+        )
+        k8s_client.resources.get(api_version="v1", kind="Service").delete(
+            name=svc_name, namespace="default"
+        )
+
+    if "bootstrap_to_restore" in context:
+        with host.sudo():
+            host.check_output(
+                "cp {} /etc/metalk8s/bootstrap.yaml".format(
+                    context["bootstrap_to_restore"]
+                )
+            )
+
+    if context.get("reconfigure_nodeport"):
+        re_configure_nodeport(host, version, ssh_config)
+
+
 @given("we run on an untainted single node")
 def running_on_single_node_untainted(k8s_client):
     nodes = k8s_client.resources.get(api_version="v1", kind="Node").get()
@@ -21,6 +65,67 @@ def running_on_single_node_untainted(k8s_client):
     assert not nodes.items[0].spec.taints, "Single node should be untainted"
 
 
+@when(
+    parsers.parse("we create a '{svc_name}' NodePort service that expose a simple pod")
+)
+def create_nodeport_svc(context, k8s_client, utils_manifest, svc_name):
+    utils_manifest["metadata"]["name"] = f"{svc_name}-pod"
+    utils_manifest["metadata"]["labels"] = {"app": f"{svc_name}-app"}
+    utils_manifest["spec"]["containers"][0]["command"] = [
+        "python3",
+        "-m",
+        "http.server",
+        "8080",
+    ]
+
+    svc_manifest = {
+        "apiVersion": "v1",
+        "kind": "Service",
+        "metadata": {"name": svc_name},
+        "spec": {
+            "type": "NodePort",
+            "selector": {"app": f"{svc_name}-app"},
+            "ports": [{"port": 8080}],
+        },
+    }
+
+    context.setdefault("svc_to_delete", []).append(svc_name)
+
+    k8s_client.resources.get(api_version="v1", kind="Pod").create(
+        body=utils_manifest, namespace="default"
+    )
+    k8s_client.resources.get(api_version="v1", kind="Service").create(
+        body=svc_manifest, namespace="default"
+    )
+
+    utils.retry(
+        kube_utils.check_pod_status(
+            k8s_client,
+            f"{svc_name}-pod",
+            namespace="default",
+            state="Running",
+        ),
+        times=10,
+        wait=3,
+        name=f"wait for Pod 'default/{svc_name}-pod'",
+    )
+
+
+@when(parsers.parse("we set nodeport CIDRs to {plane} CIDR"))
+def update_nodeport_cidr(host, context, ssh_config, version, plane):
+    pillar = {
+        "workload-plane": "networks:workload_plane:cidr",
+        "control-plane": "networks:control_plane:cidr",
+    }
+
+    new_cidrs = utils.get_pillar(host, pillar[plane])
+
+    bootstrap_patch = {"networks": {"nodeport": {"cidr": new_cidrs}}}
+
+    utils.patch_bootstrap_config(context, host, bootstrap_patch)
+    re_configure_nodeport(host, version, ssh_config, context=context)
+
+
 @then("ports check succeed")
 def check_ports(host, ssh_config):
     with host.sudo():
@@ -121,3 +226,62 @@ def check_all_listening_process(host, version, control_plane_ingress_ip):
             )
 
     assert not errors, "\n".join(errors)
+
+
+@then(
+    parsers.parse(
+        "a request on the '{svc_name}' NodePort on a {plane} IP returns {status_code}"
+    )
+)
+def nodeport_service_request_return(k8s_client, host, svc_name, plane, status_code):
+    response = do_nodeport_service_request(k8s_client, host, plane, svc_name)
+    assert response is not None
+    assert response.status_code == int(status_code)
+
+
+@then(
+    parsers.parse(
+        "a request on the '{svc_name}' NodePort on a {plane} IP should not return"
+    )
+)
+def nodeport_service_request_does_not_respond(k8s_client, host, svc_name, plane):
+    try:
+        response = do_nodeport_service_request(k8s_client, host, plane, svc_name)
+        assert (
+            False
+        ), f"Server should not answer but got {response.status_code}: {response.reason}"
+    except:
+        pass
+
+
+def do_nodeport_service_request(k8s_client, host, plane, svc_name):
+    grains = {
+        "workload-plane": "metalk8s:workload_plane_ip",
+        "control-plane": "metalk8s:control_plane_ip",
+    }
+
+    if plane not in grains:
+        raise NotImplementedError
+
+    ip = utils.get_grain(host, grains[plane])
+
+    svc = k8s_client.resources.get(api_version="v1", kind="Service").get(
+        name=svc_name, namespace="default"
+    )
+    port = svc["spec"]["ports"][0]["nodePort"]
+
+    return requests.get(f"http://{ip}:{port}")
+
+
+def re_configure_nodeport(host, version, ssh_config, context=None):
+    command = [
+        "salt-run",
+        "state.sls",
+        "metalk8s.kubernetes.kube-proxy.deployed",
+        f"saltenv=metalk8s-{version}",
+    ]
+
+    utils.run_salt_command(host, command, ssh_config)
+
+    if context is not None:
+        context["reconfigure_nodeport"] = True

tests/utils.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import re
+import os
 import operator
 import testinfra
 import time
@@ -189,6 +190,27 @@ def get_pillar(host, key, local=False):
         return json.loads(output)["local"]
 
 
+def patch_bootstrap_config(context, host, patch):
+    if "bootstrap_to_restore" not in context:
+        with host.sudo():
+            cmd_ret = host.check_output("salt-call --out json --local temp.dir")
+
+        tmp_dir = json.loads(cmd_ret)["local"]
+
+        with host.sudo():
+            host.check_output("cp /etc/metalk8s/bootstrap.yaml {}".format(tmp_dir))
+
+        context["bootstrap_to_restore"] = os.path.join(tmp_dir, "bootstrap.yaml")
+
+    with host.sudo():
+        host.check_output(
+            "salt-call --local --retcode-passthrough state.single "
+            "file.serialize /etc/metalk8s/bootstrap.yaml "
+            "dataset='{}' "
+            "merge_if_exists=True".format(json.dumps(patch))
+        )
+
+
 class BaseAPIError(Exception):
     """Some error occurred when using a `BaseAPI` subclass."""