move tests for healtcheck to pytest

tests/test.yaml → tests/healthcheck.yaml

@@ -5,6 +5,7 @@ services:
     restart: unless-stopped
     environment:
       AUTOHEAL_INTERVAL: 1
+      AUTOHEAL_CONTAINER_LABEL: "AUTOHEAL_${COMPOSE_PROJECT_NAME}"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
 
@@ -13,7 +14,7 @@ services:
       dockerfile: Dockerfile
       context: ..
       labels:
-        autoheal: "true"
+        - "AUTOHEAL_${COMPOSE_PROJECT_NAME}=true"
     depends_on:
       - target
       - autoheal
@@ -40,7 +41,7 @@ services:
       dockerfile: Dockerfile
       context: ..
       labels:
-        autoheal: "true"
+        - "AUTOHEAL_${DOCKER_COMPOSE_PROJECT}=true"
     depends_on:
       - target
       - autoheal
@@ -68,7 +69,7 @@ services:
       dockerfile: Dockerfile
       context: ..
       labels:
-        autoheal: "true"
+        - "AUTOHEAL_${COMPOSE_PROJECT_NAME}=true"
     depends_on:
       - target_smtp
       - autoheal
@@ -108,42 +109,6 @@ services:
         aliases:
           - smtp.example.com
 
-  test_ping:
-    image: bash
-    depends_on:
-      - proxy_preresolve
-      - proxy_without_preresolve
-    # ping all proxies (to make sure it is supported)
-    command:
-      bash -c 'ping -c 1 target_preresolve.example.com && ping -c 1
-      target_without_preresolve.example.com'
-
-  test_wait:
-    image: bash
-    depends_on:
-      - proxy_preresolve
-    # wait 5 seconds (default dns timeout for proxies)
-    command: timeout 10 sleep 5
-
-  test_proxy_preresolve:
-    image: curlimages/curl
-    depends_on:
-      - proxy_preresolve
-    command: timeout 10 curl -v 'target_preresolve.example.com'
-
-  test_proxy_without_preresolve:
-    image: curlimages/curl
-    depends_on:
-      - proxy_without_preresolve
-    command: timeout 10 curl -v 'target_without_preresolve.example.com'
-
-  test_proxy_smtp:
-    image: curlimages/curl
-    depends_on:
-      - proxy_smtp
-    # -X QUIT because mailhog doesn't support HELP
-    command: timeout 10 curl -v 'smtp://target_smtp.example.com:1025' -X QUIT
-
 networks:
   # we do not allow communication to the outside
   simulated_outside:

tests/test_healtcheck.py

@@ -0,0 +1,169 @@
+import json
+import logging
+import random
+from time import sleep
+
+import plumbum.commands.processes
+import pytest
+from plumbum import local
+from plumbum.cmd import docker, docker_compose
+
+PROXY_TARGET_PAIRS = [
+    ("proxy_preresolve", "target"),
+    ("proxy_smtp", "target_smtp"),
+    ("proxy_without_preresolve", "target"),
+]
+
+logger = logging.getLogger()
+
+
+def _healthcheck(*args, **kwargs):
+    args = ("-f", "tests/healthcheck.yaml") + args
+    return docker_compose(*args, **kwargs)
+
+
+def _get_container_id(service_name):
+    return _healthcheck("ps", "-q", service_name).strip()
+
+
+def _get_container_id_and_ip(service_name):
+    container_id = _get_container_id(service_name)
+    container_info = json.loads(docker("inspect", container_id))
+    return (
+        container_id,
+        container_info[0]["NetworkSettings"]["Networks"][
+            "%s_simulated_outside" % local.env["COMPOSE_PROJECT_NAME"]
+        ]["IPAddress"],
+    )
+
+
+def _new_ip(target):
+    # we get the container id of the currently running target to be able to force changing ips by scaling up
+    # and then stopping the old container
+    old_container_id, old_ip = _get_container_id_and_ip(target)
+
+    # start a second instance of the target
+    _healthcheck("up", "-d", "--scale", "%s=2" % target, target)
+
+    # stop and remove the old container
+    docker("stop", old_container_id)
+    docker("rm", old_container_id)
+
+    # verify that we got a new ip (should not be able to reuse the old one)
+    new_container_id, new_ip = _get_container_id_and_ip(target)
+    assert old_container_id != new_container_id
+    assert old_ip != new_ip
+
+
+def _wait_for(proxy, message, callback, *args):
+    try:
+        while message not in callback(*args):
+            # try again in one second (to not hammer the CPU)
+            sleep(1)
+    except Exception:
+        # add additional infos to any error to make tracing down the error easier
+        logger.error("failed waiting for '%s'" % message)
+        logger.error(_healthcheck("logs", "autoheal"))
+        logger.error(_healthcheck("ps"))
+        logger.error(_healthcheck("exec", "-T", proxy, "healthcheck", retcode=None))
+        raise
+
+
+@pytest.fixture(scope="function")
+def custom_compose_project_name():
+    compose_project_name = "docker_whitelist_tests_%d" % random.randint(0, 65536)
+    yield compose_project_name
+
+
+@pytest.fixture(scope="function", autouse=True)
+def _cleanup_docker_compose(custom_compose_project_name):
+    with local.env(COMPOSE_PROJECT_NAME=custom_compose_project_name) as env:
+        yield env
+
+        # stop autoheal first to prevent it from restarting containers to be stopped
+        _healthcheck("stop", "autoheal")
+        _healthcheck("down", "-v")
+
+
+@pytest.mark.parametrize("proxy,target", PROXY_TARGET_PAIRS)
+def test_healthcheck_ok(proxy, target):
+    # given a started proxy with healthcheck
+    _healthcheck("up", "-d", proxy)
+
+    # when everything is ok and target is Up
+    assert "Up" in _healthcheck("ps", target)
+
+    # then healthcheck should be successful
+    _healthcheck("exec", "-T", proxy, "healthcheck")
+
+
+@pytest.mark.parametrize("proxy,target", PROXY_TARGET_PAIRS)
+def test_healthcheck_failing(proxy, target):
+    # given a started proxy with healthcheck
+    _healthcheck("up", "-d", proxy)
+
+    # when target is not reachable
+    _healthcheck("stop", target)
+    assert " Exit " in _healthcheck("ps", target)
+
+    # then healthcheck should return an error (non zero exit code)
+    with pytest.raises(
+        plumbum.commands.processes.ProcessExecutionError,
+        match=r"Unexpected exit code: (1|137)",
+    ):
+        _healthcheck("exec", "-T", proxy, "healthcheck")
+
+
+@pytest.mark.parametrize(
+    "proxy,target",
+    (p for p in PROXY_TARGET_PAIRS if p[0] != "proxy_without_preresolve"),
+)
+@pytest.mark.timeout(60)
+def test_healthcheck_autoheal(proxy, target):
+    # given a started proxy with healthcheck
+    _healthcheck("up", "-d", proxy)
+    proxy_container_id = _get_container_id(proxy)
+    # that was healthy
+    _wait_for(proxy, "Up (healthy)", _healthcheck, "ps", proxy)
+
+    # when target gets a new ip
+    _new_ip(target)
+
+    # then autoheal should restart the proxy
+    _wait_for(
+        proxy,
+        "(%s) found to be unhealthy - Restarting container now"
+        % proxy_container_id[:12],
+        _healthcheck,
+        "logs",
+        "autoheal",
+    )
+
+    # and the proxy should become healthy
+    _wait_for(proxy, "Up (healthy)", _healthcheck, "ps", proxy)
+
+    # and healthcheck should be successful
+    _healthcheck("exec", "-T", proxy, "healthcheck")
+
+
+def test_healthcheck_autoheal_proxy_without_preresolve():
+    # given a started proxy with healthcheck
+    proxy = "proxy_without_preresolve"
+    _healthcheck("up", "-d", proxy)
+    # that was healthy
+    _wait_for(proxy, "Up (healthy)", _healthcheck, "ps", proxy)
+
+    # when target gets a new ip
+    _new_ip("target")
+
+    # then healthcheck should be always successful (we wait just for 5 seconds/healthchecks)
+    for _ in range(0, 5):
+        _healthcheck("exec", "-T", proxy, "healthcheck")
+        sleep(1)
+
+    # and autoheal shouldn't have restarted anything
+    assert not [
+        line
+        for line in _healthcheck("logs", "autoheal").split("\n")
+        if line and not line.startswith("Attaching to ")
+    ]