From 2f3bc2fd1d2d210972ee55240463229c70272217 Mon Sep 17 00:00:00 2001 From: Andreas Perhab Date: Thu, 8 Jul 2021 15:52:11 +0200 Subject: [PATCH] move tests for healtcheck to pytest --- tests/{test.yaml => healthcheck.yaml} | 43 +------ tests/run_tests.sh | 76 ------------ tests/test_healtcheck.py | 172 ++++++++++++++++++++++++++ tests/test_healthcheck.sh | 9 -- 4 files changed, 176 insertions(+), 124 deletions(-) rename tests/{test.yaml => healthcheck.yaml} (70%) delete mode 100755 tests/run_tests.sh create mode 100644 tests/test_healtcheck.py delete mode 100755 tests/test_healthcheck.sh diff --git a/tests/test.yaml b/tests/healthcheck.yaml similarity index 70% rename from tests/test.yaml rename to tests/healthcheck.yaml index 791eda9..6f38e7c 100644 --- a/tests/test.yaml +++ b/tests/healthcheck.yaml @@ -5,6 +5,7 @@ services: restart: unless-stopped environment: AUTOHEAL_INTERVAL: 1 + AUTOHEAL_CONTAINER_LABEL: "AUTOHEAL_${COMPOSE_PROJECT_NAME}" volumes: - /var/run/docker.sock:/var/run/docker.sock @@ -13,7 +14,7 @@ services: dockerfile: Dockerfile context: .. labels: - autoheal: "true" + - "AUTOHEAL_${COMPOSE_PROJECT_NAME}=true" depends_on: - target - autoheal @@ -40,7 +41,7 @@ services: dockerfile: Dockerfile context: .. labels: - autoheal: "true" + - "AUTOHEAL_${COMPOSE_PROJECT_NAME}=true" depends_on: - target - autoheal @@ -68,7 +69,7 @@ services: dockerfile: Dockerfile context: .. labels: - autoheal: "true" + - "AUTOHEAL_${COMPOSE_PROJECT_NAME}=true" depends_on: - target_smtp - autoheal @@ -108,42 +109,6 @@ services: aliases: - smtp.example.com - test_ping: - image: bash - depends_on: - - proxy_preresolve - - proxy_without_preresolve - # ping all proxies (to make sure it is supported) - command: - bash -c 'ping -c 1 target_preresolve.example.com && ping -c 1 - target_without_preresolve.example.com' - - test_wait: - image: bash - depends_on: - - proxy_preresolve - # wait 5 seconds (default dns timeout for proxies) - command: timeout 10 sleep 5 - - test_proxy_preresolve: - image: curlimages/curl - depends_on: - - proxy_preresolve - command: timeout 10 curl -v 'target_preresolve.example.com' - - test_proxy_without_preresolve: - image: curlimages/curl - depends_on: - - proxy_without_preresolve - command: timeout 10 curl -v 'target_without_preresolve.example.com' - - test_proxy_smtp: - image: curlimages/curl - depends_on: - - proxy_smtp - # -X QUIT because mailhog doesn't support HELP - command: timeout 10 curl -v 'smtp://target_smtp.example.com:1025' -X QUIT - networks: # we do not allow communication to the outside simulated_outside: diff --git a/tests/run_tests.sh b/tests/run_tests.sh deleted file mode 100755 index 37df699..0000000 --- a/tests/run_tests.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -set -e -for arg in "$@" ; do - echo arg $arg - if [[ "$arg" == "DEBUG" ]] ; then - DEBUG=1 - else - TEST_FILTER="$arg" - fi -done -DEBUG=${DEBUG:-0} -TEST_FILTER=${TEST_FILTER:-} - -function cleanup() { - if [[ $DEBUG == 1 ]]; then - docker-compose -f tests/test.yaml ps - docker-compose -f tests/test.yaml exec -T proxy_preresolve /usr/local/bin/healthcheck || true - docker-compose -f tests/test.yaml exec -T proxy_without_preresolve /usr/local/bin/healthcheck || true - docker-compose -f tests/test.yaml top - docker-compose -f tests/test.yaml logs - fi - docker-compose -f tests/test.yaml down -v --remove-orphans -} -trap cleanup EXIT - -function with_prefix() { - local prefix - prefix="$1" - shift - "$@" 2>&1 | while read -r line; do - echo "$prefix" "$line" - done - return "${PIPESTATUS[0]}" -} - -function run_tests() { - for service in $(docker-compose -f tests/test.yaml config --services); do - if [[ ( $service == test_* || ( $DEBUG = 1 && $service == debug_* ) ) && $service == *"$TEST_FILTER"* ]] ; then - echo "running $service" - with_prefix "$service:" docker-compose -f tests/test.yaml run --rm "$service" - fi - done -} - -function change_target_ips() { - for target in "target" "target_smtp"; do - #spin up a second target and remove the first target container to give it a new ip (simulates a new deployment of an external cloud service) - local target_container_id - target_container_id=$(docker-compose -f tests/test.yaml ps -q "$target") - if [[ "$target_container_id" != "" ]] ; then - if [[ $DEBUG == 1 ]] ; then - docker inspect "$target_container_id" | grep '"IPAddress": "[^"]\+' - fi - docker-compose -f tests/test.yaml up -d --scale "$target=2" "$target" - docker stop "$target_container_id" | xargs echo "stopped ${target}_1" - docker rm "$target_container_id" | xargs echo "removed ${target}_1" - if [[ $DEBUG == 1 ]] ; then - target_container_id=$(docker-compose -f tests/test.yaml ps -q "$target") - docker inspect "$target_container_id" | grep '"IPAddress": "[^"]\+' - fi - fi - done - # give docker some time to restart unhealthy containers - sleep 5 -} - -with_prefix "build:" docker-compose -f tests/test.yaml build - -# make sure all tests pass when target is up -run_tests - -# when target changes ip -with_prefix "changing target_ip:" change_target_ips - -# all tests still should pass -run_tests diff --git a/tests/test_healtcheck.py b/tests/test_healtcheck.py new file mode 100644 index 0000000..62c1086 --- /dev/null +++ b/tests/test_healtcheck.py @@ -0,0 +1,172 @@ +import json +import logging +import os.path +import hashlib +from time import sleep + +import plumbum.commands.processes +import pytest +from plumbum import local +from plumbum.cmd import docker, docker_compose + +HEALTHCHECK_YAML = os.path.abspath("tests/healthcheck.yaml") + +PROXY_TARGET_PAIRS = [ + ("proxy_preresolve", "target"), + ("proxy_smtp", "target_smtp"), + ("proxy_without_preresolve", "target"), +] + +logger = logging.getLogger() + + +def _healthcheck(*args, **kwargs): + args = ("-f", HEALTHCHECK_YAML) + args + return docker_compose(*args, **kwargs) + + +def _get_container_id(service_name): + return _healthcheck("ps", "-q", service_name).strip() + + +def _get_container_id_and_ip(service_name): + container_id = _get_container_id(service_name) + container_info = json.loads(docker("inspect", container_id)) + return ( + container_id, + container_info[0]["NetworkSettings"]["Networks"][ + "%s_simulated_outside" % local.env["COMPOSE_PROJECT_NAME"] + ]["IPAddress"], + ) + + +def _new_ip(target): + # we get the container id of the currently running target to be able to force changing ips by scaling up + # and then stopping the old container + old_container_id, old_ip = _get_container_id_and_ip(target) + + # start a second instance of the target + _healthcheck("up", "-d", "--scale", "%s=2" % target, target) + + # stop and remove the old container + docker("stop", old_container_id) + docker("rm", old_container_id) + + # verify that we got a new ip (should not be able to reuse the old one) + new_container_id, new_ip = _get_container_id_and_ip(target) + assert old_container_id != new_container_id + assert old_ip != new_ip + + +def _wait_for(proxy, message, callback, *args): + try: + while message not in callback(*args): + # try again in one second (to not hammer the CPU) + sleep(1) + except Exception: + # add additional infos to any error to make tracing down the error easier + logger.error("failed waiting for '%s'" % message) + logger.error(_healthcheck("logs", "autoheal")) + logger.error(_healthcheck("ps")) + logger.error(_healthcheck("exec", "-T", proxy, "healthcheck", retcode=None)) + raise + + +def _sha256(text): + return hashlib.sha256(str(text).encode('utf-8')).hexdigest() + + +@pytest.fixture(scope="function", autouse=True) +def _cleanup_docker_compose(tmp_path): + with local.cwd(tmp_path): + custom_compose_project_name = "%s_%s" % (os.path.basename(tmp_path), _sha256(tmp_path)[:6]) + with local.env(COMPOSE_PROJECT_NAME=custom_compose_project_name) as env: + yield env + + # stop autoheal first to prevent it from restarting containers to be stopped + _healthcheck("stop", "autoheal") + _healthcheck("down", "-v") + + +@pytest.mark.parametrize("proxy,target", PROXY_TARGET_PAIRS) +def test_healthcheck_ok(proxy, target): + # given a started proxy with healthcheck + _healthcheck("up", "-d", proxy) + + # when everything is ok and target is Up + assert "Up" in _healthcheck("ps", target) + + # then healthcheck should be successful + _healthcheck("exec", "-T", proxy, "healthcheck") + + +@pytest.mark.parametrize("proxy,target", PROXY_TARGET_PAIRS) +def test_healthcheck_failing(proxy, target): + # given a started proxy with healthcheck + _healthcheck("up", "-d", proxy) + + # when target is not reachable + _healthcheck("stop", target) + assert " Exit " in _healthcheck("ps", target) + + # then healthcheck should return an error (non zero exit code) + with pytest.raises( + plumbum.commands.processes.ProcessExecutionError, + match=r"Unexpected exit code: (1|137)", + ): + _healthcheck("exec", "-T", proxy, "healthcheck") + + +@pytest.mark.parametrize( + "proxy,target", + (p for p in PROXY_TARGET_PAIRS if p[0] != "proxy_without_preresolve"), +) +@pytest.mark.timeout(60) +def test_healthcheck_autoheal(proxy, target): + # given a started proxy with healthcheck + _healthcheck("up", "-d", proxy) + proxy_container_id = _get_container_id(proxy) + # that was healthy + _wait_for(proxy, "Up (healthy)", _healthcheck, "ps", proxy) + + # when target gets a new ip + _new_ip(target) + + # then autoheal should restart the proxy + _wait_for( + proxy, + "(%s) found to be unhealthy - Restarting container now" + % proxy_container_id[:12], + _healthcheck, + "logs", + "autoheal", + ) + + # and the proxy should become healthy + _wait_for(proxy, "Up (healthy)", _healthcheck, "ps", proxy) + + # and healthcheck should be successful + _healthcheck("exec", "-T", proxy, "healthcheck") + + +def test_healthcheck_autoheal_proxy_without_preresolve(): + # given a started proxy with healthcheck + proxy = "proxy_without_preresolve" + _healthcheck("up", "-d", proxy) + # that was healthy + _wait_for(proxy, "Up (healthy)", _healthcheck, "ps", proxy) + + # when target gets a new ip + _new_ip("target") + + # then healthcheck should be always successful (we wait just for 5 seconds/healthchecks) + for _ in range(0, 5): + _healthcheck("exec", "-T", proxy, "healthcheck") + sleep(1) + + # and autoheal shouldn't have restarted anything + assert not [ + line + for line in _healthcheck("logs", "autoheal").split("\n") + if line and not line.startswith("Attaching to ") + ] diff --git a/tests/test_healthcheck.sh b/tests/test_healthcheck.sh deleted file mode 100755 index 73ea79b..0000000 --- a/tests/test_healthcheck.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -e - -docker-compose -f tests/test.yaml build proxy_preresolve -docker-compose -f tests/test.yaml up -d proxy_preresolve -docker-compose -f tests/test.yaml exec proxy_preresolve /usr/local/bin/healthcheck -docker-compose -f tests/test.yaml stop target -# healthcheck should fail if target is stopped -! docker-compose -f tests/test.yaml exec proxy_preresolve /usr/local/bin/healthcheck