-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add healthcheck support for http and smtp
this enables the user to configure a healthcheck for containers that can be used to automatically restart proxies resolving to no longer active ips due to PRE_RESOLVE
- Loading branch information
Showing
6 changed files
with
421 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| #!/usr/bin/env python3 | ||
|
|
||
| import logging | ||
| import os | ||
|
|
||
| logging.basicConfig(level=logging.INFO) | ||
| logger = logging.getLogger("healthcheck") | ||
|
|
||
|
|
||
| def error(message, exception=None): | ||
| logger.error(message) | ||
| if exception is None: | ||
| exit(1) | ||
| else: | ||
| raise exception | ||
|
|
||
|
|
||
| def http_healthcheck(): | ||
| """ | ||
| Use pycurl to check if the target server is still responding via proxy.py | ||
| :return: None | ||
| """ | ||
| import re | ||
|
|
||
| import pycurl | ||
|
|
||
| check_url = os.environ.get("HTTP_HEALTHCHECK_URL", "http://localhost/") | ||
| target = os.environ.get("TARGET", "localhost") | ||
| check_url_with_target = check_url.replace("$TARGET", target) | ||
| port = re.search("https?://[^:]*(?::([^/]+))?", check_url_with_target)[1] or "80" | ||
| print("checking %s via 127.0.0.1" % check_url_with_target) | ||
| logger.info("checking %s via 127.0.0.1" % check_url_with_target) | ||
| try: | ||
| request = pycurl.Curl() | ||
| request.setopt(pycurl.URL, check_url_with_target) | ||
| # do not send the request to the target directly but use our own socat proxy process to check if it's still | ||
| # working | ||
| request.setopt(pycurl.RESOLVE, ["{}:{}:127.0.0.1".format(target, port)]) | ||
| request.perform() | ||
| request.close() | ||
| except pycurl.error as e: | ||
| error("error while checking http connection", e) | ||
|
|
||
|
|
||
| def smtp_healthcheck(): | ||
| """ | ||
| Use pycurl to check if the target server is still responding via proxy.py | ||
| :return: None | ||
| """ | ||
| import re | ||
|
|
||
| import pycurl | ||
|
|
||
| check_url = os.environ.get("SMTP_HEALTHCHECK_URL", "smtp://localhost/") | ||
| check_command = os.environ.get("SMTP_HEALTHCHECK_COMMAND", "HELP") | ||
| target = os.environ.get("TARGET", "localhost") | ||
| check_url_with_target = check_url.replace("$TARGET", target) | ||
| port = re.search("smtp://[^:]*(?::([^/]+))?", check_url_with_target)[1] or "25" | ||
| logger.info("checking %s via 127.0.0.1" % check_url_with_target) | ||
| try: | ||
| request = pycurl.Curl() | ||
| request.setopt(pycurl.URL, check_url_with_target) | ||
| request.setopt(pycurl.CUSTOMREQUEST, check_command) | ||
| # do not send the request to the target directly but use our own socat proxy process to check if it's still | ||
| # working | ||
| request.setopt(pycurl.RESOLVE, ["{}:{}:127.0.0.1".format(target, port)]) | ||
| request.perform() | ||
| request.close() | ||
| except pycurl.error as e: | ||
| error("error while checking smtp connection", e) | ||
|
|
||
|
|
||
| def process_healthcheck(): | ||
| """ | ||
| Check that at least one socat process exists per port and no more than the number of configured max connections | ||
| processes exist for each port. | ||
| :return: | ||
| """ | ||
| import subprocess | ||
|
|
||
| ports = os.environ["PORT"].split() | ||
| max_connections = int(os.environ["MAX_CONNECTIONS"]) | ||
| logger.info( | ||
| "checking socat processes for port(s) %s having at least one and less than %d socat processes" | ||
| % (ports, max_connections) | ||
| ) | ||
| socat_processes = ( | ||
| subprocess.check_output(["sh", "-c", "grep -R socat /proc/[0-9]*/cmdline"]) | ||
| .decode("utf-8") | ||
| .split("\n") | ||
| ) | ||
| pids = [process.split("/")[2] for process in socat_processes if process] | ||
| if len(pids) < len(ports): | ||
| # if we have less than the number of ports socat processes we do not need to count processes per port and can | ||
| # fail fast | ||
| error("Expected at least %d socat processes" % len(ports)) | ||
| port_process_count = {port: 0 for port in ports} | ||
| for pid in pids: | ||
| # foreach socat pid we detect the port it's for by checking the last argument (connect to) that ends with | ||
| # :{ip}:{port} for our processes | ||
| try: | ||
| with open("/proc/%d/cmdline" % int(pid), "rt") as fp: | ||
| # arguments in /proc/.../cmdline are split by null bytes | ||
| cmd = [part for part in "".join(fp.readlines()).split("\x00") if part] | ||
| port = cmd[2].split(":")[-1] | ||
| port_process_count[port] = port_process_count[port] + 1 | ||
| except FileNotFoundError: | ||
| # ignore processes no longer existing (possibly retrieved an answer) | ||
| pass | ||
| for port in ports: | ||
| if port_process_count[port] == 0: | ||
| error("Missing socat process(es) for port: %s" % port) | ||
| if port_process_count[port] >= max_connections + 1: | ||
| error( | ||
| "More than %d + 1 socat process(es) for port: %s" | ||
| % (max_connections, port) | ||
| ) | ||
|
|
||
|
|
||
| process_healthcheck() | ||
| if os.environ.get("HTTP_HEALTHCHECK", "0") == "1": | ||
| http_healthcheck() | ||
| if os.environ.get("SMTP_HEALTHCHECK", "0") == "1": | ||
| smtp_healthcheck() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,76 @@ | ||
| #!/usr/bin/env bash | ||
| set -e | ||
| for arg in "$@" ; do | ||
| echo arg $arg | ||
| if [[ "$arg" == "DEBUG" ]] ; then | ||
| DEBUG=1 | ||
| else | ||
| TEST_FILTER="$arg" | ||
| fi | ||
| done | ||
| DEBUG=${DEBUG:-0} | ||
| TEST_FILTER=${TEST_FILTER:-} | ||
|
|
||
| function cleanup() { | ||
| if [[ $DEBUG == 1 ]]; then | ||
| docker-compose -f tests/test.yaml ps | ||
| docker-compose -f tests/test.yaml exec -T proxy_preresolve /usr/local/bin/healthcheck || true | ||
| docker-compose -f tests/test.yaml exec -T proxy_without_preresolve /usr/local/bin/healthcheck || true | ||
| docker-compose -f tests/test.yaml top | ||
| docker-compose -f tests/test.yaml logs | ||
| fi | ||
| docker-compose -f tests/test.yaml down -v --remove-orphans | ||
| } | ||
| trap cleanup EXIT | ||
|
|
||
| function with_prefix() { | ||
| local prefix | ||
| prefix="$1" | ||
| shift | ||
| "$@" 2>&1 | while read -r line; do | ||
| echo "$prefix" "$line" | ||
| done | ||
| return "${PIPESTATUS[0]}" | ||
| } | ||
|
|
||
| function run_tests() { | ||
| for service in $(docker-compose -f tests/test.yaml config --services); do | ||
| if [[ ( $service == test_* || ( $DEBUG = 1 && $service == debug_* ) ) && $service == *"$TEST_FILTER"* ]] ; then | ||
| echo "running $service" | ||
| with_prefix "$service:" docker-compose -f tests/test.yaml run --rm "$service" | ||
| fi | ||
| done | ||
| } | ||
|
|
||
| function change_target_ips() { | ||
| for target in "target" "target_smtp"; do | ||
| #spin up a second target and remove the first target container to give it a new ip (simulates a new deployment of an external cloud service) | ||
| local target_container_id | ||
| target_container_id=$(docker-compose -f tests/test.yaml ps -q "$target") | ||
| if [[ "$target_container_id" != "" ]] ; then | ||
| if [[ $DEBUG == 1 ]] ; then | ||
| docker inspect "$target_container_id" | grep '"IPAddress": "[^"]\+' | ||
| fi | ||
| docker-compose -f tests/test.yaml up -d --scale "$target=2" "$target" | ||
| docker stop "$target_container_id" | xargs echo "stopped ${target}_1" | ||
| docker rm "$target_container_id" | xargs echo "removed ${target}_1" | ||
| if [[ $DEBUG == 1 ]] ; then | ||
| target_container_id=$(docker-compose -f tests/test.yaml ps -q "$target") | ||
| docker inspect "$target_container_id" | grep '"IPAddress": "[^"]\+' | ||
| fi | ||
| fi | ||
| done | ||
| # give docker some time to restart unhealthy containers | ||
| sleep 5 | ||
| } | ||
|
|
||
| with_prefix "build:" docker-compose -f tests/test.yaml build | ||
|
|
||
| # make sure all tests pass when target is up | ||
| run_tests | ||
|
|
||
| # when target changes ip | ||
| with_prefix "changing target_ip:" change_target_ips | ||
|
|
||
| # all tests still should pass | ||
| run_tests |
Oops, something went wrong.