From cd9edb9089e81a66b1705f6c51dfdd5bcfef9d97 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Mon, 23 Dec 2024 13:59:08 -0800 Subject: [PATCH] feat(devservices): Add healthcheck wait condition and parallelize starting of containers (#178) * add healthcheck wait condition and parallelize starting of containers --- devservices/commands/up.py | 29 +++- devservices/constants.py | 2 + devservices/exceptions.py | 11 ++ devservices/utils/docker.py | 61 ++++++- devservices/utils/docker_compose.py | 26 +++ tests/commands/test_up.py | 249 +++++++++++++++++++++++++++- tests/utils/test_docker.py | 187 +++++++++++++++++++++ 7 files changed, 561 insertions(+), 4 deletions(-) diff --git a/devservices/commands/up.py b/devservices/commands/up.py index e7f383f4..816634fe 100644 --- a/devservices/commands/up.py +++ b/devservices/commands/up.py @@ -1,5 +1,6 @@ from __future__ import annotations +import concurrent.futures import os import subprocess from argparse import _SubParsersAction @@ -14,6 +15,7 @@ from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY from devservices.constants import DEVSERVICES_DIR_NAME from devservices.exceptions import ConfigError +from devservices.exceptions import ContainerHealthcheckFailedError from devservices.exceptions import DependencyError from devservices.exceptions import DockerComposeError from devservices.exceptions import ModeDoesNotExistError @@ -23,7 +25,9 @@ from devservices.utils.dependencies import construct_dependency_graph from devservices.utils.dependencies import install_and_verify_dependencies from devservices.utils.dependencies import InstalledRemoteDependency +from devservices.utils.docker import check_all_containers_healthy from devservices.utils.docker_compose import DockerComposeCommand +from devservices.utils.docker_compose import get_container_names_for_project from devservices.utils.docker_compose import get_docker_compose_commands_to_run from devservices.utils.docker_compose import run_cmd from devservices.utils.services import find_matching_service @@ -144,8 +148,31 @@ def _up( mode_dependencies=mode_dependencies, ) + containers_to_check = [] + with concurrent.futures.ThreadPoolExecutor() as dependency_executor: + futures = [ + dependency_executor.submit(_bring_up_dependency, cmd, current_env, status) + for cmd in docker_compose_commands + ] + for future in concurrent.futures.as_completed(futures): + _ = future.result() + for cmd in docker_compose_commands: - _bring_up_dependency(cmd, current_env, status) + try: + container_names = get_container_names_for_project( + cmd.project_name, cmd.config_path + ) + containers_to_check.extend(container_names) + except DockerComposeError as dce: + status.failure( + f"Failed to get containers to healthcheck for {cmd.project_name}: {dce.stderr}" + ) + exit(1) + try: + check_all_containers_healthy(status, containers_to_check) + except ContainerHealthcheckFailedError as e: + status.failure(str(e)) + exit(1) def _create_devservices_network() -> None: diff --git a/devservices/constants.py b/devservices/constants.py index 23f83cec..b61b6eb6 100644 --- a/devservices/constants.py +++ b/devservices/constants.py @@ -37,3 +37,5 @@ DEVSERVICES_CACHE_DIR, "latest_version.txt" ) DEVSERVICES_LATEST_VERSION_CACHE_TTL = timedelta(minutes=15) +HEALTHCHECK_TIMEOUT = 30 +HEALTHCHECK_INTERVAL = 5 diff --git a/devservices/exceptions.py b/devservices/exceptions.py index 8e1269bf..6c66ad5f 100644 --- a/devservices/exceptions.py +++ b/devservices/exceptions.py @@ -127,3 +127,14 @@ class FailedToSetGitConfigError(GitConfigError): """Raised when a git config cannot be set.""" pass + + +class ContainerHealthcheckFailedError(Exception): + """Raised when a container is not healthy.""" + + def __init__(self, container_name: str, timeout: int): + self.container_name = container_name + self.timeout = timeout + + def __str__(self) -> str: + return f"Container {self.container_name} did not become healthy within {self.timeout} seconds." diff --git a/devservices/utils/docker.py b/devservices/utils/docker.py index d12d909b..c5581dda 100644 --- a/devservices/utils/docker.py +++ b/devservices/utils/docker.py @@ -1,9 +1,15 @@ from __future__ import annotations +import concurrent.futures import subprocess +import time +from devservices.constants import HEALTHCHECK_INTERVAL +from devservices.constants import HEALTHCHECK_TIMEOUT +from devservices.exceptions import ContainerHealthcheckFailedError from devservices.exceptions import DockerDaemonNotRunningError from devservices.exceptions import DockerError +from devservices.utils.console import Status def check_docker_daemon_running() -> None: @@ -19,9 +25,62 @@ def check_docker_daemon_running() -> None: raise DockerDaemonNotRunningError from e +def check_all_containers_healthy(status: Status, containers: list[str]) -> None: + """Ensures all containers are healthy.""" + with concurrent.futures.ThreadPoolExecutor() as healthcheck_executor: + futures = [ + healthcheck_executor.submit(wait_for_healthy, container, status) + for container in containers + ] + for future in concurrent.futures.as_completed(futures): + future.result() + + +def wait_for_healthy(container_name: str, status: Status) -> None: + """ + Polls a Docker container's health status until it becomes healthy or a timeout is reached. + """ + start = time.time() + while time.time() - start < HEALTHCHECK_TIMEOUT: + # Run docker inspect to get the container's health status + try: + # For containers with no healthchecks, the output will be "unknown" + result = subprocess.check_output( + [ + "docker", + "inspect", + "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}", + container_name, + ], + stderr=subprocess.DEVNULL, + text=True, + ).strip() + except subprocess.CalledProcessError as e: + raise DockerError( + command=f"docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' {container_name}", + returncode=e.returncode, + stdout=e.stdout, + stderr=e.stderr, + ) from e + + if result == "healthy": + return + elif result == "unknown": + status.warning( + f"WARNING: Container {container_name} does not have a healthcheck" + ) + return + + # If not healthy, wait and try again + time.sleep(HEALTHCHECK_INTERVAL) + + raise ContainerHealthcheckFailedError(container_name, HEALTHCHECK_TIMEOUT) + + def get_matching_containers(label: str) -> list[str]: """ - Returns a list of container IDs with the given label + Returns a list of container names with the given label """ check_docker_daemon_running() try: diff --git a/devservices/utils/docker_compose.py b/devservices/utils/docker_compose.py index 64d9767d..ff43f4d7 100644 --- a/devservices/utils/docker_compose.py +++ b/devservices/utils/docker_compose.py @@ -95,6 +95,32 @@ def install_docker_compose() -> None: console.success(f"Verified Docker Compose installation: v{version}") +def get_container_names_for_project(project_name: str, config_path: str) -> list[str]: + try: + container_names = subprocess.check_output( + [ + "docker", + "compose", + "-p", + project_name, + "-f", + config_path, + "ps", + "--format", + "{{.Name}}", + ], + text=True, + ).splitlines() + return container_names + except subprocess.CalledProcessError as e: + raise DockerComposeError( + command=f"docker compose -p {project_name} -f {config_path} ps --format {{.Name}}", + returncode=e.returncode, + stdout=e.stdout, + stderr=e.stderr, + ) from e + + def check_docker_compose_version() -> None: console = Console() # Throw an error if docker daemon isn't running diff --git a/tests/commands/test_up.py b/tests/commands/test_up.py index a86b4290..745e9c9d 100644 --- a/tests/commands/test_up.py +++ b/tests/commands/test_up.py @@ -13,7 +13,9 @@ from devservices.constants import DEPENDENCY_CONFIG_VERSION from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY from devservices.constants import DEVSERVICES_DIR_NAME +from devservices.constants import HEALTHCHECK_TIMEOUT from devservices.exceptions import ConfigError +from devservices.exceptions import ContainerHealthcheckFailedError from devservices.exceptions import DependencyError from devservices.exceptions import ServiceNotFoundError from devservices.utils.state import State @@ -32,7 +34,14 @@ ) @mock.patch("devservices.utils.state.State.update_started_service") @mock.patch("devservices.commands.up._create_devservices_network") +@mock.patch("devservices.commands.up.check_all_containers_healthy") +@mock.patch( + "devservices.commands.up.subprocess.check_output", + return_value="clickhouse\nredis\n", +) def test_up_simple( + mock_subprocess_check_output: mock.Mock, + mock_check_all_containers_healthy: mock.Mock, mock_create_devservices_network: mock.Mock, mock_update_started_service: mock.Mock, mock_run: mock.Mock, @@ -100,6 +109,7 @@ def test_up_simple( ) mock_update_started_service.assert_called_with("example-service", "default") + mock_check_all_containers_healthy.assert_called_once() captured = capsys.readouterr() assert "Retrieving dependencies" in captured.out.strip() assert "Starting 'example-service' in mode: 'default'" in captured.out.strip() @@ -110,7 +120,9 @@ def test_up_simple( @mock.patch("devservices.utils.docker_compose.subprocess.run") @mock.patch("devservices.utils.state.State.update_started_service") @mock.patch("devservices.commands.up._create_devservices_network") +@mock.patch("devservices.commands.up.check_all_containers_healthy") def test_up_dependency_error( + mock_check_all_containers_healthy: mock.Mock, mock_create_devservices_network: mock.Mock, mock_update_started_service: mock.Mock, mock_run: mock.Mock, @@ -150,6 +162,7 @@ def test_up_dependency_error( up(args) mock_create_devservices_network.assert_not_called() + mock_check_all_containers_healthy.assert_not_called() # Capture the printed output captured = capsys.readouterr() @@ -171,7 +184,9 @@ def test_up_dependency_error( @mock.patch("devservices.utils.docker_compose.subprocess.run") @mock.patch("devservices.utils.state.State.update_started_service") @mock.patch("devservices.commands.up._create_devservices_network") +@mock.patch("devservices.commands.up.check_all_containers_healthy") def test_up_error( + mock_check_all_containers_healthy: mock.Mock, mock_create_devservices_network: mock.Mock, mock_update_started_service: mock.Mock, mock_run: mock.Mock, @@ -208,7 +223,7 @@ def test_up_error( up(args) mock_create_devservices_network.assert_called_once() - + mock_check_all_containers_healthy.assert_not_called() # Capture the printed output captured = capsys.readouterr() @@ -251,7 +266,220 @@ def test_up_error( ) @mock.patch("devservices.utils.state.State.update_started_service") @mock.patch("devservices.commands.up._create_devservices_network") +@mock.patch("devservices.commands.up.check_all_containers_healthy") +@mock.patch( + "devservices.commands.up.subprocess.check_output", + side_effect=[ + "clickhouse\nredis\n", + subprocess.CalledProcessError(returncode=1, cmd="docker compose ps"), + ], +) +def test_up_docker_compose_container_lookup_error( + mock_subprocess_check_output: mock.Mock, + mock_check_all_containers_healthy: mock.Mock, + mock_create_devservices_network: mock.Mock, + mock_update_started_service: mock.Mock, + mock_run: mock.Mock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + with mock.patch( + "devservices.commands.up.DEVSERVICES_DEPENDENCIES_CACHE_DIR", + str(tmp_path / "dependency-dir"), + ): + config = { + "x-sentry-service-config": { + "version": 0.1, + "service_name": "example-service", + "dependencies": { + "redis": {"description": "Redis"}, + "clickhouse": {"description": "Clickhouse"}, + }, + "modes": {"default": ["redis", "clickhouse"]}, + }, + "services": { + "redis": {"image": "redis:6.2.14-alpine"}, + "clickhouse": { + "image": "altinity/clickhouse-server:23.8.11.29.altinitystable" + }, + }, + } + + service_path = tmp_path / "example-service" + create_config_file(service_path, config) + os.chdir(service_path) + + args = Namespace(service_name=None, debug=False, mode="default") + + with pytest.raises(SystemExit): + up(args) + + # Ensure the DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY is set and is relative + env_vars = mock_run.call_args[1]["env"] + assert ( + env_vars[DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY] + == f"../dependency-dir/{DEPENDENCY_CONFIG_VERSION}" + ) + + mock_create_devservices_network.assert_called_once() + + mock_run.assert_called_with( + [ + "docker", + "compose", + "-p", + "example-service", + "-f", + f"{service_path}/{DEVSERVICES_DIR_NAME}/{CONFIG_FILE_NAME}", + "up", + "clickhouse", + "redis", + "-d", + "--pull", + "always", + ], + check=True, + capture_output=True, + text=True, + env=mock.ANY, + ) + + mock_update_started_service.assert_not_called() + mock_check_all_containers_healthy.assert_not_called() + captured = capsys.readouterr() + assert "Retrieving dependencies" in captured.out.strip() + assert "Starting 'example-service' in mode: 'default'" in captured.out.strip() + assert "Starting clickhouse" in captured.out.strip() + assert "Starting redis" in captured.out.strip() + assert ( + "Failed to get containers to healthcheck for example-service" + in captured.out.strip() + ) + + +@mock.patch( + "devservices.utils.docker_compose.subprocess.run", + return_value=subprocess.CompletedProcess( + args=["docker", "compose", "config", "--services"], + returncode=0, + stdout="clickhouse\nredis\n", + ), +) +@mock.patch("devservices.utils.state.State.update_started_service") +@mock.patch("devservices.commands.up._create_devservices_network") +@mock.patch( + "devservices.commands.up.check_all_containers_healthy", + side_effect=ContainerHealthcheckFailedError("container1", HEALTHCHECK_TIMEOUT), +) +@mock.patch( + "devservices.commands.up.subprocess.check_output", + side_effect=[ + "clickhouse\nredis\n", + "healthy", + "unhealthy", + ], +) +def test_up_docker_compose_container_healthcheck_failed( + mock_subprocess_check_output: mock.Mock, + mock_check_all_containers_healthy: mock.Mock, + mock_create_devservices_network: mock.Mock, + mock_update_started_service: mock.Mock, + mock_run: mock.Mock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + with mock.patch( + "devservices.commands.up.DEVSERVICES_DEPENDENCIES_CACHE_DIR", + str(tmp_path / "dependency-dir"), + ): + config = { + "x-sentry-service-config": { + "version": 0.1, + "service_name": "example-service", + "dependencies": { + "redis": {"description": "Redis"}, + "clickhouse": {"description": "Clickhouse"}, + }, + "modes": {"default": ["redis", "clickhouse"]}, + }, + "services": { + "redis": {"image": "redis:6.2.14-alpine"}, + "clickhouse": { + "image": "altinity/clickhouse-server:23.8.11.29.altinitystable" + }, + }, + } + + service_path = tmp_path / "example-service" + create_config_file(service_path, config) + os.chdir(service_path) + + args = Namespace(service_name=None, debug=False, mode="default") + + with pytest.raises(SystemExit): + up(args) + + # Ensure the DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY is set and is relative + env_vars = mock_run.call_args[1]["env"] + assert ( + env_vars[DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY] + == f"../dependency-dir/{DEPENDENCY_CONFIG_VERSION}" + ) + + mock_create_devservices_network.assert_called_once() + + mock_run.assert_called_with( + [ + "docker", + "compose", + "-p", + "example-service", + "-f", + f"{service_path}/{DEVSERVICES_DIR_NAME}/{CONFIG_FILE_NAME}", + "up", + "clickhouse", + "redis", + "-d", + "--pull", + "always", + ], + check=True, + capture_output=True, + text=True, + env=mock.ANY, + ) + + mock_update_started_service.assert_not_called() + mock_check_all_containers_healthy.assert_called_once() + captured = capsys.readouterr() + assert "Retrieving dependencies" in captured.out.strip() + assert "Starting 'example-service' in mode: 'default'" in captured.out.strip() + assert "Starting clickhouse" in captured.out.strip() + assert "Starting redis" in captured.out.strip() + assert ( + "Container container1 did not become healthy within 30 seconds." + in captured.out.strip() + ) + + +@mock.patch( + "devservices.utils.docker_compose.subprocess.run", + return_value=subprocess.CompletedProcess( + args=["docker", "compose", "config", "--services"], + returncode=0, + stdout="clickhouse\nredis\n", + ), +) +@mock.patch("devservices.utils.state.State.update_started_service") +@mock.patch("devservices.commands.up._create_devservices_network") +@mock.patch("devservices.commands.up.check_all_containers_healthy") +@mock.patch( + "devservices.commands.up.subprocess.check_output", + return_value="clickhouse\nredis\n", +) def test_up_mode_simple( + mock_subprocess_check_output: mock.Mock, + mock_check_all_containers_healthy: mock.Mock, mock_create_devservices_network: mock.Mock, mock_update_started_service: mock.Mock, mock_run: mock.Mock, @@ -318,6 +546,7 @@ def test_up_mode_simple( ) mock_update_started_service.assert_called_with("example-service", "test") + mock_check_all_containers_healthy.assert_called_once() captured = capsys.readouterr() assert "Retrieving dependencies" in captured.out.strip() assert "Starting 'example-service' in mode: 'test'" in captured.out.strip() @@ -333,7 +562,9 @@ def test_up_mode_simple( ), ) @mock.patch("devservices.utils.state.State.update_started_service") +@mock.patch("devservices.commands.up.check_all_containers_healthy") def test_up_mode_does_not_exist( + mock_check_all_containers_healthy: mock.Mock, mock_update_started_service: mock.Mock, mock_run: mock.Mock, tmp_path: Path, @@ -379,6 +610,7 @@ def test_up_mode_does_not_exist( ) mock_update_started_service.assert_not_called() + mock_check_all_containers_healthy.assert_not_called() mock_run.assert_not_called() @@ -397,7 +629,9 @@ def test_up_mode_does_not_exist( stdout="clickhouse\nredis\n", ), ) +@mock.patch("devservices.commands.up.check_all_containers_healthy") def test_up_mutliple_modes( + mock_check_all_containers_healthy: mock.Mock, mock_run: mock.Mock, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -461,6 +695,7 @@ def test_up_mutliple_modes( ], any_order=True, ) + mock_check_all_containers_healthy.assert_called_once() captured = capsys.readouterr() assert "Starting 'example-service' in mode: 'test'" in captured.out.strip() @@ -468,7 +703,9 @@ def test_up_mutliple_modes( assert "Starting redis" in captured.out.strip() +@mock.patch("devservices.commands.up.check_all_containers_healthy") def test_up_multiple_modes_overlapping_running_service( + mock_check_all_containers_healthy: mock.Mock, tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: @@ -586,6 +823,7 @@ def test_up_multiple_modes_overlapping_running_service( ), ], ) + mock_check_all_containers_healthy.assert_called_once() captured = capsys.readouterr() assert "Starting 'example-service' in mode: 'test'" in captured.out.strip() @@ -594,7 +832,9 @@ def test_up_multiple_modes_overlapping_running_service( @mock.patch("devservices.commands.up.find_matching_service") +@mock.patch("devservices.commands.up.check_all_containers_healthy") def test_up_config_error( + mock_check_all_containers_healthy: mock.Mock, find_matching_service_mock: mock.Mock, capsys: pytest.CaptureFixture[str], ) -> None: @@ -605,13 +845,17 @@ def test_up_config_error( up(args) find_matching_service_mock.assert_called_once_with("example-service") + mock_check_all_containers_healthy.assert_not_called() captured = capsys.readouterr() assert "Config error" in captured.out.strip() @mock.patch("devservices.commands.up.find_matching_service") +@mock.patch("devservices.commands.up.check_all_containers_healthy") def test_up_service_not_found_error( - find_matching_service_mock: mock.Mock, capsys: pytest.CaptureFixture[str] + mock_check_all_containers_healthy: mock.Mock, + find_matching_service_mock: mock.Mock, + capsys: pytest.CaptureFixture[str], ) -> None: find_matching_service_mock.side_effect = ServiceNotFoundError("Service not found") args = Namespace(service_name="example-service", debug=False, mode="test") @@ -620,5 +864,6 @@ def test_up_service_not_found_error( up(args) find_matching_service_mock.assert_called_once_with("example-service") + mock_check_all_containers_healthy.assert_not_called() captured = capsys.readouterr() assert "Service not found" in captured.out.strip() diff --git a/tests/utils/test_docker.py b/tests/utils/test_docker.py index dfee8f7e..0e9ac6aa 100644 --- a/tests/utils/test_docker.py +++ b/tests/utils/test_docker.py @@ -1,19 +1,26 @@ from __future__ import annotations import subprocess +from datetime import timedelta from unittest import mock import pytest +from freezegun import freeze_time from devservices.constants import DEVSERVICES_ORCHESTRATOR_LABEL from devservices.constants import DOCKER_NETWORK_NAME +from devservices.constants import HEALTHCHECK_INTERVAL +from devservices.constants import HEALTHCHECK_TIMEOUT +from devservices.exceptions import ContainerHealthcheckFailedError from devservices.exceptions import DockerDaemonNotRunningError from devservices.exceptions import DockerError +from devservices.utils.docker import check_all_containers_healthy from devservices.utils.docker import check_docker_daemon_running from devservices.utils.docker import get_matching_containers from devservices.utils.docker import get_matching_networks from devservices.utils.docker import get_volumes_for_containers from devservices.utils.docker import stop_containers +from devservices.utils.docker import wait_for_healthy @mock.patch("subprocess.run") @@ -290,3 +297,183 @@ def test_stop_containers_remove_error( ), ] ) + + +@mock.patch("devservices.utils.docker.subprocess.check_output", return_value="healthy") +def test_wait_for_healthy_success(mock_check_output: mock.Mock) -> None: + mock_status = mock.Mock() + wait_for_healthy("container1", mock_status) + mock_check_output.assert_called_once_with( + [ + "docker", + "inspect", + "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}", + "container1", + ], + stderr=subprocess.DEVNULL, + text=True, + ) + mock_status.failure.assert_not_called() + + +@mock.patch("devservices.utils.docker.subprocess.check_output", return_value="unknown") +def test_wait_for_healthy_no_healthcheck(mock_check_output: mock.Mock) -> None: + mock_status = mock.Mock() + wait_for_healthy("container1", mock_status) + mock_check_output.assert_called_once_with( + [ + "docker", + "inspect", + "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}", + "container1", + ], + stderr=subprocess.DEVNULL, + text=True, + ) + mock_status.failure.assert_not_called() + + +@mock.patch("devservices.utils.docker.subprocess.check_output") +@mock.patch("devservices.utils.docker.time.sleep") +def test_wait_for_healthy_initial_check_failed_then_success( + mock_sleep: mock.Mock, + mock_check_output: mock.Mock, +) -> None: + mock_status = mock.Mock() + mock_check_output.side_effect = ["unhealthy", "healthy"] + + with (freeze_time("2024-05-14 00:00:00") as frozen_time,): + mock_sleep.side_effect = lambda _: frozen_time.tick(timedelta(seconds=1)) + wait_for_healthy("container1", mock_status) + + mock_check_output.assert_has_calls( + [ + mock.call( + [ + "docker", + "inspect", + "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}", + "container1", + ], + stderr=subprocess.DEVNULL, + text=True, + ), + mock.call( + [ + "docker", + "inspect", + "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}", + "container1", + ], + stderr=subprocess.DEVNULL, + text=True, + ), + ] + ) + mock_sleep.assert_called_once_with(HEALTHCHECK_INTERVAL) + mock_status.failure.assert_not_called() + + +@mock.patch("devservices.utils.docker.subprocess.check_output") +@mock.patch("devservices.utils.docker.time.sleep") +def test_wait_for_healthy_docker_error( + mock_sleep: mock.Mock, + mock_check_output: mock.Mock, +) -> None: + mock_status = mock.Mock() + mock_check_output.side_effect = subprocess.CalledProcessError(1, "cmd") + with pytest.raises(DockerError): + with freeze_time("2024-05-14 00:00:00") as frozen_time: + mock_sleep.side_effect = lambda _: frozen_time.tick(timedelta(seconds=1)) + wait_for_healthy("container1", mock_status) + mock_check_output.assert_called_once_with( + [ + "docker", + "inspect", + "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}", + "container1", + ], + stderr=subprocess.DEVNULL, + text=True, + ) + + +@mock.patch("devservices.utils.docker.subprocess.check_output") +@mock.patch("devservices.utils.docker.time.sleep") +def test_wait_for_healthy_healthcheck_failed( + mock_sleep: mock.Mock, + mock_check_output: mock.Mock, +) -> None: + mock_status = mock.Mock() + mock_check_output.return_value = "unhealthy" + with freeze_time("2024-05-14 00:00:00") as frozen_time: + with pytest.raises(ContainerHealthcheckFailedError): + mock_sleep.side_effect = lambda _: frozen_time.tick( + timedelta(seconds=HEALTHCHECK_TIMEOUT / 2) + ) + wait_for_healthy("container1", mock_status) + mock_check_output.assert_has_calls( + [ + mock.call( + [ + "docker", + "inspect", + "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}", + "container1", + ], + stderr=subprocess.DEVNULL, + text=True, + ), + mock.call( + [ + "docker", + "inspect", + "-f", + "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}", + "container1", + ], + stderr=subprocess.DEVNULL, + text=True, + ), + ] + ) + + +@mock.patch("devservices.utils.docker.wait_for_healthy") +def test_check_all_containers_healthy_success( + mock_wait_for_healthy: mock.Mock, +) -> None: + mock_status = mock.Mock() + mock_wait_for_healthy.side_effect = [None, None] + check_all_containers_healthy(mock_status, ["container1", "container2"]) + mock_wait_for_healthy.assert_has_calls( + [ + mock.call("container1", mock_status), + mock.call("container2", mock_status), + ] + ) + + +@mock.patch("devservices.utils.docker.wait_for_healthy") +def test_check_all_containers_healthy_failure( + mock_wait_for_healthy: mock.Mock, +) -> None: + mock_status = mock.Mock() + mock_wait_for_healthy.side_effect = [ + None, + ContainerHealthcheckFailedError("container2", HEALTHCHECK_TIMEOUT), + ] + with pytest.raises(ContainerHealthcheckFailedError): + check_all_containers_healthy(mock_status, ["container1", "container2"]) + mock_wait_for_healthy.assert_has_calls( + [ + mock.call("container1", mock_status), + mock.call("container2", mock_status), + ] + )