feat(devservices): Add healthcheck wait condition and parallelize sta…

…rting of containers (#178)

* add healthcheck wait condition and parallelize starting of containers

devservices/commands/up.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import concurrent.futures
 import os
 import subprocess
 from argparse import _SubParsersAction
@@ -14,6 +15,7 @@
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
 from devservices.exceptions import ConfigError
+from devservices.exceptions import ContainerHealthcheckFailedError
 from devservices.exceptions import DependencyError
 from devservices.exceptions import DockerComposeError
 from devservices.exceptions import ModeDoesNotExistError
@@ -23,7 +25,9 @@
 from devservices.utils.dependencies import construct_dependency_graph
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
+from devservices.utils.docker import check_all_containers_healthy
 from devservices.utils.docker_compose import DockerComposeCommand
+from devservices.utils.docker_compose import get_container_names_for_project
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run
 from devservices.utils.docker_compose import run_cmd
 from devservices.utils.services import find_matching_service
@@ -144,8 +148,31 @@ def _up(
         mode_dependencies=mode_dependencies,
     )
 
+    containers_to_check = []
+    with concurrent.futures.ThreadPoolExecutor() as dependency_executor:
+        futures = [
+            dependency_executor.submit(_bring_up_dependency, cmd, current_env, status)
+            for cmd in docker_compose_commands
+        ]
+        for future in concurrent.futures.as_completed(futures):
+            _ = future.result()
+
     for cmd in docker_compose_commands:
-        _bring_up_dependency(cmd, current_env, status)
+        try:
+            container_names = get_container_names_for_project(
+                cmd.project_name, cmd.config_path
+            )
+            containers_to_check.extend(container_names)
+        except DockerComposeError as dce:
+            status.failure(
+                f"Failed to get containers to healthcheck for {cmd.project_name}: {dce.stderr}"
+            )
+            exit(1)
+    try:
+        check_all_containers_healthy(status, containers_to_check)
+    except ContainerHealthcheckFailedError as e:
+        status.failure(str(e))
+        exit(1)
 
 
 def _create_devservices_network() -> None:

devservices/constants.py

@@ -37,3 +37,5 @@
     DEVSERVICES_CACHE_DIR, "latest_version.txt"
 )
 DEVSERVICES_LATEST_VERSION_CACHE_TTL = timedelta(minutes=15)
+HEALTHCHECK_TIMEOUT = 30
+HEALTHCHECK_INTERVAL = 5

devservices/exceptions.py

@@ -127,3 +127,14 @@ class FailedToSetGitConfigError(GitConfigError):
     """Raised when a git config cannot be set."""
 
     pass
+
+
+class ContainerHealthcheckFailedError(Exception):
+    """Raised when a container is not healthy."""
+
+    def __init__(self, container_name: str, timeout: int):
+        self.container_name = container_name
+        self.timeout = timeout
+
+    def __str__(self) -> str:
+        return f"Container {self.container_name} did not become healthy within {self.timeout} seconds."

devservices/utils/docker.py

@@ -1,9 +1,15 @@
 from __future__ import annotations
 
+import concurrent.futures
 import subprocess
+import time
 
+from devservices.constants import HEALTHCHECK_INTERVAL
+from devservices.constants import HEALTHCHECK_TIMEOUT
+from devservices.exceptions import ContainerHealthcheckFailedError
 from devservices.exceptions import DockerDaemonNotRunningError
 from devservices.exceptions import DockerError
+from devservices.utils.console import Status
 
 
 def check_docker_daemon_running() -> None:
@@ -19,9 +25,62 @@ def check_docker_daemon_running() -> None:
         raise DockerDaemonNotRunningError from e
 
 
+def check_all_containers_healthy(status: Status, containers: list[str]) -> None:
+    """Ensures all containers are healthy."""
+    with concurrent.futures.ThreadPoolExecutor() as healthcheck_executor:
+        futures = [
+            healthcheck_executor.submit(wait_for_healthy, container, status)
+            for container in containers
+        ]
+        for future in concurrent.futures.as_completed(futures):
+            future.result()
+
+
+def wait_for_healthy(container_name: str, status: Status) -> None:
+    """
+    Polls a Docker container's health status until it becomes healthy or a timeout is reached.
+    """
+    start = time.time()
+    while time.time() - start < HEALTHCHECK_TIMEOUT:
+        # Run docker inspect to get the container's health status
+        try:
+            # For containers with no healthchecks, the output will be "unknown"
+            result = subprocess.check_output(
+                [
+                    "docker",
+                    "inspect",
+                    "-f",
+                    "{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}",
+                    container_name,
+                ],
+                stderr=subprocess.DEVNULL,
+                text=True,
+            ).strip()
+        except subprocess.CalledProcessError as e:
+            raise DockerError(
+                command=f"docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}unknown{{end}}' {container_name}",
+                returncode=e.returncode,
+                stdout=e.stdout,
+                stderr=e.stderr,
+            ) from e
+
+        if result == "healthy":
+            return
+        elif result == "unknown":
+            status.warning(
+                f"WARNING: Container {container_name} does not have a healthcheck"
+            )
+            return
+
+        # If not healthy, wait and try again
+        time.sleep(HEALTHCHECK_INTERVAL)
+
+    raise ContainerHealthcheckFailedError(container_name, HEALTHCHECK_TIMEOUT)
+
+
 def get_matching_containers(label: str) -> list[str]:
     """
-    Returns a list of container IDs with the given label
+    Returns a list of container names with the given label
     """
     check_docker_daemon_running()
     try:

devservices/utils/docker_compose.py

@@ -95,6 +95,32 @@ def install_docker_compose() -> None:
     console.success(f"Verified Docker Compose installation: v{version}")
 
 
+def get_container_names_for_project(project_name: str, config_path: str) -> list[str]:
+    try:
+        container_names = subprocess.check_output(
+            [
+                "docker",
+                "compose",
+                "-p",
+                project_name,
+                "-f",
+                config_path,
+                "ps",
+                "--format",
+                "{{.Name}}",
+            ],
+            text=True,
+        ).splitlines()
+        return container_names
+    except subprocess.CalledProcessError as e:
+        raise DockerComposeError(
+            command=f"docker compose -p {project_name} -f {config_path} ps --format {{.Name}}",
+            returncode=e.returncode,
+            stdout=e.stdout,
+            stderr=e.stderr,
+        ) from e
+
+
 def check_docker_compose_version() -> None:
     console = Console()
     # Throw an error if docker daemon isn't running