diff --git a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py index 2e14ed62c16..9412f0111ee 100644 --- a/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py +++ b/packages/models-library/src/models_library/api_schemas_dynamic_sidecar/containers.py @@ -16,3 +16,5 @@ class ActivityInfo(BaseModel): ActivityInfoOrNone: TypeAlias = ActivityInfo | None + +DcokerComposeYamlStr: TypeAlias = str diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/containers.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/containers.py new file mode 100644 index 00000000000..3de2d56ecd9 --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/containers.py @@ -0,0 +1,29 @@ +import logging + +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr +from models_library.projects_nodes_io import NodeID +from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace +from pydantic import TypeAdapter + +from ....logging_utils import log_decorator +from ... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def store_compose_spec( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + node_id: NodeID, + docker_compose_yaml: DcokerComposeYamlStr, +) -> None: + rpc_namespace = RPCNamespace.from_entries( + {"service": "dy-sidecar", "node_id": f"{node_id}"} + ) + result = await rabbitmq_rpc_client.request( + rpc_namespace, + TypeAdapter(RPCMethodName).validate_python("store_compose_spec"), + docker_compose_yaml=docker_compose_yaml, + ) + assert result is None # nosec diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk.py new file mode 100644 index 00000000000..fbacd1e905a --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk.py @@ -0,0 +1,26 @@ +import logging + +from models_library.projects_nodes_io import NodeID +from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace +from pydantic import TypeAdapter + +from ....logging_utils import log_decorator +from ... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def free_reserved_disk_space( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + node_id: NodeID, +) -> None: + rpc_namespace = RPCNamespace.from_entries( + {"service": "dy-sidecar", "node_id": f"{node_id}"} + ) + result = await rabbitmq_rpc_client.request( + rpc_namespace, + TypeAdapter(RPCMethodName).validate_python("free_reserved_disk_space"), + ) + assert result is None # nosec diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py index dbace2f1f4b..d21a71f7d9b 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py @@ -1,5 +1,4 @@ import logging -from typing import Final from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage from models_library.projects_nodes_io import NodeID @@ -11,10 +10,6 @@ _logger = logging.getLogger(__name__) -_UPDATE_DISK_USAGE: Final[RPCMethodName] = TypeAdapter(RPCMethodName).validate_python( - "update_disk_usage" -) - @log_decorator(_logger, level=logging.DEBUG) async def update_disk_usage( @@ -28,7 +23,7 @@ async def update_disk_usage( ) result = await rabbitmq_rpc_client.request( rpc_namespace, - _UPDATE_DISK_USAGE, + TypeAdapter(RPCMethodName).validate_python("update_disk_usage"), usage=usage, ) assert result is None # nosec diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/volumes.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/volumes.py new file mode 100644 index 00000000000..4858f3e927f --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/volumes.py @@ -0,0 +1,31 @@ +import logging + +from models_library.projects_nodes_io import NodeID +from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace +from models_library.sidecar_volumes import VolumeCategory, VolumeStatus +from pydantic import TypeAdapter + +from ....logging_utils import log_decorator +from ... import RabbitMQRPCClient + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def save_volume_state( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + node_id: NodeID, + status: VolumeStatus, + category: VolumeCategory, +) -> None: + rpc_namespace = RPCNamespace.from_entries( + {"service": "dy-sidecar", "node_id": f"{node_id}"} + ) + result = await rabbitmq_rpc_client.request( + rpc_namespace, + TypeAdapter(RPCMethodName).validate_python("save_volume_state"), + status=status, + category=category, + ) + assert result is None # nosec diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py index 6ada9de83de..c697844c1c8 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers.py @@ -5,7 +5,7 @@ from asyncio import Lock from typing import Annotated, Any, Final -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, FastAPI, HTTPException from fastapi import Path as PathParam from fastapi import Query, Request, status from models_library.api_schemas_dynamic_sidecar.containers import ( @@ -22,18 +22,14 @@ ContainerExecTimeoutError, ) from ...core.settings import ApplicationSettings -from ...core.validation import ( - ComposeSpecValidation, - parse_compose_spec, - validate_compose_spec, -) +from ...core.validation import parse_compose_spec from ...models.schemas.containers import ContainersComposeSpec from ...models.shared_store import SharedStore from ...modules.container_utils import run_command_in_container -from ...modules.mounted_fs import MountedVolumes +from ...services import containers from ._dependencies import ( + get_application, get_container_restart_lock, - get_mounted_volumes, get_settings, get_shared_store, ) @@ -65,31 +61,16 @@ def _raise_if_container_is_missing( @cancel_on_disconnect async def store_compose_spec( request: Request, - settings: Annotated[ApplicationSettings, Depends(get_settings)], containers_compose_spec: ContainersComposeSpec, - shared_store: Annotated[SharedStore, Depends(get_shared_store)], - mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], + app: Annotated[FastAPI, Depends(get_application)], ): - """ - Validates and stores the docker compose spec for the user services. - """ - _ = request + """Validates and stores the docker compose spec for the user services.""" - async with shared_store: - compose_spec_validation: ComposeSpecValidation = await validate_compose_spec( - settings=settings, - compose_file_content=containers_compose_spec.docker_compose_yaml, - mounted_volumes=mounted_volumes, - ) - shared_store.compose_spec = compose_spec_validation.compose_spec - shared_store.container_names = compose_spec_validation.current_container_names - shared_store.original_to_container_names = ( - compose_spec_validation.original_to_current_container_names - ) - - _logger.info("Validated compose-spec:\n%s", f"{shared_store.compose_spec}") - - assert shared_store.compose_spec # nosec + _ = request + await containers.store_conpose_spec( + app, + docker_compose_yaml=containers_compose_spec.docker_compose_yaml, + ) @router.get( diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_extension.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_extension.py index d5cf21b8723..ddcaba4cfe8 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_extension.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_extension.py @@ -135,7 +135,7 @@ async def attach_container_to_network( ) return - # NOTE: A docker network is only visible on a docker node when it is + # NOTE: A docker overlay network is only visible on a docker node when it is # used by a container network = DockerNetwork(docker=docker, id_=item.network_id) await network.connect( @@ -172,7 +172,7 @@ async def detach_container_from_network( ) return - # NOTE: A docker network is only visible on a docker node when it is + # NOTE: A docker overlay network is only visible on a docker node when it is # used by a container network = DockerNetwork(docker=docker, id_=item.network_id) await network.disconnect({"Container": container_id, "Force": True}) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py index f8ce581024a..4ff0cc2dbca 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/disk.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, status -from ...core.reserved_space import remove_reserved_disk_space +from ...services import disk router = APIRouter() @@ -11,4 +11,4 @@ status_code=status.HTTP_204_NO_CONTENT, ) async def free_reserved_disk_space() -> None: - remove_reserved_disk_space() + disk.remove_reserved_disk_space() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py index ac2e833a3ab..793fbc687e9 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/volumes.py @@ -1,11 +1,13 @@ -from fastapi import APIRouter, Depends +from typing import Annotated + +from fastapi import APIRouter, Depends, FastAPI from fastapi import Path as PathParam from fastapi import status -from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus +from models_library.sidecar_volumes import VolumeCategory, VolumeStatus from pydantic import BaseModel -from ...models.shared_store import SharedStore -from ._dependencies import get_shared_store +from ...services import volumes +from ._dependencies import get_application router = APIRouter() @@ -21,8 +23,7 @@ class PutVolumeItem(BaseModel): ) async def put_volume_state( item: PutVolumeItem, - volume_category: VolumeCategory = PathParam(..., alias="id"), - shared_store: SharedStore = Depends(get_shared_store), + app: Annotated[FastAPI, Depends(get_application)], + volume_category: Annotated[VolumeCategory, PathParam(..., alias="id")], ) -> None: - async with shared_store: - shared_store.volume_states[volume_category] = VolumeState(status=item.status) + await volumes.save_volume_state(app, status=item.status, category=volume_category) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_containers.py new file mode 100644 index 00000000000..0e1f6e3ab86 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_containers.py @@ -0,0 +1,16 @@ +from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr +from pydantic import validate_call +from servicelib.rabbitmq import RPCRouter + +from ...services import containers + +router = RPCRouter() + + +@router.expose() +@validate_call(config={"arbitrary_types_allowed": True}) +async def store_compose_spec( + app: FastAPI, *, docker_compose_yaml: DcokerComposeYamlStr +) -> None: + await containers.store_conpose_spec(app, docker_compose_yaml=docker_compose_yaml) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py new file mode 100644 index 00000000000..f3bba913ac9 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk.py @@ -0,0 +1,11 @@ +from fastapi import FastAPI +from servicelib.rabbitmq import RPCRouter + +from ...services import disk + +router = RPCRouter() + + +@router.expose() +async def free_reserved_disk_space(_: FastAPI) -> None: + disk.remove_reserved_disk_space() diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk_usage.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk_usage.py index 6968250005c..8e658b769fa 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk_usage.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_disk_usage.py @@ -1,5 +1,6 @@ from fastapi import FastAPI from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage +from pydantic import validate_call from servicelib.rabbitmq import RPCRouter from ...modules.system_monitor import get_disk_usage_monitor @@ -8,5 +9,6 @@ @router.expose() +@validate_call(config={"arbitrary_types_allowed": True}) async def update_disk_usage(app: FastAPI, *, usage: dict[str, DiskUsage]) -> None: get_disk_usage_monitor(app).set_disk_usage_for_path(usage) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_volumes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_volumes.py new file mode 100644 index 00000000000..6f51d7ff629 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/_volumes.py @@ -0,0 +1,16 @@ +from fastapi import FastAPI +from models_library.sidecar_volumes import VolumeCategory, VolumeStatus +from pydantic import validate_call +from servicelib.rabbitmq import RPCRouter + +from ...services import volumes + +router = RPCRouter() + + +@router.expose() +@validate_call(config={"arbitrary_types_allowed": True}) +async def save_volume_state( + app: FastAPI, *, status: VolumeStatus, category: VolumeCategory +) -> None: + await volumes.save_volume_state(app, status=status, category=category) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py index d1c0b0c2a1e..18fbafda339 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rpc/routes.py @@ -4,10 +4,13 @@ from ...core.rabbitmq import get_rabbitmq_rpc_server from ...core.settings import ApplicationSettings -from . import _disk_usage +from . import _containers, _disk, _disk_usage, _volumes ROUTERS: list[RPCRouter] = [ + _containers.router, _disk_usage.router, + _disk.router, + _volumes.router, ] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py index e374c924070..91e57c05854 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/containers.py @@ -1,9 +1,10 @@ +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr from models_library.services_creation import CreateServiceMetricsAdditionalParams from pydantic import BaseModel class ContainersComposeSpec(BaseModel): - docker_compose_yaml: str + docker_compose_yaml: DcokerComposeYamlStr class ContainersCreate(BaseModel): diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py index 0ca422a3390..fff75a8816c 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/shared_store.py @@ -4,6 +4,7 @@ import aiofiles from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus from pydantic import BaseModel, Field, PrivateAttr @@ -50,7 +51,7 @@ class SharedStore(_StoreMixin): shared_store.container_names = copied_list """ - compose_spec: str | None = Field( + compose_spec: DcokerComposeYamlStr | None = Field( default=None, description="stores the stringified compose spec" ) container_names: list[ContainerNameStr] = Field( @@ -119,3 +120,8 @@ async def on_startup() -> None: ) app.add_event_handler("startup", on_startup) + + +def get_shared_store(app: FastAPI) -> SharedStore: + shared_store: SharedStore = app.state.shared_store + return shared_store diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/__init__.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py new file mode 100644 index 00000000000..cbd6af4db52 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/containers.py @@ -0,0 +1,41 @@ +import logging + +from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr + +from ..core.settings import ApplicationSettings +from ..core.validation import ComposeSpecValidation, validate_compose_spec +from ..models.shared_store import SharedStore, get_shared_store +from ..modules.mounted_fs import MountedVolumes + +_logger = logging.getLogger(__name__) + + +async def store_conpose_spec( + app: FastAPI, + *, + docker_compose_yaml: DcokerComposeYamlStr, +) -> None: + """ + Validates and stores the docker compose spec for the user services. + """ + + settings: ApplicationSettings = app.state.settings + mounted_volumes: MountedVolumes = app.state.mounted_volumes + shared_store: SharedStore = get_shared_store(app) + + async with shared_store: + compose_spec_validation: ComposeSpecValidation = await validate_compose_spec( + settings=settings, + compose_file_content=docker_compose_yaml, + mounted_volumes=mounted_volumes, + ) + shared_store.compose_spec = compose_spec_validation.compose_spec + shared_store.container_names = compose_spec_validation.current_container_names + shared_store.original_to_container_names = ( + compose_spec_validation.original_to_current_container_names + ) + + _logger.info("Validated compose-spec:\n%s", f"{shared_store.compose_spec}") + + assert shared_store.compose_spec # nosec diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py new file mode 100644 index 00000000000..316a86d6036 --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/disk.py @@ -0,0 +1,5 @@ +from ..core.reserved_space import remove_reserved_disk_space + +__all__: tuple[str, ...] = ("remove_reserved_disk_space",) + +# nopycln: file diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/volumes.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/volumes.py new file mode 100644 index 00000000000..366276bbaed --- /dev/null +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/services/volumes.py @@ -0,0 +1,12 @@ +from fastapi import FastAPI +from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus + +from ..models.shared_store import get_shared_store + + +async def save_volume_state( + app: FastAPI, *, status: VolumeStatus, category: VolumeCategory +) -> None: + shared_store = get_shared_store(app) + async with shared_store: + shared_store.volume_states[category] = VolumeState(status=status) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/api/rest/test_containers.py similarity index 99% rename from services/dynamic-sidecar/tests/unit/test_api_rest_containers.py rename to services/dynamic-sidecar/tests/unit/api/rest/test_containers.py index 27ec615b631..777dc32fc90 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/api/rest/test_containers.py @@ -5,10 +5,10 @@ import asyncio import json -from collections.abc import AsyncIterable +from collections.abc import AsyncIterable, AsyncIterator from inspect import signature from pathlib import Path -from typing import Any, AsyncIterator, Final +from typing import Any, Final from unittest.mock import AsyncMock, Mock from uuid import uuid4 @@ -267,10 +267,10 @@ def not_started_containers() -> list[str]: def mock_outputs_labels() -> dict[str, ServiceOutput]: return { "output_port_1": TypeAdapter(ServiceOutput).validate_python( - ServiceOutput.model_config["json_schema_extra"]["examples"][3] + ServiceOutput.model_json_schema()["examples"][3] ), "output_port_2": TypeAdapter(ServiceOutput).validate_python( - ServiceOutput.model_config["json_schema_extra"]["examples"][3] + ServiceOutput.model_json_schema()["examples"][3] ), } diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_disk.py b/services/dynamic-sidecar/tests/unit/api/rest/test_disk.py similarity index 100% rename from services/dynamic-sidecar/tests/unit/test_api_rest_disk.py rename to services/dynamic-sidecar/tests/unit/api/rest/test_disk.py diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_volumes.py b/services/dynamic-sidecar/tests/unit/api/rest/test_volumes.py similarity index 100% rename from services/dynamic-sidecar/tests/unit/test_api_rest_volumes.py rename to services/dynamic-sidecar/tests/unit/api/rest/test_volumes.py diff --git a/services/dynamic-sidecar/tests/unit/test_api_rpc__disk_usage.py b/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py similarity index 56% rename from services/dynamic-sidecar/tests/unit/test_api_rpc__disk_usage.py rename to services/dynamic-sidecar/tests/unit/api/rpc/conftest.py index 1383f165416..6097c7a6d43 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rpc__disk_usage.py +++ b/services/dynamic-sidecar/tests/unit/api/rpc/conftest.py @@ -1,38 +1,22 @@ -# pylint:disable=protected-access # pylint:disable=redefined-outer-name # pylint:disable=unused-argument -from collections.abc import Awaitable, Callable -from typing import AsyncIterable +from collections.abc import AsyncIterable, Awaitable, Callable from unittest.mock import AsyncMock import pytest from asgi_lifespan import LifespanManager from fastapi import FastAPI -from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage -from pydantic import ByteSize from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.rabbitmq import RabbitMQRPCClient -from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import disk_usage from settings_library.rabbit import RabbitSettings -from settings_library.redis import RedisSettings from simcore_service_dynamic_sidecar.core.application import create_app -from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings -from simcore_service_dynamic_sidecar.modules.system_monitor._disk_usage import ( - get_disk_usage_monitor, -) - -pytest_simcore_core_services_selection = [ - "redis", - "rabbit", -] @pytest.fixture def mock_environment( monkeypatch: pytest.MonkeyPatch, rabbit_service: RabbitSettings, - redis_service: RedisSettings, mock_environment: EnvVarsDict, mock_registry_service: AsyncMock, ) -> EnvVarsDict: @@ -62,19 +46,3 @@ async def rpc_client( rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], ) -> RabbitMQRPCClient: return await rabbitmq_rpc_client("client") - - -async def test_get_state(app: FastAPI, rpc_client: RabbitMQRPCClient): - usage = { - "some_path": DiskUsage( - total=ByteSize(0), used=ByteSize(0), free=ByteSize(0), used_percent=0 - ) - } - settings: ApplicationSettings = app.state.settings - - result = await disk_usage.update_disk_usage( - rpc_client, node_id=settings.DY_SIDECAR_NODE_ID, usage=usage - ) - assert result is None - - assert get_disk_usage_monitor(app)._usage_overwrite == usage # noqa: SLF001 diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__containers.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__containers.py new file mode 100644 index 00000000000..d40d5904bf5 --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__containers.py @@ -0,0 +1,79 @@ +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +import pytest +import yaml +from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.containers import DcokerComposeYamlStr +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import containers +from settings_library.redis import RedisSettings +from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings +from simcore_service_dynamic_sidecar.models.shared_store import ( + SharedStore, + get_shared_store, +) + +pytest_simcore_core_services_selection = [ + "redis", + "rabbit", +] + + +@pytest.fixture +def mock_environment( + redis_service: RedisSettings, mock_environment: EnvVarsDict +) -> EnvVarsDict: + return mock_environment + + +@pytest.fixture +def dynamic_sidecar_network_name() -> str: + return "entrypoint_container_network" + + +@pytest.fixture +def docker_compose_yaml(dynamic_sidecar_network_name: str) -> DcokerComposeYamlStr: + return yaml.dump( + { + "version": "3", + "services": { + "first-box": { + "image": "busybox:latest", + "networks": { + dynamic_sidecar_network_name: None, + }, + "labels": {"io.osparc.test-label": "mark-entrypoint"}, + }, + "second-box": {"image": "busybox:latest"}, + "egress": { + "image": "busybox:latest", + "networks": { + dynamic_sidecar_network_name: None, + }, + }, + }, + "networks": {dynamic_sidecar_network_name: None}, + } + ) + + +async def test_store_compose_spec( + app: FastAPI, + rpc_client: RabbitMQRPCClient, + docker_compose_yaml: DcokerComposeYamlStr, + ensure_external_volumes: None, +): + settings: ApplicationSettings = app.state.settings + + result = await containers.store_compose_spec( + rpc_client, + node_id=settings.DY_SIDECAR_NODE_ID, + docker_compose_yaml=docker_compose_yaml, + ) + assert result is None + + shared_store: SharedStore = get_shared_store(app) + assert shared_store.compose_spec is not None diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py new file mode 100644 index 00000000000..7ed592976e7 --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk.py @@ -0,0 +1,30 @@ +# pylint:disable=unused-argument + + +from fastapi import FastAPI +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import disk +from simcore_service_dynamic_sidecar.core.reserved_space import ( + _RESERVED_DISK_SPACE_NAME, +) +from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +async def test_free_reserved_disk_space( + cleanup_reserved_disk_space: None, app: FastAPI, rpc_client: RabbitMQRPCClient +): + assert _RESERVED_DISK_SPACE_NAME.exists() + + settings: ApplicationSettings = app.state.settings + + result = await disk.free_reserved_disk_space( + rpc_client, + node_id=settings.DY_SIDECAR_NODE_ID, + ) + assert result is None + + assert not _RESERVED_DISK_SPACE_NAME.exists() diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py new file mode 100644 index 00000000000..105d4a85ccc --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__disk_usage.py @@ -0,0 +1,44 @@ +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +import pytest +from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage +from pydantic import ByteSize +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import disk_usage +from settings_library.redis import RedisSettings +from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings +from simcore_service_dynamic_sidecar.modules.system_monitor._disk_usage import ( + get_disk_usage_monitor, +) + +pytest_simcore_core_services_selection = [ + "redis", + "rabbit", +] + + +@pytest.fixture +def mock_environment( + redis_service: RedisSettings, mock_environment: EnvVarsDict +) -> EnvVarsDict: + return mock_environment + + +async def test_get_state(app: FastAPI, rpc_client: RabbitMQRPCClient): + usage = { + "some_path": DiskUsage( + total=ByteSize(0), used=ByteSize(0), free=ByteSize(0), used_percent=0 + ) + } + settings: ApplicationSettings = app.state.settings + + result = await disk_usage.update_disk_usage( + rpc_client, node_id=settings.DY_SIDECAR_NODE_ID, usage=usage + ) + assert result is None + + assert get_disk_usage_monitor(app)._usage_overwrite == usage # noqa: SLF001 diff --git a/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py b/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py new file mode 100644 index 00000000000..e19b50916c1 --- /dev/null +++ b/services/dynamic-sidecar/tests/unit/api/rpc/test__volumes.py @@ -0,0 +1,74 @@ +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +from pathlib import Path + +import pytest +from fastapi import FastAPI +from models_library.sidecar_volumes import VolumeCategory, VolumeState, VolumeStatus +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq._errors import RPCServerError +from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar import volumes +from simcore_service_dynamic_sidecar.core.settings import ApplicationSettings +from simcore_service_dynamic_sidecar.models.shared_store import SharedStore + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +@pytest.mark.parametrize( + "volume_category, initial_expected_status", + [ + (VolumeCategory.STATES, VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED), + (VolumeCategory.OUTPUTS, VolumeStatus.CONTENT_NEEDS_TO_BE_SAVED), + (VolumeCategory.INPUTS, VolumeStatus.CONTENT_NO_SAVE_REQUIRED), + (VolumeCategory.SHARED_STORE, VolumeStatus.CONTENT_NO_SAVE_REQUIRED), + ], +) +async def test_volumes_state_saved_ok( + ensure_shared_store_dir: Path, + app: FastAPI, + rpc_client: RabbitMQRPCClient, + volume_category: VolumeCategory, + initial_expected_status: VolumeStatus, +): + shared_store: SharedStore = app.state.shared_store + settings: ApplicationSettings = app.state.settings + + # check that initial status is as expected + assert shared_store.volume_states[volume_category] == VolumeState( + status=initial_expected_status + ) + + await volumes.save_volume_state( + rpc_client, + node_id=settings.DY_SIDECAR_NODE_ID, + status=VolumeStatus.CONTENT_WAS_SAVED, + category=volume_category, + ) + + # check that content was saved + assert shared_store.volume_states[volume_category] == VolumeState( + status=VolumeStatus.CONTENT_WAS_SAVED + ) + + +@pytest.mark.parametrize("invalid_volume_category", ["outputs", "outputS"]) +async def test_volumes_state_saved_error( + ensure_shared_store_dir: Path, + app: FastAPI, + rpc_client: RabbitMQRPCClient, + invalid_volume_category: VolumeCategory, +): + + settings: ApplicationSettings = app.state.settings + + with pytest.raises(RPCServerError, match="ValidationError"): + await volumes.save_volume_state( + rpc_client, + node_id=settings.DY_SIDECAR_NODE_ID, + status=VolumeStatus.CONTENT_WAS_SAVED, + category=invalid_volume_category, + ) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py index 9c050ae8a0e..9b1d987203f 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py @@ -41,6 +41,7 @@ from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, ContainersCreate, + DcokerComposeYamlStr, ) from simcore_service_dynamic_sidecar.models.shared_store import SharedStore from simcore_service_dynamic_sidecar.modules.inputs import enable_inputs_pulling @@ -150,7 +151,7 @@ def dynamic_sidecar_network_name() -> str: }, ] ) -def compose_spec(request: pytest.FixtureRequest) -> str: +def compose_spec(request: pytest.FixtureRequest) -> DcokerComposeYamlStr: spec_dict: dict[str, Any] = request.param # type: ignore return json.dumps(spec_dict) @@ -282,7 +283,7 @@ async def _get_task_id_pull_user_servcices_docker_images( async def _get_task_id_create_service_containers( httpx_async_client: AsyncClient, - compose_spec: str, + compose_spec: DcokerComposeYamlStr, mock_metrics_params: CreateServiceMetricsAdditionalParams, *args, **kwargs, diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py index 89bb741fc30..893f328360e 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py @@ -26,6 +26,7 @@ from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, ContainersCreate, + DcokerComposeYamlStr, ) from simcore_service_dynamic_sidecar.modules.prometheus_metrics import ( _USER_SERVICES_NOT_STARTED, @@ -85,7 +86,7 @@ def client( @pytest.fixture -def compose_spec() -> str: +def compose_spec() -> DcokerComposeYamlStr: return json.dumps( { "version": "3", @@ -101,7 +102,7 @@ def compose_spec() -> str: async def _get_task_id_create_service_containers( httpx_async_client: AsyncClient, - compose_spec: str, + compose_spec: DcokerComposeYamlStr, mock_metrics_params: CreateServiceMetricsAdditionalParams, ) -> TaskId: ctontainers_compose_spec = ContainersComposeSpec( diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py index 662b7033b88..3db4573a731 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py @@ -43,6 +43,7 @@ from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, ContainersCreate, + DcokerComposeYamlStr, ) from simcore_service_dynamic_sidecar.models.shared_store import SharedStore from tenacity import AsyncRetrying, TryAgain @@ -73,7 +74,7 @@ def raw_compose_spec(container_names: list[str]) -> dict[str, Any]: @pytest.fixture -def compose_spec(raw_compose_spec: dict[str, Any]) -> str: +def compose_spec(raw_compose_spec: dict[str, Any]) -> DcokerComposeYamlStr: return json.dumps(raw_compose_spec) @@ -145,7 +146,7 @@ def mock_user_services_fail_to_stop(mocker: MockerFixture) -> None: async def _get_task_id_create_service_containers( httpx_async_client: AsyncClient, - compose_spec: str, + compose_spec: DcokerComposeYamlStr, mock_metrics_params: CreateServiceMetricsAdditionalParams, ) -> TaskId: containers_compose_spec = ContainersComposeSpec(