✨ Add storage dummy rpc endpoints for zipping

api/specs/web-server/_storage.py

@@ -7,7 +7,9 @@
 from typing import TypeAlias
 
 from fastapi import APIRouter, Query, status
-from models_library.api_schemas_storage import (
+from models_library.generics import Envelope
+from models_library.projects_nodes_io import LocationID
+from models_library.storage_schemas import (
     FileMetaDataGet,
     FileUploadCompleteFutureResponse,
     FileUploadCompleteResponse,
@@ -16,8 +18,6 @@
     LinkType,
     PresignedLink,
 )
-from models_library.generics import Envelope
-from models_library.projects_nodes_io import LocationID
 from pydantic import AnyUrl, ByteSize
 from simcore_service_webserver._meta import API_VTAG
 from simcore_service_webserver.storage.schemas import DatasetMetaData, FileMetaData

packages/aws-library/src/aws_library/s3/_client.py

@@ -13,8 +13,8 @@
 from boto3.s3.transfer import TransferConfig
 from botocore import exceptions as botocore_exc
 from botocore.client import Config
-from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart
 from models_library.basic_types import SHA256Str
+from models_library.storage_schemas import ETag, S3BucketName, UploadedPart
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.logging_utils import log_catch, log_context
 from servicelib.utils import limited_gather

packages/aws-library/src/aws_library/s3/_models.py

@@ -2,8 +2,8 @@
 from dataclasses import dataclass
 from typing import TypeAlias
 
-from models_library.api_schemas_storage import ETag
 from models_library.basic_types import SHA256Str
+from models_library.storage_schemas import ETag
 from pydantic import AnyUrl, BaseModel, ByteSize
 from types_aiobotocore_s3.type_defs import HeadObjectOutputTypeDef, ObjectTypeDef
 

packages/aws-library/tests/test_s3_client.py

@@ -29,8 +29,8 @@
 )
 from aws_library.s3._models import MultiPartUploadLinks
 from faker import Faker
-from models_library.api_schemas_storage import S3BucketName, UploadedPart
 from models_library.basic_types import SHA256Str
+from models_library.storage_schemas import S3BucketName, UploadedPart
 from moto.server import ThreadedMotoServer
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from pytest_benchmark.plugin import BenchmarkFixture

packages/models-library/src/models_library/api_schemas_storage/__init__.py

@@ -0,0 +1,9 @@
+from typing import Final
+
+from pydantic import TypeAdapter
+
+from ..rabbitmq_basic_types import RPCNamespace
+
+STORAGE_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter(RPCNamespace).validate_python(
+    "storage"
+)

packages/models-library/src/models_library/api_schemas_storage/zipping_tasks.py

@@ -0,0 +1,14 @@
+# pylint: disable=R6301
+from pathlib import Path
+
+from models_library.api_schemas_long_running_tasks.base import TaskId
+from pydantic import BaseModel, Field
+
+
+class ZipTaskStartInput(BaseModel):
+    paths: list[Path] = Field(..., min_length=1)
+
+
+class ZipTaskAbortOutput(BaseModel):
+    result: bool
+    task_id: TaskId

packages/pytest-simcore/src/pytest_simcore/helpers/s3.py

@@ -8,7 +8,7 @@
 import orjson
 from aws_library.s3 import MultiPartUploadLinks
 from fastapi import status
-from models_library.api_schemas_storage import ETag, FileUploadSchema, UploadedPart
+from models_library.storage_schemas import ETag, FileUploadSchema, UploadedPart
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.utils import limited_as_completed, logged_gather
 from types_aiobotocore_s3 import S3Client

packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py

@@ -12,7 +12,10 @@
 from aioresponses.core import CallbackResult
 from faker import Faker
 from models_library.api_schemas_directorv2.comp_tasks import ComputationGet
-from models_library.api_schemas_storage import (
+from models_library.generics import Envelope
+from models_library.projects_pipeline import ComputationTask
+from models_library.projects_state import RunningState
+from models_library.storage_schemas import (
     FileMetaDataGet,
     FileUploadCompleteFutureResponse,
     FileUploadCompleteResponse,
@@ -22,9 +25,6 @@
     LinkType,
     PresignedLink,
 )
-from models_library.generics import Envelope
-from models_library.projects_pipeline import ComputationTask
-from models_library.projects_state import RunningState
 from models_library.utils.fastapi_encoders import jsonable_encoder
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.aiohttp import status

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/storage/zipping.py

@@ -0,0 +1,73 @@
+from typing import Final
+
+from models_library.api_schemas_long_running_tasks.base import TaskId
+from models_library.api_schemas_long_running_tasks.tasks import (
+    TaskGet,
+    TaskResult,
+    TaskStatus,
+)
+from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE
+from models_library.api_schemas_storage.zipping_tasks import (
+    ZipTaskAbortOutput,
+    ZipTaskStartInput,
+)
+from models_library.rabbitmq_basic_types import RPCMethodName
+from pydantic import NonNegativeInt, TypeAdapter
+
+from ... import RabbitMQRPCClient
+
+_DEFAULT_TIMEOUT_S: Final[NonNegativeInt] = 30
+
+_RPC_METHOD_NAME_ADAPTER = TypeAdapter(RPCMethodName)
+
+
+async def start_zipping(
+    rabbitmq_rpc_client: RabbitMQRPCClient, *, paths: ZipTaskStartInput
+) -> TaskGet:
+    result = await rabbitmq_rpc_client.request(
+        STORAGE_RPC_NAMESPACE,
+        _RPC_METHOD_NAME_ADAPTER.validate_python("start_zipping"),
+        paths=paths,
+        timeout_s=_DEFAULT_TIMEOUT_S,
+    )
+    assert isinstance(result, TaskGet)
+    return result
+
+
+async def abort_zipping(
+    rabbitmq_rpc_client: RabbitMQRPCClient, *, task_id: TaskId
+) -> ZipTaskAbortOutput:
+    result = await rabbitmq_rpc_client.request(
+        STORAGE_RPC_NAMESPACE,
+        _RPC_METHOD_NAME_ADAPTER.validate_python("abort_zipping"),
+        task_id=task_id,
+        timeout_s=_DEFAULT_TIMEOUT_S,
+    )
+    assert isinstance(result, ZipTaskAbortOutput)
+    return result
+
+
+async def get_zipping_status(
+    rabbitmq_rpc_client: RabbitMQRPCClient, *, task_id: TaskId
+) -> TaskStatus:
+    result = await rabbitmq_rpc_client.request(
+        STORAGE_RPC_NAMESPACE,
+        _RPC_METHOD_NAME_ADAPTER.validate_python("get_zipping_status"),
+        task_id=task_id,
+        timeout_s=_DEFAULT_TIMEOUT_S,
+    )
+    assert isinstance(result, TaskStatus)
+    return result
+
+
+async def get_zipping_result(
+    rabbitmq_rpc_client: RabbitMQRPCClient, *, task_id: TaskId
+) -> TaskResult:
+    result = await rabbitmq_rpc_client.request(
+        STORAGE_RPC_NAMESPACE,
+        _RPC_METHOD_NAME_ADAPTER.validate_python("get_zipping_result"),
+        task_id=task_id,
+        timeout_s=_DEFAULT_TIMEOUT_S,
+    )
+    assert isinstance(result, TaskResult)
+    return result

packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py

@@ -2,16 +2,16 @@
 from typing import cast
 
 from aiohttp import ClientError, ClientSession
-from models_library.api_schemas_storage import (
+from models_library.generics import Envelope
+from models_library.projects_nodes_io import LocationID, LocationName
+from models_library.storage_schemas import (
     ETag,
     FileUploadCompleteFutureResponse,
     FileUploadCompleteResponse,
     FileUploadCompleteState,
     FileUploadCompletionBody,
     UploadedPart,
 )
-from models_library.generics import Envelope
-from models_library.projects_nodes_io import LocationID, LocationName
 from models_library.users import UserID
 from models_library.utils.fastapi_encoders import jsonable_encoder
 from pydantic import AnyUrl, TypeAdapter

packages/simcore-sdk/src/simcore_sdk/node_ports_common/file_io_utils.py

@@ -17,8 +17,8 @@
     ClientSession,
     RequestInfo,
 )
-from models_library.api_schemas_storage import ETag, FileUploadSchema, UploadedPart
 from models_library.basic_types import IDStr, SHA256Str
+from models_library.storage_schemas import ETag, FileUploadSchema, UploadedPart
 from multidict import MultiMapping
 from pydantic import AnyUrl, NonNegativeInt
 from servicelib.aiohttp import status

packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py

@@ -5,15 +5,15 @@
 
 import aiofiles
 from aiohttp import ClientSession
-from models_library.api_schemas_storage import (
+from models_library.basic_types import IDStr, SHA256Str
+from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID
+from models_library.storage_schemas import (
     ETag,
     FileMetaDataGet,
     FileUploadSchema,
     LinkType,
     UploadedPart,
 )
-from models_library.basic_types import IDStr, SHA256Str
-from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID
 from models_library.users import UserID
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.file_utils import create_sha256_checksum

packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py

@@ -10,16 +10,16 @@
 from aiohttp import ClientResponse, ClientSession
 from aiohttp import client as aiohttp_client_module
 from aiohttp.client_exceptions import ClientConnectionError, ClientResponseError
-from models_library.api_schemas_storage import (
+from models_library.basic_types import SHA256Str
+from models_library.generics import Envelope
+from models_library.projects_nodes_io import LocationID, StorageFileID
+from models_library.storage_schemas import (
     FileLocationArray,
     FileMetaDataGet,
     FileUploadSchema,
     LinkType,
     PresignedLink,
 )
-from models_library.basic_types import SHA256Str
-from models_library.generics import Envelope
-from models_library.projects_nodes_io import LocationID, StorageFileID
 from models_library.users import UserID
 from pydantic import ByteSize
 from pydantic.networks import AnyUrl

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py

@@ -1,8 +1,8 @@
 import logging
 
-from models_library.api_schemas_storage import LinkType as FileLinkType
 from models_library.projects import ProjectIDStr
 from models_library.projects_nodes_io import NodeIDStr
+from models_library.storage_schemas import LinkType as FileLinkType
 from models_library.users import UserID
 from settings_library.aws_s3_cli import AwsS3CliSettings
 from settings_library.r_clone import RCloneSettings

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py

@@ -6,11 +6,11 @@
 from pathlib import Path
 from typing import Any
 
-from models_library.api_schemas_storage import LinkType
 from models_library.basic_types import IDStr
 from models_library.projects import ProjectIDStr
 from models_library.projects_nodes_io import NodeIDStr
 from models_library.services_types import ServicePortKey
+from models_library.storage_schemas import LinkType
 from models_library.users import UserID
 from pydantic import BaseModel, ConfigDict, Field, ValidationError
 from pydantic_core import InitErrorDetails

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py

@@ -6,10 +6,10 @@
 from pprint import pformat
 from typing import Any
 
-from models_library.api_schemas_storage import LinkType
 from models_library.basic_types import IDStr
 from models_library.services_io import BaseServiceIOModel
 from models_library.services_types import ServicePortKey
+from models_library.storage_schemas import LinkType
 from pydantic import (
     AnyUrl,
     ConfigDict,

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py

@@ -4,9 +4,9 @@
 from pathlib import Path
 from typing import Any
 
-from models_library.api_schemas_storage import FileUploadSchema, LinkType
 from models_library.basic_types import IDStr, SHA256Str
 from models_library.services_types import FileName, ServicePortKey
+from models_library.storage_schemas import FileUploadSchema, LinkType
 from models_library.users import UserID
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.progress_bar import ProgressBarData

packages/simcore-sdk/tests/integration/conftest.py

@@ -13,9 +13,9 @@
 import pytest
 import sqlalchemy as sa
 from aiohttp import ClientSession
-from models_library.api_schemas_storage import FileUploadSchema
 from models_library.generics import Envelope
 from models_library.projects_nodes_io import LocationID, NodeIDStr, SimcoreS3FileID
+from models_library.storage_schemas import FileUploadSchema
 from models_library.users import UserID
 from pydantic import TypeAdapter
 from pytest_simcore.helpers.faker_factories import random_project, random_user

packages/simcore-sdk/tests/unit/test_node_ports_common_file_io_utils.py

@@ -13,12 +13,12 @@
 from aiohttp import ClientResponse, ClientSession, TCPConnector
 from aioresponses import aioresponses
 from faker import Faker
-from models_library.api_schemas_storage import (
+from models_library.basic_types import IDStr
+from models_library.storage_schemas import (
     FileUploadLinks,
     FileUploadSchema,
     UploadedPart,
 )
-from models_library.basic_types import IDStr
 from moto.server import ThreadedMotoServer
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from pytest_mock import MockerFixture

packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py

@@ -21,8 +21,8 @@
 from aiohttp.client import ClientSession
 from aioresponses import aioresponses as AioResponsesMock
 from faker import Faker
-from models_library.api_schemas_storage import FileMetaDataGet
 from models_library.projects_nodes_io import LocationID
+from models_library.storage_schemas import FileMetaDataGet
 from pydantic import TypeAdapter, ValidationError
 from pytest_mock.plugin import MockerFixture
 from servicelib.progress_bar import ProgressBarData

packages/simcore-sdk/tests/unit/test_storage_client.py

@@ -12,13 +12,13 @@
 import pytest
 from aioresponses import aioresponses as AioResponsesMock
 from faker import Faker
-from models_library.api_schemas_storage import (
+from models_library.projects_nodes_io import SimcoreS3FileID
+from models_library.storage_schemas import (
     FileLocationArray,
     FileMetaDataGet,
     FileUploadSchema,
     LocationID,
 )
-from models_library.projects_nodes_io import SimcoreS3FileID
 from models_library.users import UserID
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict

services/api-server/src/simcore_service_api_server/api/routes/files.py

@@ -10,8 +10,8 @@
 from fastapi import Header, Request, UploadFile, status
 from fastapi.exceptions import HTTPException
 from fastapi_pagination.api import create_page
-from models_library.api_schemas_storage import ETag, FileUploadCompletionBody, LinkType
 from models_library.basic_types import SHA256Str
+from models_library.storage_schemas import ETag, FileUploadCompletionBody, LinkType
 from pydantic import AnyUrl, ByteSize, PositiveInt, TypeAdapter, ValidationError
 from servicelib.fastapi.requests_decorators import cancel_on_disconnect
 from simcore_sdk.node_ports_common.constants import SIMCORE_LOCATION

services/api-server/src/simcore_service_api_server/models/schemas/files.py

@@ -7,9 +7,9 @@
 
 import aiofiles
 from fastapi import UploadFile
-from models_library.api_schemas_storage import ETag
 from models_library.basic_types import SHA256Str
 from models_library.projects_nodes_io import StorageFileID
+from models_library.storage_schemas import ETag
 from pydantic import (
     AnyHttpUrl,
     BaseModel,

services/api-server/src/simcore_service_api_server/services_http/storage.py

@@ -8,11 +8,11 @@
 
 from fastapi import FastAPI
 from fastapi.encoders import jsonable_encoder
-from models_library.api_schemas_storage import FileMetaDataArray
-from models_library.api_schemas_storage import FileMetaDataGet as StorageFileMetaData
-from models_library.api_schemas_storage import FileUploadSchema, PresignedLink
 from models_library.basic_types import SHA256Str
 from models_library.generics import Envelope
+from models_library.storage_schemas import FileMetaDataArray
+from models_library.storage_schemas import FileMetaDataGet as StorageFileMetaData
+from models_library.storage_schemas import FileUploadSchema, PresignedLink
 from pydantic import AnyUrl, PositiveInt
 from settings_library.tracing import TracingSettings
 from starlette.datastructures import URL

services/api-server/tests/unit/conftest.py

@@ -27,12 +27,12 @@
     TaskProgress,
     TaskStatus,
 )
-from models_library.api_schemas_storage import HealthCheck
 from models_library.api_schemas_webserver.projects import ProjectGet
 from models_library.app_diagnostics import AppStatusCheck
 from models_library.generics import Envelope
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import BaseFileLink, SimcoreS3FileID
+from models_library.storage_schemas import HealthCheck
 from models_library.users import UserID
 from moto.server import ThreadedMotoServer
 from packaging.version import Version