✨ Add exporter code to storage

packages/models-library/src/models_library/basic_regex.py

@@ -4,6 +4,7 @@
 
 SEE tests_basic_regex.py for examples
 """
+
 # TODO: for every pattern we should have a formatter function
 # NOTE: some sites to manualy check ideas
 #   https://regex101.com/
@@ -45,7 +46,9 @@
 )
 
 # Storage basic file ID
-SIMCORE_S3_FILE_ID_RE = rf"^(api|({UUID_RE_BASE}))\/({UUID_RE_BASE})\/(.+)$"
+SIMCORE_S3_FILE_ID_RE = (
+    rf"^(api\/{UUID_RE_BASE}|exports\/\d+|{UUID_RE_BASE}\/{UUID_RE_BASE})\/(.+)$"
+)
 SIMCORE_S3_DIRECTORY_ID_RE = rf"^({UUID_RE_BASE})\/({UUID_RE_BASE})\/(.+)\/$"
 
 # S3 - AWS bucket names [https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html]

packages/models-library/src/models_library/rabbitmq_messages.py

@@ -95,9 +95,9 @@ class ProgressType(StrAutoEnum):
 
 
 class ProgressMessageMixin(RabbitMessageBase):
-    channel_name: Literal[
+    channel_name: Literal["simcore.services.progress.v2"] = (
         "simcore.services.progress.v2"
-    ] = "simcore.services.progress.v2"
+    )
     progress_type: ProgressType = (
         ProgressType.COMPUTATION_RUNNING
     )  # NOTE: backwards compatible
@@ -118,9 +118,9 @@ def routing_key(self) -> str | None:
 
 
 class InstrumentationRabbitMessage(RabbitMessageBase, NodeMessageBase):
-    channel_name: Literal[
+    channel_name: Literal["simcore.services.instrumentation"] = (
         "simcore.services.instrumentation"
-    ] = "simcore.services.instrumentation"
+    )
     metrics: str
     service_uuid: NodeID
     service_type: str
@@ -210,9 +210,9 @@ def routing_key(self) -> str | None:
 
 
 class RabbitResourceTrackingStartedMessage(RabbitResourceTrackingBaseMessage):
-    message_type: Literal[
+    message_type: Literal[RabbitResourceTrackingMessageType.TRACKING_STARTED] = (
         RabbitResourceTrackingMessageType.TRACKING_STARTED
-    ] = RabbitResourceTrackingMessageType.TRACKING_STARTED
+    )
 
     wallet_id: WalletID | None
     wallet_name: str | None
@@ -250,9 +250,9 @@ class RabbitResourceTrackingStartedMessage(RabbitResourceTrackingBaseMessage):
 
 
 class RabbitResourceTrackingHeartbeatMessage(RabbitResourceTrackingBaseMessage):
-    message_type: Literal[
+    message_type: Literal[RabbitResourceTrackingMessageType.TRACKING_HEARTBEAT] = (
         RabbitResourceTrackingMessageType.TRACKING_HEARTBEAT
-    ] = RabbitResourceTrackingMessageType.TRACKING_HEARTBEAT
+    )
 
 
 class SimcorePlatformStatus(StrAutoEnum):
@@ -261,9 +261,9 @@ class SimcorePlatformStatus(StrAutoEnum):
 
 
 class RabbitResourceTrackingStoppedMessage(RabbitResourceTrackingBaseMessage):
-    message_type: Literal[
+    message_type: Literal[RabbitResourceTrackingMessageType.TRACKING_STOPPED] = (
         RabbitResourceTrackingMessageType.TRACKING_STOPPED
-    ] = RabbitResourceTrackingMessageType.TRACKING_STOPPED
+    )
 
     simcore_platform_status: SimcorePlatformStatus = Field(
         ...,
@@ -297,9 +297,9 @@ class CreditsLimit(IntEnum):
 
 
 class WalletCreditsLimitReachedMessage(RabbitMessageBase):
-    channel_name: Literal[
+    channel_name: Literal["io.simcore.service.wallets-credit-limit-reached"] = (
         "io.simcore.service.wallets-credit-limit-reached"
-    ] = "io.simcore.service.wallets-credit-limit-reached"
+    )
     created_at: datetime.datetime = Field(
         default_factory=lambda: arrow.utcnow().datetime,
         description="message creation datetime",

packages/models-library/tests/test_project_nodes_io.py

@@ -11,7 +11,9 @@
     DatCoreFileLink,
     SimCoreFileLink,
     SimcoreS3DirectoryID,
+    SimcoreS3FileID,
 )
+from models_library.users import UserID
 from pydantic import TypeAdapter, ValidationError
 
 
@@ -180,3 +182,21 @@ def test_simcore_s3_directory_get_parent():
         SimcoreS3DirectoryID._get_parent(  # noqa SLF001
             "/hello/object/", parent_index=4
         )
+
+
+USER_ID_0: UserID = 0
+
+
+@pytest.mark.parametrize(
+    "object_key",
+    [
+        f"api/{UUID_0}/some-random-file.png",
+        f"exports/{USER_ID_0}/some-random-file.png",
+        f"{UUID_0}/{UUID_0}/some-random-file.png",
+        f"api/{UUID_0}/some-path/some-random-file.png",
+        f"exports/{USER_ID_0}/some-path/some-random-file.png",
+        f"{UUID_0}/{UUID_0}/some-path/some-random-file.png",
+    ],
+)
+def test_simcore_s3_file_id_accepted_patterns(object_key: str):
+    assert str(TypeAdapter(SimcoreS3FileID).validate_python(object_key)) == object_key

packages/models-library/tests/test_rabbit_messages.py

@@ -1,5 +1,3 @@
-from typing import Union
-
 import pytest
 from faker import Faker
 from models_library.progress_bar import ProgressReport
@@ -41,6 +39,6 @@
 )
 async def test_raw_message_parsing(raw_data: str, class_type: type):
     result = TypeAdapter(
-        Union[ProgressRabbitMessageNode, ProgressRabbitMessageProject]
+        ProgressRabbitMessageNode | ProgressRabbitMessageProject
     ).validate_json(raw_data)
-    assert type(result) == class_type
+    assert type(result) is class_type

packages/notifications-library/tests/email/conftest.py

@@ -13,13 +13,7 @@ def app_environment(
     env_devel_dict: EnvVarsDict,
     external_envfile_dict: EnvVarsDict,
 ) -> EnvVarsDict:
-    return setenvs_from_dict(
-        monkeypatch,
-        {
-            **env_devel_dict,
-            **external_envfile_dict,
-        },
-    )
+    return setenvs_from_dict(monkeypatch, {**env_devel_dict, **external_envfile_dict})
 
 
 @pytest.fixture

packages/service-library/src/servicelib/archiving_utils/_tdqm_utils.py

@@ -1,8 +1,7 @@
 from typing import Final
 
-from pydantic import ByteSize, NonNegativeFloat, NonNegativeInt
+from pydantic import ByteSize, NonNegativeInt
 
-_UNIT_MULTIPLIER: Final[NonNegativeFloat] = 1024.0
 TQDM_FILE_OPTIONS: Final[dict] = {
     "unit": "byte",
     "unit_scale": True,

services/storage/openapi.json

@@ -332,7 +332,7 @@
               "anyOf": [
                 {
                   "type": "string",
-                  "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                  "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
                 },
                 {
                   "type": "string",
@@ -430,7 +430,7 @@
               "anyOf": [
                 {
                   "type": "string",
-                  "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                  "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
                 },
                 {
                   "type": "string",
@@ -509,7 +509,7 @@
               "anyOf": [
                 {
                   "type": "string",
-                  "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                  "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
                 },
                 {
                   "type": "string",
@@ -643,7 +643,7 @@
               "anyOf": [
                 {
                   "type": "string",
-                  "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                  "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
                 },
                 {
                   "type": "string",
@@ -707,7 +707,7 @@
               "anyOf": [
                 {
                   "type": "string",
-                  "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                  "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
                 },
                 {
                   "type": "string",
@@ -771,7 +771,7 @@
               "anyOf": [
                 {
                   "type": "string",
-                  "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                  "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
                 },
                 {
                   "type": "string",
@@ -852,7 +852,7 @@
               "anyOf": [
                 {
                   "type": "string",
-                  "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                  "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
                 },
                 {
                   "type": "string",
@@ -923,7 +923,7 @@
               "anyOf": [
                 {
                   "type": "string",
-                  "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                  "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
                 },
                 {
                   "type": "string",
@@ -2078,7 +2078,7 @@
             "anyOf": [
               {
                 "type": "string",
-                "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
+                "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$"
               },
               {
                 "type": "string",
@@ -2629,7 +2629,7 @@
         "properties": {
           "link_id": {
             "type": "string",
-            "pattern": "^(api|([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}))\\/([0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$",
+            "pattern": "^(api\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}|exports\\/\\d+|[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}\\/[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12})\\/(.+)$",
             "title": "Link Id"
           }
         },

services/storage/requirements/_test.in

@@ -29,4 +29,5 @@ python-dotenv
 respx
 simcore-service-storage-sdk @ git+https://github.com/ITISFoundation/osparc-simcore.git@cfdf4f86d844ebb362f4f39e9c6571d561b72897#subdirectory=services/storage/client-sdk/python # to test backwards compatibility against deprecated client-sdk (used still in old versions of simcore-sdk)
 sqlalchemy[mypy] # adds Mypy / Pep-484 Support for ORM Mappings SEE https://docs.sqlalchemy.org/en/20/orm/extensions/mypy.html
+types_tqdm
 types-aiofiles

services/storage/requirements/_test.txt

@@ -402,6 +402,10 @@ termcolor==2.5.0
     # via pytest-sugar
 types-aiofiles==24.1.0.20241221
     # via -r requirements/_test.in
+types-requests==2.32.0.20241016
+    # via types-tqdm
+types-tqdm==4.67.0.20241221
+    # via -r requirements/_test.in
 typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
@@ -429,6 +433,7 @@ urllib3==2.3.0
     #   requests
     #   responses
     #   simcore-service-storage-sdk
+    #   types-requests
 vine==5.1.0
     # via
     #   -c requirements/_base.txt

services/storage/src/simcore_service_storage/api/_worker_tasks/_data_export.py

@@ -0,0 +1,40 @@
+import logging
+from typing import cast
+
+from celery import Task  # type: ignore[import-untyped]
+from models_library.progress_bar import ProgressReport
+from models_library.projects_nodes_io import StorageFileID
+from models_library.users import UserID
+
+from ...dsm import get_dsm_provider
+from ...modules.celery.utils import get_celery_worker_client, get_fastapi_app
+from ...simcore_s3_dsm import SimcoreS3DataManager
+from ._progress_utils import get_tqdm_progress, set_tqdm_absolute_progress
+
+_logger = logging.getLogger(__name__)
+
+
+async def data_export(
+    task: Task,
+    *,
+    user_id: UserID,
+    paths_to_export: list[StorageFileID],
+) -> StorageFileID:
+    _logger.info("Exporting (for user='%s') selection: %s", user_id, paths_to_export)
+
+    dsm = cast(
+        SimcoreS3DataManager,
+        get_dsm_provider(get_fastapi_app(task.app)).get(
+            SimcoreS3DataManager.get_location_id()
+        ),
+    )
+
+    with get_tqdm_progress(total=1, description=f"{task.name}") as pbar:
+
+        async def _progress_cb(report: ProgressReport) -> None:
+            set_tqdm_absolute_progress(pbar, report)
+            await get_celery_worker_client(task.app).set_task_progress(task, report)
+
+        return await dsm.create_s3_export(
+            user_id, paths_to_export, progress_cb=_progress_cb
+        )

services/storage/src/simcore_service_storage/api/_worker_tasks/_progress_utils.py

@@ -0,0 +1,24 @@
+from typing import Final
+
+from models_library.progress_bar import ProgressReport
+from pydantic import NonNegativeFloat
+from tqdm import tqdm
+
+TQDM_EXPORT_OPTIONS: Final[dict] = {
+    "unit": "",
+    "unit_scale": True,
+    "unit_divisor": 1024,
+    "colour": "yellow",
+    "miniters": 1,
+    "ncols": 100,
+}
+
+
+def get_tqdm_progress(total: NonNegativeFloat, *, description: str) -> tqdm:
+    return tqdm(**TQDM_EXPORT_OPTIONS, total=total, desc=description)
+
+
+def set_tqdm_absolute_progress(pbar: tqdm, report: ProgressReport) -> None:
+    """used when the progress does not come in chunk by chunk but as the total current value"""
+    pbar.n = report.actual_value
+    pbar.refresh()

services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py

@@ -5,7 +5,7 @@
 
 from ...modules.celery._celery_types import register_celery_types
 from ...modules.celery._task import define_task
-from ...modules.celery.tasks import export_data
+from ._data_export import data_export
 from ._paths import compute_path_size
 
 _logger = logging.getLogger(__name__)
@@ -18,5 +18,5 @@ def setup_worker_tasks(app: Celery) -> None:
         logging.INFO,
         msg="Storage setup Worker Tasks",
     ):
-        define_task(app, export_data)
+        define_task(app, data_export)
         define_task(app, compute_path_size)

services/storage/src/simcore_service_storage/api/rpc/_data_export.py

@@ -18,6 +18,7 @@
 from ...modules.celery import get_celery_client
 from ...modules.datcore_adapter.datcore_adapter_exceptions import DatcoreAdapterError
 from ...simcore_s3_dsm import SimcoreS3DataManager
+from .._worker_tasks._data_export import data_export
 
 router = RPCRouter()
 
@@ -54,9 +55,10 @@ async def start_data_export(
 
     try:
         task_uuid = await get_celery_client(app).send_task(
-            "export_data",
+            data_export.__name__,
             task_context=job_id_data.model_dump(),
-            files=data_export_start.file_and_folder_ids,  # ANE: adapt here your signature
+            user_id=job_id_data.user_id,
+            paths_to_export=data_export_start.file_and_folder_ids,
         )
     except CeleryError as exc:
         raise JobSchedulerError(exc=f"{exc}") from exc