From 9680d4f17ad35a39c2d56cfbbd85465271b809c7 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 22 Oct 2025 13:30:17 +0200 Subject: [PATCH 1/8] Add failing test case --- .../renku_data_services/search/test_db.py | 146 +++++++++++++++++- 1 file changed, 145 insertions(+), 1 deletion(-) diff --git a/test/components/renku_data_services/search/test_db.py b/test/components/renku_data_services/search/test_db.py index 1cc4d2284..1997f9db5 100644 --- a/test/components/renku_data_services/search/test_db.py +++ b/test/components/renku_data_services/search/test_db.py @@ -7,12 +7,13 @@ from ulid import ULID from renku_data_services.authz.models import Visibility +from renku_data_services.base_models import AuthenticatedAPIUser from renku_data_services.base_models.core import ( - AuthenticatedAPIUser, NamespacePath, NamespaceSlug, ProjectPath, ProjectSlug, + Slug, ) from renku_data_services.data_api.dependencies import DependencyManager from renku_data_services.data_connectors.models import CloudStorageCore, DataConnector, UnsavedDataConnector @@ -39,6 +40,149 @@ ) +@pytest.mark.asyncio +async def test_remove_group_removes_descendant_entities(app_manager_instance: DependencyManager): + run_migrations_for_app("common") + repo = SearchUpdatesRepo(app_manager_instance.config.db.async_session_maker) + user_repo = app_manager_instance.kc_user_repo + group_repo = app_manager_instance.group_repo + proj_repo = app_manager_instance.project_repo + dc_repo = app_manager_instance.data_connector_repo + + user_id = uuid.uuid4() + user = AuthenticatedAPIUser(id=str(user_id), access_token="abc") + + await user_repo.get_or_create_user(user, user.id) + group = await group_repo.insert_group(user, UnsavedGroup(slug="group3", name="Group 3")) + proj1 = await proj_repo.insert_project( + user, + UnsavedProject( + namespace=group.slug, + name="proj in group 1", + slug="proj-group-1", + visibility=Visibility.PUBLIC, + created_by=user.id, + ), + ) + dc_in_proj = await dc_repo.insert_namespaced_data_connector( + user, + UnsavedDataConnector( + name="dc 1", + slug="dc1", + visibility=Visibility.PUBLIC, + created_by=user.id, + namespace=proj1.path, + storage=CloudStorageCore( + storage_type="csc", configuration={}, source_path="", target_path="", readonly=True + ), + ), + ) + dc_in_group = await dc_repo.insert_namespaced_data_connector( + user, + UnsavedDataConnector( + name="dc 2", + slug="dc2", + visibility=Visibility.PUBLIC, + created_by=user.id, + namespace=group.path, + storage=CloudStorageCore( + storage_type="csc", configuration={}, source_path="", target_path="", readonly=True + ), + ), + ) + + updates = await repo.select_next(10) + assert len(updates) == 5 + await repo.mark_processed([e.id for e in updates]) + + # until here was preparation of the data. there is now a group, + # containing a project and a data connector. the project also + # contains a data connector + + deleted_group = await group_repo.delete_group(user, Slug.from_name(group.slug)) + assert deleted_group + assert deleted_group.id == group.id + + updates = await repo.select_next(10) + + assert len(updates) == 4 + del_group = next(g for g in updates if g.entity_id == group.id) + assert del_group.entity_type == "Group" + assert del_group.payload["deleted"] + + del_proj = next(p for p in updates if p.entity_id == proj1.id) + assert del_proj.entity_type == "Project" + assert del_proj.payload["deleted"] + + del_dc1 = next(d for d in updates if d.entity_id == dc_in_proj.id) + assert del_dc1.entity_type == "DataConnector" + assert del_dc1.payload["deleted"] + + del_dc2 = next(d for d in updates if d.entity_id == dc_in_group.id) + assert del_dc2.entity_type == "DataConnector" + assert del_dc2.payload["deleted"] + + +@pytest.mark.asyncio +async def test_remove_project_removes_data_connector(app_manager_instance: DependencyManager): + run_migrations_for_app("common") + repo = SearchUpdatesRepo(app_manager_instance.config.db.async_session_maker) + user_repo = app_manager_instance.kc_user_repo + proj_repo = app_manager_instance.project_repo + dc_repo = app_manager_instance.data_connector_repo + + user_id = uuid.uuid4() + user = AuthenticatedAPIUser(id=str(user_id), access_token="abc", first_name="Huhu") + + u = await user_repo.get_or_create_user(user, user.id) + assert u + proj1 = await proj_repo.insert_project( + user, + UnsavedProject( + namespace=u.namespace.path.first.value, + name=f"proj of user {u.first_name}", + slug="proj-group-1", + visibility=Visibility.PUBLIC, + created_by=user.id, + ), + ) + dc = await dc_repo.insert_namespaced_data_connector( + user, + UnsavedDataConnector( + name="dc 1", + slug="dc1", + visibility=Visibility.PUBLIC, + created_by=user.id, + namespace=proj1.path, + storage=CloudStorageCore( + storage_type="csc", configuration={}, source_path="", target_path="", readonly=True + ), + ), + ) + + updates = await repo.select_next(10) + assert len(updates) == 3 + await repo.mark_processed([e.id for e in updates]) + + # until here was preparation of the data. there is now a group, + # containing a project and a data connector. the project also + # contains a data connector + + deleted_proj = await proj_repo.delete_project(user, proj1.id) + assert deleted_proj + assert deleted_proj.id == proj1.id + + updates = await repo.select_next(10) + assert len(updates) == 2 + del_proj = next(p for p in updates if p.entity_id == proj1.id) + assert del_proj.entity_type == "Project" + assert del_proj.payload["deleted"] + + del_dc = next(d for d in updates if d.entity_id == dc.id) + assert del_dc.entity_type == "DataConnector" + assert del_dc.payload["deleted"] + + @pytest.mark.asyncio async def test_dc_in_group_project(app_manager_instance: DependencyManager) -> None: run_migrations_for_app("common") From 977be4ac6843e9918890390129018f82bc5b95cb Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 22 Oct 2025 16:21:47 +0200 Subject: [PATCH 2/8] Update search with project data connectors if project is removed --- components/renku_data_services/project/db.py | 9 ++++++++- components/renku_data_services/project/models.py | 1 + components/renku_data_services/search/decorators.py | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index ce51df7b2..d92695b9e 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -401,13 +401,20 @@ async def delete_project( if project is None: return None + dcs = await session.execute( + select(ns_schemas.EntitySlugORM.data_connector_id) + .where(ns_schemas.EntitySlugORM.project_id == project_id) + .where(ns_schemas.EntitySlugORM.data_connector_id.is_not(None)) + ) + dcs = [e for e in dcs.scalars().all() if e] + await session.execute(delete(schemas.ProjectORM).where(schemas.ProjectORM.id == project_id)) await session.execute( delete(storage_schemas.CloudStorageORM).where(storage_schemas.CloudStorageORM.project_id == str(project_id)) ) - return models.DeletedProject(id=project.id) + return models.DeletedProject(id=project.id, data_connectors=dcs) async def get_project_permissions(self, user: base_models.APIUser, project_id: ULID) -> models.ProjectPermissions: """Get the permissions of the user on a given project.""" diff --git a/components/renku_data_services/project/models.py b/components/renku_data_services/project/models.py index da2e24a21..8699fb986 100644 --- a/components/renku_data_services/project/models.py +++ b/components/renku_data_services/project/models.py @@ -90,6 +90,7 @@ class DeletedProject: """Indicates that a project was deleted.""" id: ULID + data_connectors: list[ULID] @dataclass diff --git a/components/renku_data_services/search/decorators.py b/components/renku_data_services/search/decorators.py index c8a384988..57b3ff496 100644 --- a/components/renku_data_services/search/decorators.py +++ b/components/renku_data_services/search/decorators.py @@ -63,7 +63,10 @@ async def func_wrapper(self: _WithSearchUpdateRepo, *args: _P.args, **kwargs: _P case DeletedProject() as p: record = DeleteDoc.project(p.id) + dcs = [DeleteDoc.data_connector(id) for id in p.data_connectors] await self.search_updates_repo.upsert(record) + for d in dcs: + await self.search_updates_repo.upsert(d) case UserInfo() as u: await self.search_updates_repo.upsert(u) From 2b6841a3bbf884f534ddea57cc7273d0f009757c Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 22 Oct 2025 16:30:26 +0200 Subject: [PATCH 3/8] Delete containing data connectors with a project When deleting a project, contained data connectors should be deleted as well. This is currently handled by a trigger in the db watching the `entity_slugs` table. --- components/renku_data_services/project/db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index d92695b9e..3a3365089 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -24,6 +24,7 @@ from renku_data_services.base_api.pagination import PaginationRequest from renku_data_services.base_models import RESET, ProjectPath, ProjectSlug from renku_data_services.base_models.core import Slug +from renku_data_services.data_connectors import orm as dc_schemas from renku_data_services.namespace import orm as ns_schemas from renku_data_services.namespace.db import GroupRepository from renku_data_services.project import apispec as project_apispec @@ -414,6 +415,9 @@ async def delete_project( delete(storage_schemas.CloudStorageORM).where(storage_schemas.CloudStorageORM.project_id == str(project_id)) ) + if dcs != []: + await session.execute(delete(dc_schemas.DataConnectorORM).where(dc_schemas.DataConnectorORM.id.in_(dcs))) + return models.DeletedProject(id=project.id, data_connectors=dcs) async def get_project_permissions(self, user: base_models.APIUser, project_id: ULID) -> models.ProjectPermissions: From 237127a5ae29d1a9282d96b29d1236ebd8de0ae5 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 23 Oct 2025 10:52:29 +0200 Subject: [PATCH 4/8] Delete containing data connectors and projects with a group Removes the trigger that did this before and implements it into the corresponding repositories --- ...db8_remove_cleanup_after_slug_deletion_.py | 42 ++++++++ .../renku_data_services/namespace/db.py | 33 +++++-- .../renku_data_services/namespace/models.py | 2 + components/renku_data_services/project/db.py | 4 +- .../renku_data_services/namespace/test_db.py | 96 +++++++++++++++++++ .../renku_data_services/project/test_db.py | 61 ++++++++++++ 6 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 components/renku_data_services/migrations/versions/42049656cdb8_remove_cleanup_after_slug_deletion_.py create mode 100644 test/components/renku_data_services/namespace/test_db.py create mode 100644 test/components/renku_data_services/project/test_db.py diff --git a/components/renku_data_services/migrations/versions/42049656cdb8_remove_cleanup_after_slug_deletion_.py b/components/renku_data_services/migrations/versions/42049656cdb8_remove_cleanup_after_slug_deletion_.py new file mode 100644 index 000000000..cd6bebf72 --- /dev/null +++ b/components/renku_data_services/migrations/versions/42049656cdb8_remove_cleanup_after_slug_deletion_.py @@ -0,0 +1,42 @@ +"""remove cleanup_after_slug_deletion trigger + +Removes the cleanup trigger added in revision 8413f10ef77f + +Revision ID: 42049656cdb8 +Revises: d437be68a4fb +Create Date: 2025-10-23 09:55:19.905709 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "42049656cdb8" +down_revision = "d437be68a4fb" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("DROP TRIGGER IF EXISTS cleanup_after_slug_deletion ON common.entity_slugs") + op.execute("DROP FUNCTION cleanup_after_slug_deletion") + + +def downgrade() -> None: + op.execute("""CREATE OR REPLACE FUNCTION cleanup_after_slug_deletion() +RETURNS TRIGGER AS +$$ +BEGIN + IF OLD.project_id IS NOT NULL AND OLD.data_connector_id IS NULL THEN + DELETE FROM projects.projects WHERE projects.id = OLD.project_id; + ELSIF old.data_connector_id IS NOT NULL THEN + DELETE FROM storage.data_connectors WHERE data_connectors.id = OLD.data_connector_id; + END IF; + RETURN OLD; +END; +$$ +LANGUAGE plpgsql;""") + op.execute("""CREATE OR REPLACE TRIGGER cleanup_after_slug_deletion +AFTER DELETE ON common.entity_slugs +FOR EACH ROW +EXECUTE FUNCTION cleanup_after_slug_deletion();""") diff --git a/components/renku_data_services/namespace/db.py b/components/renku_data_services/namespace/db.py index d496de6c7..282be2821 100644 --- a/components/renku_data_services/namespace/db.py +++ b/components/renku_data_services/namespace/db.py @@ -9,7 +9,7 @@ from datetime import UTC, datetime from typing import Any, overload -from sqlalchemy import Select, and_, delete, func, select, text +from sqlalchemy import Select, and_, delete, distinct, func, select, text from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, AsyncSessionTransaction from sqlalchemy.orm import joinedload, selectinload @@ -30,6 +30,7 @@ ProjectPath, Slug, ) +from renku_data_services.data_connectors import orm as dc_schemas from renku_data_services.data_connectors.models import DataConnector from renku_data_services.data_connectors.orm import DataConnectorORM from renku_data_services.namespace import models @@ -311,13 +312,33 @@ async def delete_group( message=f"You cannot delete a group by using an old group slug {slug.value}.", detail=f"The latest slug is {group.namespace.slug}, please use this for deletions.", ) - # NOTE: We have a stored procedure that gets triggered when a project slug is removed to remove the project. - # This is required because the slug has a foreign key pointing to the project, so when a project is removed - # the slug is removed but the converse is not true. The stored procedure in migration 89aa4573cfa9 has the - # trigger and procedure that does the cleanup when a slug is removed. + + dcs = await session.execute( + select(distinct(schemas.EntitySlugORM.data_connector_id)) + .join(schemas.NamespaceORM, schemas.NamespaceORM.id == schemas.EntitySlugORM.namespace_id) + .where(schemas.NamespaceORM.group_id == group.id) + .where(schemas.EntitySlugORM.data_connector_id.is_not(None)) + ) + dcs = [e for e in dcs.scalars().all() if e] + + projs = await session.execute( + select(distinct(schemas.EntitySlugORM.project_id)) + .join(schemas.NamespaceORM, schemas.NamespaceORM.id == schemas.EntitySlugORM.namespace_id) + .where(schemas.NamespaceORM.group_id == group.id) + .where(schemas.EntitySlugORM.project_id.is_not(None)) + ) + projs = [e for e in projs.scalars().all() if e] + stmt = delete(schemas.GroupORM).where(schemas.GroupORM.id == group.id) await session.execute(stmt) - return models.DeletedGroup(id=group.id) + + if projs != []: + await session.execute(delete(ProjectORM).where(ProjectORM.id.in_(projs))) + + if dcs != []: + await session.execute(delete(dc_schemas.DataConnectorORM).where(dc_schemas.DataConnectorORM.id.in_(dcs))) + + return models.DeletedGroup(id=group.id, data_connectors=dcs, projects=projs) @with_db_transaction async def delete_group_member( diff --git a/components/renku_data_services/namespace/models.py b/components/renku_data_services/namespace/models.py index b525d4071..fbf4e43d2 100644 --- a/components/renku_data_services/namespace/models.py +++ b/components/renku_data_services/namespace/models.py @@ -41,6 +41,8 @@ class DeletedGroup: """A group that was deleted from the DB.""" id: ULID + data_connectors: list[ULID] + projects: list[ULID] @dataclass diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index 3a3365089..31e260dc2 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -11,7 +11,7 @@ from typing import Concatenate, ParamSpec, TypeVar from cryptography.hazmat.primitives.asymmetric import rsa -from sqlalchemy import ColumnElement, Select, delete, func, or_, select, update +from sqlalchemy import ColumnElement, Select, delete, distinct, func, or_, select, update from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import undefer from sqlalchemy.sql.functions import coalesce @@ -403,7 +403,7 @@ async def delete_project( return None dcs = await session.execute( - select(ns_schemas.EntitySlugORM.data_connector_id) + select(distinct(ns_schemas.EntitySlugORM.data_connector_id)) .where(ns_schemas.EntitySlugORM.project_id == project_id) .where(ns_schemas.EntitySlugORM.data_connector_id.is_not(None)) ) diff --git a/test/components/renku_data_services/namespace/test_db.py b/test/components/renku_data_services/namespace/test_db.py new file mode 100644 index 000000000..62ed0e05d --- /dev/null +++ b/test/components/renku_data_services/namespace/test_db.py @@ -0,0 +1,96 @@ +"""Tests for the db module.""" + +import uuid + +import pytest + +from renku_data_services.authz.models import Visibility +from renku_data_services.base_models.core import AuthenticatedAPIUser +from renku_data_services.data_api.dependencies import DependencyManager +from renku_data_services.data_connectors.models import CloudStorageCore, UnsavedDataConnector +from renku_data_services.errors import MissingResourceError +from renku_data_services.migrations.core import run_migrations_for_app +from renku_data_services.namespace.models import UnsavedGroup +from renku_data_services.project.models import UnsavedProject + + +@pytest.mark.asyncio +async def test_remove_group_removes_containing_entities(app_manager_instance: DependencyManager) -> None: + run_migrations_for_app("common") + + user_repo = app_manager_instance.kc_user_repo + group_repo = app_manager_instance.group_repo + proj_repo = app_manager_instance.project_repo + dc_repo = app_manager_instance.data_connector_repo + + user = AuthenticatedAPIUser(id=str(uuid.uuid4()), access_token="abc", first_name="Huhu") + u = await user_repo.get_or_create_user(user, user.id) + assert u + + group = await group_repo.insert_group(user, UnsavedGroup(slug="grr1", name="Group Grr")) + + proju = await proj_repo.insert_project( + user, + UnsavedProject( + namespace=u.namespace.path.first.value, + name=f"proj of user {u.first_name}", + slug="proj-user", + visibility=Visibility.PUBLIC, + created_by=user.id, + ), + ) + proj1 = await proj_repo.insert_project( + user, + UnsavedProject( + namespace=group.slug, + name=f"proj of group {group.name}", + slug="proj-group-1", + visibility=Visibility.PUBLIC, + created_by=user.id, + ), + ) + dc_in_proj = await dc_repo.insert_namespaced_data_connector( + user, + UnsavedDataConnector( + name="dc 1", + slug="dc1", + visibility=Visibility.PUBLIC, + created_by=user.id, + namespace=proj1.path, + storage=CloudStorageCore( + storage_type="csc", configuration={}, source_path="", target_path="", readonly=True + ), + ), + ) + dc_in_group = await dc_repo.insert_namespaced_data_connector( + user, + UnsavedDataConnector( + name="dc 1", + slug="dc1", + visibility=Visibility.PUBLIC, + created_by=user.id, + namespace=group.path, + storage=CloudStorageCore( + storage_type="csc", configuration={}, source_path="", target_path="", readonly=True + ), + ), + ) + + deleted_group = await group_repo.delete_group(user, group.path.first) + assert deleted_group + assert deleted_group.id == group.id + assert deleted_group.data_connectors == [dc_in_proj.id, dc_in_group.id] + assert len(deleted_group.projects) == 1 + assert deleted_group.projects == [proj1.id] + + # this must still exist + await proj_repo.get_project(user, proju.id) + + with pytest.raises(MissingResourceError): + await group_repo.get_group(user, group.path.first) + + with pytest.raises(MissingResourceError): + await proj_repo.get_project(user, proj1.id) + + with pytest.raises(MissingResourceError): + await dc_repo.get_data_connector(user, dc_in_proj.id) diff --git a/test/components/renku_data_services/project/test_db.py b/test/components/renku_data_services/project/test_db.py new file mode 100644 index 000000000..9602a670b --- /dev/null +++ b/test/components/renku_data_services/project/test_db.py @@ -0,0 +1,61 @@ +"""Tests for the db module.""" + +import uuid + +import pytest + +from renku_data_services.authz.models import Visibility +from renku_data_services.base_models.core import AuthenticatedAPIUser +from renku_data_services.data_api.dependencies import DependencyManager +from renku_data_services.data_connectors.models import CloudStorageCore, UnsavedDataConnector +from renku_data_services.errors import MissingResourceError +from renku_data_services.migrations.core import run_migrations_for_app +from renku_data_services.project.models import UnsavedProject + + +@pytest.mark.asyncio +async def test_remove_project_removes_data_connector(app_manager_instance: DependencyManager) -> None: + run_migrations_for_app("common") + + user_repo = app_manager_instance.kc_user_repo + proj_repo = app_manager_instance.project_repo + dc_repo = app_manager_instance.data_connector_repo + + user = AuthenticatedAPIUser(id=str(uuid.uuid4()), access_token="abc", first_name="Huhu") + u = await user_repo.get_or_create_user(user, user.id) + assert u + + proj1 = await proj_repo.insert_project( + user, + UnsavedProject( + namespace=u.namespace.path.first.value, + name=f"proj of user {u.first_name}", + slug="proj-group-1", + visibility=Visibility.PUBLIC, + created_by=user.id, + ), + ) + dc_in_proj = await dc_repo.insert_namespaced_data_connector( + user, + UnsavedDataConnector( + name="dc 1", + slug="dc1", + visibility=Visibility.PUBLIC, + created_by=user.id, + namespace=proj1.path, + storage=CloudStorageCore( + storage_type="csc", configuration={}, source_path="", target_path="", readonly=True + ), + ), + ) + + deleted_proj = await proj_repo.delete_project(user, proj1.id) + assert deleted_proj + assert deleted_proj.id == proj1.id + assert deleted_proj.data_connectors == [dc_in_proj.id] + + with pytest.raises(MissingResourceError): + await proj_repo.get_project(user, proj1.id) + + with pytest.raises(MissingResourceError): + await dc_repo.get_data_connector(user, dc_in_proj.id) From a7b380cfaa357d444e4dec171d09d1344247b3da Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 30 Oct 2025 17:09:57 +0100 Subject: [PATCH 5/8] Update search with removed entities when a group is removed --- components/renku_data_services/search/decorators.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/renku_data_services/search/decorators.py b/components/renku_data_services/search/decorators.py index 57b3ff496..846a5cc62 100644 --- a/components/renku_data_services/search/decorators.py +++ b/components/renku_data_services/search/decorators.py @@ -83,7 +83,11 @@ async def func_wrapper(self: _WithSearchUpdateRepo, *args: _P.args, **kwargs: _P case DeletedGroup() as g: record = DeleteDoc.group(g.id) + dcs = [DeleteDoc.data_connector(id) for id in g.data_connectors] + prs = [DeleteDoc.project(id) for id in g.projects] await self.search_updates_repo.upsert(record) + for d in dcs + prs: + await self.search_updates_repo.upsert(d) case DataConnector() as dc: await self.search_updates_repo.upsert(dc) From 4631944b733a258d285db78326d522ae48f56765 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 23 Oct 2025 12:39:51 +0200 Subject: [PATCH 6/8] Remove obsolete tests --- .../data_api/test_namespaces.py | 104 ------------------ 1 file changed, 104 deletions(-) diff --git a/test/bases/renku_data_services/data_api/test_namespaces.py b/test/bases/renku_data_services/data_api/test_namespaces.py index 47f7866f9..a947d4e37 100644 --- a/test/bases/renku_data_services/data_api/test_namespaces.py +++ b/test/bases/renku_data_services/data_api/test_namespaces.py @@ -1,7 +1,6 @@ import contextlib import pytest -from sqlalchemy import select from sqlalchemy.exc import IntegrityError from renku_data_services.authz.models import Visibility @@ -24,7 +23,6 @@ ) from renku_data_services.errors.errors import ConflictError, MissingResourceError, ValidationError from renku_data_services.namespace.models import UnsavedGroup -from renku_data_services.namespace.orm import EntitySlugORM from renku_data_services.project.models import Project, ProjectPatch, UnsavedProject from renku_data_services.users.models import UserInfo @@ -404,108 +402,6 @@ async def test_listing_project_namespaces(sanic_client, user_headers) -> None: assert response.json[1]["path"] == "test1/proj2" -@pytest.mark.asyncio -async def test_stored_procedure_cleanup_after_project_slug_deletion( - create_project, - user_headers, - app_manager: DependencyManager, - sanic_client, - create_data_connector, -) -> None: - # We use stored procedures to remove a project when its slug is removed - proj = await create_project(name="test1") - proj_id = proj.get("id") - assert proj_id is not None - namespace = proj.get("namespace") - assert namespace is not None - proj_slug = proj.get("slug") - assert proj_slug is not None - _, response = await sanic_client.get(f"/api/data/namespaces/{namespace}", headers=user_headers) - assert response.status_code == 200 - dc = await create_data_connector(name="test-dc", namespace=f"{namespace}/{proj_slug}") - dc_id = dc.get("id") - assert dc_id is not None - assert dc is not None - async with app_manager.config.db.async_session_maker() as session, session.begin(): - # We do not have APIs exposed that will remove the slug so this is the only way to trigger this - stmt = ( - select(EntitySlugORM) - .where(EntitySlugORM.project_id == proj_id) - .where(EntitySlugORM.namespace_id.is_not(None)) - .where(EntitySlugORM.data_connector_id.is_(None)) - ) - res = await session.scalar(stmt) - assert res is not None - await session.delete(res) - await session.flush() - # The project namespace is not there - _, response = await sanic_client.get(f"/api/data/namespaces/{namespace}/{proj_slug}", headers=user_headers) - assert response.status_code == 404 - # The user or group namespace is untouched - _, response = await sanic_client.get(f"/api/data/namespaces/{namespace}", headers=user_headers) - assert response.status_code == 200 - # The project and data connector are both gone - _, response = await sanic_client.get(f"/api/data/projects/{proj_id}", headers=user_headers) - assert response.status_code == 404 - _, response = await sanic_client.get(f"/api/data/data_connectors/{dc_id}", headers=user_headers) - assert response.status_code == 404 - - -@pytest.mark.asyncio -async def test_stored_procedure_cleanup_after_data_connector_slug_deletion( - create_project, - user_headers, - app_manager: DependencyManager, - sanic_client, - create_data_connector, -) -> None: - # We use stored procedures to remove a data connector when its slug is removed - proj = await create_project(name="test1") - proj_id = proj.get("id") - assert proj_id is not None - namespace = proj.get("namespace") - assert namespace is not None - proj_slug = proj.get("slug") - assert proj_slug is not None - _, response = await sanic_client.get(f"/api/data/namespaces/{namespace}", headers=user_headers) - assert response.status_code == 200 - dc1 = await create_data_connector(name="test-dc", namespace=f"{namespace}/{proj_slug}") - dc1_id = dc1.get("id") - assert dc1_id is not None - assert dc1 is not None - dc2 = await create_data_connector(name="test-dc", namespace=namespace) - dc2_id = dc2.get("id") - assert dc2_id is not None - assert dc2 is not None - async with app_manager.config.db.async_session_maker() as session, session.begin(): - # We do not have APIs exposed that will remove the slug so this is the only way to trigger this - stmt = select(EntitySlugORM).where(EntitySlugORM.data_connector_id == dc1_id) - scalars = await session.scalars(stmt) - res = scalars.one_or_none() - assert res is not None - await session.delete(res) - stmt = select(EntitySlugORM).where(EntitySlugORM.data_connector_id == dc2_id) - scalars = await session.scalars(stmt) - res = scalars.one_or_none() - assert res is not None - await session.delete(res) - await session.flush() - # The project namespace is still there - _, response = await sanic_client.get(f"/api/data/namespaces/{namespace}/{proj_slug}", headers=user_headers) - assert response.status_code == 200 - # The user or group namespace is untouched - _, response = await sanic_client.get(f"/api/data/namespaces/{namespace}", headers=user_headers) - assert response.status_code == 200 - # The project is still there - _, response = await sanic_client.get(f"/api/data/projects/{proj_id}", headers=user_headers) - assert response.status_code == 200 - # The data connectors are gone - _, response = await sanic_client.get(f"/api/data/data_connectors/{dc1_id}", headers=user_headers) - assert response.status_code == 404 - _, response = await sanic_client.get(f"/api/data/data_connectors/{dc2_id}", headers=user_headers) - assert response.status_code == 404 - - async def test_cleanup_with_group_deletion( create_project, create_group, From 07f44eddc9ddac8e941587099962dde07914d456 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 23 Oct 2025 15:42:48 +0200 Subject: [PATCH 7/8] Add magic files to make pytest find test cases --- test/components/renku_data_services/namespace/__init__.py | 1 + test/components/renku_data_services/project/__init__.py | 1 + test/components/renku_data_services/search/__init__.py | 1 + 3 files changed, 3 insertions(+) create mode 100644 test/components/renku_data_services/namespace/__init__.py create mode 100644 test/components/renku_data_services/project/__init__.py create mode 100644 test/components/renku_data_services/search/__init__.py diff --git a/test/components/renku_data_services/namespace/__init__.py b/test/components/renku_data_services/namespace/__init__.py new file mode 100644 index 000000000..d66655493 --- /dev/null +++ b/test/components/renku_data_services/namespace/__init__.py @@ -0,0 +1 @@ +"""Only here to make pytest work.""" diff --git a/test/components/renku_data_services/project/__init__.py b/test/components/renku_data_services/project/__init__.py new file mode 100644 index 000000000..d66655493 --- /dev/null +++ b/test/components/renku_data_services/project/__init__.py @@ -0,0 +1 @@ +"""Only here to make pytest work.""" diff --git a/test/components/renku_data_services/search/__init__.py b/test/components/renku_data_services/search/__init__.py new file mode 100644 index 000000000..d66655493 --- /dev/null +++ b/test/components/renku_data_services/search/__init__.py @@ -0,0 +1 @@ +"""Only here to make pytest work.""" From abc85e658c916b1fc12a0d5253c69c3c46dd74df Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 30 Oct 2025 17:12:47 +0100 Subject: [PATCH 8/8] Change comment in __init__py in tests --- test/components/renku_data_services/namespace/__init__.py | 2 +- test/components/renku_data_services/project/__init__.py | 2 +- test/components/renku_data_services/search/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/components/renku_data_services/namespace/__init__.py b/test/components/renku_data_services/namespace/__init__.py index d66655493..0a7cebfe2 100644 --- a/test/components/renku_data_services/namespace/__init__.py +++ b/test/components/renku_data_services/namespace/__init__.py @@ -1 +1 @@ -"""Only here to make pytest work.""" +"""Namespaces test module.""" diff --git a/test/components/renku_data_services/project/__init__.py b/test/components/renku_data_services/project/__init__.py index d66655493..711d37cef 100644 --- a/test/components/renku_data_services/project/__init__.py +++ b/test/components/renku_data_services/project/__init__.py @@ -1 +1 @@ -"""Only here to make pytest work.""" +"""Project test module.""" diff --git a/test/components/renku_data_services/search/__init__.py b/test/components/renku_data_services/search/__init__.py index d66655493..4184cbbc8 100644 --- a/test/components/renku_data_services/search/__init__.py +++ b/test/components/renku_data_services/search/__init__.py @@ -1 +1 @@ -"""Only here to make pytest work.""" +"""Search test modul."""