diff --git a/keep/api/models/db/migrations/versions/2025-03-14-17-49_aaec81b991bd.py b/keep/api/models/db/migrations/versions/2025-03-14-17-49_aaec81b991bd.py new file mode 100644 index 000000000..41a839161 --- /dev/null +++ b/keep/api/models/db/migrations/versions/2025-03-14-17-49_aaec81b991bd.py @@ -0,0 +1,316 @@ +"""Crecreate topology tables with tenant_id in all keys and transfer data + +Revision ID: aaec81b991bd +Revises: f3ecc7411f38 +Create Date: 2025-03-14 17:49:13.393091 + +""" + +import sqlalchemy as sa +import sqlmodel +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = "aaec81b991bd" +down_revision = "f3ecc7411f38" +branch_labels = None +depends_on = None + +def transfer_data(): + session = Session(bind=op.get_bind()) + dialect = session.bind.dialect.name + + uuid_generation_func = "replace(uuid(),'-','')" + if dialect == "sqlite": + uuid_generation_func = """ + lower( + hex(randomblob(4)) || '-' || hex(randomblob(2)) || '-' || '4' || + substr(hex( randomblob(2)), 2) || '-' || + substr('AB89', 1 + (abs(random()) % 4) , 1) || + substr(hex(randomblob(2)), 2) || '-' || + hex(randomblob(6)) + ) + """ + elif dialect == "postgresql": + uuid_generation_func = "gen_random_uuid()" + + session.execute(sa.text(f""" + INSERT INTO topologyservice ( + id, external_id, tenant_id, source_provider_id, repository, tags, service, environment, display_name, description, + team, email, slack, ip_address, mac_address, category, manufacturer, namespace, is_manual + ) + SELECT + {uuid_generation_func} as id, id as external_id, tenant_id, source_provider_id, repository, tags, service, environment, display_name, description, + team, email, slack, ip_address, mac_address, category, manufacturer, namespace, is_manual + FROM topologyservice_tmp + """)) + + session.execute(sa.text(""" + INSERT INTO topologyapplication (id, tenant_id, name, description, repository) + SELECT id, tenant_id, name, description, repository FROM topologyapplication_tmp + """)) + + session.execute(sa.text(""" + INSERT INTO topologyserviceapplication (service_id, application_id, tenant_id) + SELECT ts.id, tsa.application_id, ts.tenant_id FROM topologyserviceapplication_tmp as tsa + JOIN topologyservice as ts ON tsa.service_id = ts.external_id + """)) + + session.execute(sa.text(f""" + INSERT INTO topologyservicedependency (id, service_id, depends_on_service_id, updated_at, protocol, tenant_id) + SELECT {uuid_generation_func} as id, ts.id as service_id, ts_dep.id as depends_on_service_id, tsd.updated_at, tsd.protocol, ts.tenant_id + FROM topologyservicedependency_tmp as tsd + JOIN topologyservice as ts ON tsd.service_id = ts.external_id + JOIN topologyservice as ts_dep ON tsd.depends_on_service_id = ts_dep.external_id + """)) + + +def transfer_data_back(): + session = Session(bind=op.get_bind()) + + session.execute(sa.text(""" + INSERT INTO topologyservice ( + id, tenant_id, source_provider_id, repository, tags, service, environment, display_name, description, + team, email, slack, ip_address, mac_address, category, manufacturer, namespace, is_manual + ) + SELECT external_id as id, tenant_id, source_provider_id, repository, tags, service, environment, display_name, description, + team, email, slack, ip_address, mac_address, category, manufacturer, namespace, is_manual + FROM topologyservice_tmp + """)) + + session.execute(sa.text(""" + INSERT INTO topologyapplication (id, tenant_id, name, description, repository) + SELECT id, tenant_id, name, description, repository FROM topologyapplication_tmp + """)) + + session.execute(sa.text(""" + INSERT INTO topologyserviceapplication (service_id, application_id) + SELECT ts.external_id, tsa.application_id FROM topologyserviceapplication_tmp as tsa + JOIN topologyservice_tmp as ts ON tsa.service_id = ts.id + + """)) + + session.execute(sa.text(""" + INSERT INTO topologyservicedependency (service_id, depends_on_service_id, updated_at, protocol) + SELECT ts.external_id, ts_dep.external_id, tsd.updated_at, tsd.protocol + FROM topologyservicedependency_tmp as tsd + JOIN topologyservice_tmp as ts ON tsd.service_id = ts.id + JOIN topologyservice_tmp as ts_dep ON tsd.depends_on_service_id = ts_dep.id + """)) + +def upgrade(): + op.rename_table('topologyapplication', 'topologyapplication_tmp') + op.rename_table('topologyservice', 'topologyservice_tmp') + op.rename_table('topologyserviceapplication', 'topologyserviceapplication_tmp') + op.rename_table('topologyservicedependency', 'topologyservicedependency_tmp') + + op.create_table( + "topologyapplication", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("tenant_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("repository", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.ForeignKeyConstraint( + ["tenant_id"], + ["tenant.id"], + ), + sa.PrimaryKeyConstraint("id", "tenant_id"), + ) + op.create_table( + "topologyservice", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("external_id", sa.Integer(), nullable=True), + sa.Column("tenant_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column( + "source_provider_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False + ), + sa.Column("repository", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("tags", sa.JSON(), nullable=True), + sa.Column("service", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("environment", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("display_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("team", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("email", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("slack", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("ip_address", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("mac_address", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("category", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("manufacturer", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("namespace", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("is_manual", sa.Boolean(), nullable=True), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.current_timestamp(), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["tenant_id"], + ["tenant.id"], + ), + sa.PrimaryKeyConstraint("id", "tenant_id"), + ) + op.create_table( + "topologyserviceapplication", + sa.Column("tenant_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("service_id", sa.Uuid(), nullable=False), + sa.Column("application_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["application_id", "tenant_id"], + ["topologyapplication.id", "topologyapplication.tenant_id"], + ), + sa.ForeignKeyConstraint( + ["service_id", "tenant_id"], + ["topologyservice.id", "topologyservice.tenant_id"], + ), + sa.ForeignKeyConstraint( + ["tenant_id"], + ["tenant.id"], + ), + sa.PrimaryKeyConstraint("tenant_id", "service_id", "application_id"), + ) + op.create_table( + "topologyservicedependency", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("tenant_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("service_id", sa.Uuid(), nullable=False), + sa.Column("depends_on_service_id", sa.Uuid(), nullable=False), + sa.Column("protocol", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.current_timestamp(), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["depends_on_service_id", "tenant_id"], + ["topologyservice.id", "topologyservice.tenant_id"], + name="topologyservicedependency_depends_on_service_id_tenant_id_fkey", + ), + sa.ForeignKeyConstraint( + ["service_id", "tenant_id"], + ["topologyservice.id", "topologyservice.tenant_id"], + name="topologyservicedependency_service_id_tenant_id_fkey", + ), + sa.ForeignKeyConstraint( + ["tenant_id"], + ["tenant.id"], + ), + sa.PrimaryKeyConstraint("id", "tenant_id"), + ) + + transfer_data() + + # Let's do not drop this as backup for a while + + op.rename_table('topologyapplication_tmp', 'topologyapplication_backup') + op.rename_table('topologyservice_tmp', 'topologyservice_backup') + op.rename_table('topologyserviceapplication_tmp', 'topologyserviceapplication_backup') + op.rename_table('topologyservicedependency_tmp', 'topologyservicedependency_backup') + + # But after some time we will need to execute this: + + # op.drop_table("topologyservicedependency_backup") + # op.drop_table("topologyserviceapplication_backup") + # op.drop_table("topologyapplication_backup") + # op.drop_table("topologyservice_backup") + + +def downgrade(): + + op.rename_table('topologyapplication', 'topologyapplication_tmp') + op.rename_table('topologyservice', 'topologyservice_tmp') + op.rename_table('topologyserviceapplication', 'topologyserviceapplication_tmp') + op.rename_table('topologyservicedependency', 'topologyservicedependency_tmp') + + op.create_table( + "topologyapplication", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("tenant_id", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("repository", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.ForeignKeyConstraint( + ["tenant_id"], + ["tenant.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "topologyservice", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("tenant_id", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "source_provider_id", sqlmodel.sql.sqltypes.AutoString(), nullable=False + ), + sa.Column("repository", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("tags", sa.JSON(), nullable=True), + sa.Column("service", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("environment", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("display_name", sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("team", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("email", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("slack", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("ip_address", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("mac_address", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("category", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("manufacturer", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("namespace", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("is_manual", sa.Boolean(), nullable=True), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["tenant_id"], + ["tenant.id"], + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "topologyserviceapplication", + sa.Column("service_id", sa.Integer(), nullable=False), + sa.Column("application_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["application_id"], + ["topologyapplication.id"], + ), + sa.ForeignKeyConstraint( + ["service_id"], + ["topologyservice.id"], + ), + sa.PrimaryKeyConstraint("service_id", "application_id"), + ) + op.create_table( + "topologyservicedependency", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("service_id", sa.Integer(), nullable=True), + sa.Column("depends_on_service_id", sa.Integer(), nullable=True), + sa.Column("protocol", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("(CURRENT_TIMESTAMP)"), + nullable=True, + ), + sa.ForeignKeyConstraint( + ["depends_on_service_id"], ["topologyservice.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["service_id"], ["topologyservice.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + + transfer_data_back() + + op.drop_table("topologyservicedependency_tmp") + op.drop_table("topologyserviceapplication_tmp") + op.drop_table("topologyapplication_tmp") + op.drop_table("topologyservice_tmp") diff --git a/keep/api/models/db/topology.py b/keep/api/models/db/topology.py index 9489f3721..246c116d2 100644 --- a/keep/api/models/db/topology.py +++ b/keep/api/models/db/topology.py @@ -3,31 +3,45 @@ from uuid import UUID, uuid4 from pydantic import BaseModel -from sqlalchemy import DateTime, ForeignKey +from sqlalchemy import DateTime, ForeignKey, PrimaryKeyConstraint, ForeignKeyConstraint from sqlmodel import JSON, Column, Field, Relationship, SQLModel, func class TopologyServiceApplication(SQLModel, table=True): - service_id: int = Field(foreign_key="topologyservice.id", primary_key=True) - application_id: UUID = Field(foreign_key="topologyapplication.id", primary_key=True) + tenant_id: str = Field(sa_column=Column(ForeignKey("tenant.id"), primary_key=True)) + service_id: UUID = Field(primary_key=True) + application_id: UUID = Field(primary_key=True) service: "TopologyService" = Relationship( sa_relationship_kwargs={ - "primaryjoin": "TopologyService.id == TopologyServiceApplication.service_id", + "primaryjoin": "and_(TopologyService.id == TopologyServiceApplication.service_id," + "TopologyService.tenant_id == TopologyServiceApplication.tenant_id)", "viewonly": "True", }, ) application: "TopologyApplication" = Relationship( sa_relationship_kwargs={ - "primaryjoin": "TopologyApplication.id == TopologyServiceApplication.application_id", + "primaryjoin": "and_(TopologyApplication.id == TopologyServiceApplication.application_id," + "TopologyService.tenant_id == TopologyServiceApplication.tenant_id)", "viewonly": "True", }, ) + __table_args__ = ( + ForeignKeyConstraint( + ['service_id', 'tenant_id'], + ['topologyservice.id', 'topologyservice.tenant_id'], + ), + ForeignKeyConstraint( + ['application_id', 'tenant_id'], + ['topologyapplication.id', 'topologyapplication.tenant_id'], + ), + ) + class TopologyApplication(SQLModel, table=True): id: UUID = Field(default_factory=uuid4, primary_key=True) - tenant_id: str = Field(sa_column=Column(ForeignKey("tenant.id"))) + tenant_id: str = Field(sa_column=Column(ForeignKey("tenant.id"), primary_key=True)) name: str description: str = Field(default_factory=str) repository: str = Field(default_factory=str) @@ -37,7 +51,8 @@ class TopologyApplication(SQLModel, table=True): class TopologyService(SQLModel, table=True): - id: Optional[int] = Field(primary_key=True, default=None) + id: UUID = Field(default_factory=uuid4) + external_id: Optional[int] tenant_id: str = Field(sa_column=Column(ForeignKey("tenant.id"))) source_provider_id: str = "unknown" repository: Optional[str] @@ -77,19 +92,20 @@ class TopologyService(SQLModel, table=True): back_populates="services", link_model=TopologyServiceApplication ) + __table_args__ = ( + PrimaryKeyConstraint("id", "tenant_id"), # Composite PK + ) + class Config: orm_mode = True - unique_together = ["tenant_id", "service", "environment", "source_provider_id"] + unique_together = [["id", "tenant_id"], ["tenant_id", "service", "environment", "source_provider_id"]] class TopologyServiceDependency(SQLModel, table=True): - id: Optional[int] = Field(primary_key=True, default=None) - service_id: int = Field( - sa_column=Column(ForeignKey("topologyservice.id", ondelete="CASCADE")) - ) - depends_on_service_id: int = Field( - sa_column=Column(ForeignKey("topologyservice.id", ondelete="CASCADE")) - ) # service_id calls deponds_on_service_id (A->B) + id: UUID = Field(default_factory=uuid4) + tenant_id: str = Field(sa_column=Column(ForeignKey("tenant.id"))) + service_id: UUID + depends_on_service_id: UUID protocol: Optional[str] = "unknown" updated_at: Optional[datetime] = Field( sa_column=Column( @@ -112,6 +128,19 @@ class TopologyServiceDependency(SQLModel, table=True): } ) + __table_args__ = ( + PrimaryKeyConstraint("id", "tenant_id"), # Composite PK + ForeignKeyConstraint( + ['service_id', 'tenant_id'], + ['topologyservice.id', 'topologyservice.tenant_id'], + "topologyservicedependency_service_id_tenant_id_fkey" + ), + ForeignKeyConstraint( + ['depends_on_service_id', 'tenant_id'], + ['topologyservice.id', 'topologyservice.tenant_id'], + "topologyservicedependency_depends_on_service_id_tenant_id_fkey" + ), + ) class TopologyServiceDtoBase(BaseModel, extra="ignore"): source_provider_id: Optional[str] @@ -140,8 +169,8 @@ class TopologyServiceInDto(TopologyServiceDtoBase): class TopologyServiceDependencyDto(BaseModel, extra="ignore"): - id: Optional[str] = None - serviceId: str + id: str | UUID + serviceId: str | UUID serviceName: str protocol: Optional[str] = "unknown" @@ -166,7 +195,7 @@ class TopologyApplicationDto(BaseModel, extra="ignore"): class TopologyServiceDtoIn(BaseModel, extra="ignore"): - id: int + id: UUID class TopologyApplicationDtoIn(BaseModel, extra="ignore"): @@ -211,7 +240,7 @@ def from_orm( class TopologyServiceDtoOut(TopologyServiceDtoBase): - id: str + id: UUID | str dependencies: List[TopologyServiceDependencyDto] application_ids: List[UUID] updated_at: Optional[datetime] @@ -270,28 +299,28 @@ class TopologyServiceCreateRequestDTO(BaseModel, extra="ignore"): class TopologyServiceUpdateRequestDTO(TopologyServiceCreateRequestDTO, extra="ignore"): - id: int + id: UUID | str class TopologyServiceDependencyCreateRequestDto(BaseModel, extra="ignore"): - service_id: int - depends_on_service_id: int + service_id: UUID | str + depends_on_service_id: UUID | str protocol: Optional[str] = "unknown" class TopologyServiceDependencyUpdateRequestDto( TopologyServiceDependencyCreateRequestDto, extra="ignore" ): - service_id: Optional[int] - depends_on_service_id: Optional[int] - id: int + service_id: Optional[UUID | str] + depends_on_service_id: Optional[UUID | str] + id: UUID | str class DeleteServicesRequest(BaseModel, extra="ignore"): - service_ids: List[int] + service_ids: List[UUID | str] class TopologyServiceYAML(TopologyServiceCreateRequestDTO, extra="ignore"): - id: int - source_provider_id: Optional[str] = None + id: UUID | str + source_provider_id: Optional[UUID | str] = None is_manual: Optional[bool] = None diff --git a/keep/api/routes/topology.py b/keep/api/routes/topology.py index ff8a1050c..1f14edd2a 100644 --- a/keep/api/routes/topology.py +++ b/keep/api/routes/topology.py @@ -425,7 +425,7 @@ def update_dependency( "/dependency/{dependency_id}", description="Deleting a dependency manually" ) def delete_dependency( - dependency_id: int, + dependency_id: UUID | str, authenticated_entity: AuthenticatedEntity = Depends( IdentityManagerFactory.get_auth_verifier(["write:topology"]) ), diff --git a/keep/api/tasks/process_topology_task.py b/keep/api/tasks/process_topology_task.py index a4a0e8d41..4804ea2c3 100644 --- a/keep/api/tasks/process_topology_task.py +++ b/keep/api/tasks/process_topology_task.py @@ -117,6 +117,7 @@ def process_topology( TopologyServiceDependency( service_id=service_id, depends_on_service_id=depends_on_service_id, + tenant_id=tenant_id, protocol=service.dependencies.get(dependency, "unknown"), ) ) diff --git a/keep/topologies/topologies_service.py b/keep/topologies/topologies_service.py index 70d56a725..c69317d3f 100644 --- a/keep/topologies/topologies_service.py +++ b/keep/topologies/topologies_service.py @@ -100,7 +100,7 @@ def get_service_application_ids_dict( def validate_non_manual_exists( - service_ids: list[int], session: Session, tenant_id: str + service_ids: list[UUID | str], session: Session, tenant_id: str ) -> bool: non_manual_exists = session.query( exists() @@ -240,7 +240,7 @@ def create_application_by_tenant_id( # Create TopologyServiceApplication links new_links = [ TopologyServiceApplication( - service_id=service.id, application_id=new_application.id + service_id=service.id, application_id=new_application.id, tenant_id=tenant_id ) for service in services_to_add if service.id @@ -298,7 +298,7 @@ def create_applications_by_tenant_id( new_links.extend( [ TopologyServiceApplication( - service_id=service.id, application_id=new_application.id + service_id=service.id, application_id=new_application.id, tenant_id=tenant_id ) for service in application.services if service.id @@ -439,9 +439,11 @@ def get_service_by_id( ).first() @staticmethod - def get_dependency_by_id(_id: int, session: Session) -> TopologyServiceDependency: + def get_dependency_by_id(_id: int, tenant_id: str, session: Session) -> TopologyServiceDependency: return session.exec( - select(TopologyServiceDependency).where(TopologyServiceDependency.id == _id) + select(TopologyServiceDependency) + .where(TopologyServiceDependency.id == _id) + .where(TopologyServiceDependency.tenant_id == tenant_id) ).first() @staticmethod @@ -585,7 +587,7 @@ def create_dependency( ): raise ServiceNotManualException() - db_dependency = TopologyServiceDependency(**dependency.dict()) + db_dependency = TopologyServiceDependency(tenant_id=tenant_id, **dependency.dict()) session.add(db_dependency) session.commit() session.refresh(db_dependency) @@ -621,7 +623,7 @@ def create_dependencies( ): raise ServiceNotManualException() - db_dependency = TopologyServiceDependency(**dependency.dict()) + db_dependency = TopologyServiceDependency(tenant_id=tenant_id, **dependency.dict()) session.add(db_dependency) db_dependencies.append(db_dependency) @@ -651,7 +653,7 @@ def update_dependency( db_dependency: TopologyServiceDependency = ( TopologiesService.get_dependency_by_id( - _id=dependency.id, session=session + _id=dependency.id, tenant_id=tenant_id, session=session ) ) service_dict = dependency.dict() @@ -675,11 +677,11 @@ def update_dependency( session.close() @staticmethod - def delete_dependency(dependency_id: int, session: Session, tenant_id: str): + def delete_dependency(dependency_id: UUID | str, session: Session, tenant_id: str): try: db_dependency: TopologyServiceDependency = ( TopologiesService.get_dependency_by_id( - _id=dependency_id, session=session + _id=dependency_id, tenant_id=tenant_id, session=session ) ) # Enforcing is_manual on the service_id and depends_on_service_id diff --git a/tests/e2e_tests/test_topology.py b/tests/e2e_tests/test_topology.py index 2c4e243b3..2ff55fcd5 100644 --- a/tests/e2e_tests/test_topology.py +++ b/tests/e2e_tests/test_topology.py @@ -24,7 +24,7 @@ def test_topology_manual(browser): # Open the Service Topology page browser.get_by_role("link", name="Service Topology").hover() browser.get_by_role("link", name="Service Topology").click() - browser.wait_for_timeout(2000) # Added extra wait for page to fully load + browser.wait_for_timeout(5000) # Added extra wait for page to fully load max_retries = 5 retries = 0 @@ -33,6 +33,7 @@ def test_topology_manual(browser): while retries <= max_retries: try: browser.get_by_role("button", name="Add Node", exact=True).click() + browser.wait_for_timeout(2000) browser.get_by_placeholder("Enter service here...").fill("service_id_1") break except Exception: @@ -89,9 +90,7 @@ def test_topology_manual(browser): browser.wait_for_timeout(2000) # Improved edge connection with retries - def connect_nodes(source_selector, target_selector, edge_label, max_attempts=3): - source_handle = browser.locator(source_selector) - target_handle = browser.locator(target_selector) + def connect_nodes(source_handle, target_handle, edge_label, max_attempts=3): for attempt in range(max_attempts): try: @@ -147,13 +146,22 @@ def connect_nodes(source_selector, target_selector, edge_label, max_attempts=3): return False # Define handles with stable selectors - source_handle_1 = "div[data-id='1-1-right-source']" - target_handle_2 = "div[data-id='1-2-left-target']" - target_handle_3 = "div[data-id='1-3-left-target']" + node_1 = browser.locator("div.react-flow__node-service").filter(has_text="SERVICE_ID_1") + node_2 = browser.locator("div.react-flow__node-service").filter(has_text="SERVICE_ID_2") + node_3 = browser.locator("div.react-flow__node-service").filter(has_text="SERVICE_ID_3") + + node_1_id = node_1.get_attribute('data-id') + node_2_id = node_2.get_attribute('data-id') + node_3_id = node_3.get_attribute('data-id') + + # Connect nodes by dragging source to target handles + source_handle_1 = node_1.locator(f"div[data-id='1-{node_1_id}-right-source']") + target_handle_2 = node_2.locator(f"div[data-id='1-{node_2_id}-left-target']") + target_handle_3 = node_3.locator(f"div[data-id='1-{node_3_id}-left-target']") # Connect nodes with retry logic edge1_created = connect_nodes( - source_handle_1, target_handle_2, "Edge from 1 to 2" + source_handle_1, target_handle_2, f"Edge from {node_1_id} to {node_2_id}" ) if not edge1_created: # Take diagnostic screenshots @@ -162,7 +170,7 @@ def connect_nodes(source_selector, target_selector, edge_label, max_attempts=3): print("Failed to create edge from node 1 to node 2 after multiple attempts") edge2_created = connect_nodes( - source_handle_1, target_handle_3, "Edge from 1 to 3" + source_handle_1, target_handle_3, f"Edge from {node_1_id} to {node_3_id}" ) if not edge2_created: # Take diagnostic screenshots @@ -172,18 +180,13 @@ def connect_nodes(source_selector, target_selector, edge_label, max_attempts=3): # Validate edge connections with more flexible assertions browser.wait_for_timeout(2000) - - edge_1_to_2 = browser.locator( - "g.react-flow__edge[aria-label='Edge from 1 to 2']" - ) + edge_1_to_2 = browser.locator(f"g.react-flow__edge[aria-label='Edge from {node_1_id} to {node_2_id}']") expect(edge_1_to_2).to_have_count(1, timeout=10000) # Increased timeout - - edge_1_to_3 = browser.locator( - "g.react-flow__edge[aria-label='Edge from 1 to 3']" - ) + edge_1_to_3 = browser.locator(f"g.react-flow__edge[aria-label='Edge from {node_1_id} to {node_3_id}']") expect(edge_1_to_3).to_have_count(1, timeout=10000) # Increased timeout # Continue with rest of the test... + # Delete edge edge_end = edge_1_to_2.locator("circle.react-flow__edgeupdater-target") edge_end.scroll_into_view_if_needed() @@ -194,21 +197,22 @@ def connect_nodes(source_selector, target_selector, edge_label, max_attempts=3): # Ensure edge was deleted with retry for _ in range(5): if ( - browser.locator( - "g.react-flow__edge[aria-label='Edge from 1 to 2']" - ).count() - == 0 + browser.locator( + f"g.react-flow__edge[aria-label='Edge from {node_1_id} to {node_2_id}']" + ).count() + == 0 ): - break + if browser.locator("g.react-flow__edge").count() == 1: + break browser.wait_for_timeout(1000) expect( - browser.locator("g.react-flow__edge[aria-label='Edge from 1 to 2']") + browser.locator(f"g.react-flow__edge[aria-label='Edge {node_1_id} to {node_2_id}']") ).to_have_count(0, timeout=5000) # Ensure remaining edges are intact expect( - browser.locator("g.react-flow__edge[aria-label='Edge from 1 to 3']") + browser.locator(f"g.react-flow__edge[aria-label='Edge from {node_1_id} to {node_3_id}']") ).to_have_count(1, timeout=5000) browser.wait_for_timeout(2000) @@ -241,7 +245,7 @@ def connect_nodes(source_selector, target_selector, edge_label, max_attempts=3): for _ in range(5): if ( browser.locator( - "g.react-flow__edge[aria-label='Edge from 1 to 3']" + f"g.react-flow__edge[aria-label='Edge from {node_1_id} to {node_3_id}']" ).count() == 0 ): @@ -249,7 +253,7 @@ def connect_nodes(source_selector, target_selector, edge_label, max_attempts=3): browser.wait_for_timeout(1000) expect( - browser.locator("g.react-flow__edge[aria-label='Edge from 1 to 3']") + browser.locator(f"g.react-flow__edge[aria-label='Edge from {node_1_id} to {node_3_id}']") ).to_have_count(0, timeout=5000) # Update node name and verify the change diff --git a/tests/test_enrichments.py b/tests/test_enrichments.py index 649b76c8c..35de67790 100644 --- a/tests/test_enrichments.py +++ b/tests/test_enrichments.py @@ -599,7 +599,7 @@ def test_disposable_enrichment(db_session, client, test_app, mock_alert_dto): def test_topology_mapping_rule_enrichment(mock_session, mock_alert_dto): # Mock a TopologyService with dependencies to simulate the DB structure mock_topology_service = TopologyService( - id=1, tenant_id="keep", service="test-service", display_name="Test Service" + external_id=1, tenant_id="keep", service="test-service", display_name="Test Service" ) # Create a mock MappingRule for topology @@ -643,6 +643,7 @@ def test_topology_mapping_rule_enrichment(mock_session, mock_alert_dto): { "source_provider_id": "unknown", "service": "test-service", + "external_id": 1, "environment": "unknown", "display_name": "Test Service", "is_manual": False, diff --git a/tests/test_topology.py b/tests/test_topology.py index e9911b151..671341f83 100644 --- a/tests/test_topology.py +++ b/tests/test_topology.py @@ -23,11 +23,12 @@ VALID_API_KEY = "valid_api_key" -def create_service(db_session, tenant_id, id): +def create_service(db_session, tenant_id, external_id): service = TopologyService( + external_id=external_id, tenant_id=tenant_id, - service="test_service_" + id, - display_name=id, + service="test_service_" + external_id, + display_name=external_id, repository="test_repository", tags=["test_tag"], description="test_description", @@ -50,6 +51,7 @@ def test_get_all_topology_data(db_session): assert len(result) == 0 dependency = TopologyServiceDependency( + tenant_id=SINGLE_TENANT_UUID, service_id=service_1.id, depends_on_service_id=service_2.id, updated_at=datetime.now(), @@ -103,7 +105,7 @@ def test_create_application_by_tenant_id(db_session): SINGLE_TENANT_UUID, application_dto, db_session ) - application_dto.services.append(TopologyServiceDtoIn(id=123)) + application_dto.services.append(TopologyServiceDtoIn(id=uuid.uuid4())) with pytest.raises(ServiceNotFoundException): TopologiesService.create_application_by_tenant_id( SINGLE_TENANT_UUID, application_dto, db_session @@ -129,8 +131,10 @@ def test_create_application_by_tenant_id(db_session): assert len(result) == 1 assert result[0].name == "New Application" assert len(result[0].services) == 2 - assert result[0].services[0].service == "test_service_1" - assert result[0].services[1].service == "test_service_2" + + services_names = [s.service for s in result[0].services] + assert "test_service_1" in services_names + assert "test_service_2" in services_names def test_update_application_by_id(db_session): @@ -211,7 +215,7 @@ def test_create_application(db_session, client, test_app): service = create_service(db_session, SINGLE_TENANT_UUID, "1") - application_data = {"name": "New Application", "services": [{"id": service.id}]} + application_data = {"name": "New Application", "services": [{"id": str(service.id)}]} response = client.post( "/topology/applications", @@ -254,7 +258,7 @@ def test_update_application(db_session, client, test_app): assert response.status_code == 200 assert response.json()["name"] == "Updated Application" - invalid_update_data = {"name": "Invalid Application", "services": [{"id": "123"}]} + invalid_update_data = {"name": "Invalid Application", "services": [{"id": str(random_uuid)}]} response = client.put( f"/topology/applications/{application.id}", @@ -307,6 +311,7 @@ def test_clean_before_import(db_session): dependency = TopologyServiceDependency( service_id=service_1.id, depends_on_service_id=service_2.id, + tenant_id=tenant_id, updated_at=datetime.now(), ) db_session.add(dependency) @@ -315,7 +320,7 @@ def test_clean_before_import(db_session): # Assert data exists before cleaning assert db_session.exec(select(TopologyService).where(TopologyService.tenant_id == tenant_id)).all() assert db_session.exec(select(TopologyApplication).where(TopologyApplication.tenant_id == tenant_id)).all() - assert db_session.exec(select(TopologyServiceDependency)).all() + assert db_session.exec(select(TopologyServiceDependency).where(TopologyApplication.tenant_id == tenant_id)).all() # Act: Call the clean_before_import function TopologiesService.clean_before_import(tenant_id, db_session) @@ -323,7 +328,7 @@ def test_clean_before_import(db_session): # Assert: Ensure all data is deleted for this tenant assert not db_session.exec(select(TopologyService).where(TopologyService.tenant_id == tenant_id)).all() assert not db_session.exec(select(TopologyApplication).where(TopologyApplication.tenant_id == tenant_id)).all() - assert not db_session.exec(select(TopologyServiceDependency)).all() + assert not db_session.exec(select(TopologyServiceDependency).where(TopologyApplication.tenant_id == tenant_id)).all() def test_import_to_db(db_session): @@ -332,10 +337,15 @@ def test_import_to_db(db_session): # Do same operation twice - import and re-import for i in range(2): + + s1_id = str(uuid.uuid4()) + s2_id = str(uuid.uuid4()) + topology_data = { "services": [ { - "id": 1, + "id": s1_id, + "external_id": "1", "service": "test_service_1", "display_name": "Service 1", "tags": ["tag1"], @@ -343,7 +353,8 @@ def test_import_to_db(db_session): "email": "test1@example.com", }, { - "id": 2, + "id": s2_id, + "external_id": "2", "service": "test_service_2", "display_name": "Service 2", "tags": ["tag2"], @@ -355,18 +366,18 @@ def test_import_to_db(db_session): { "name": "Test Application 1", "description": "Application 1 description", - "services": [1], + "services": [s1_id], }, { "name": "Test Application 2", "description": "Application 2 description", - "services": [2], + "services": [s2_id], }, ], "dependencies": [ { - "service_id": 1, - "depends_on_service_id": 2, + "service_id": s1_id, + "depends_on_service_id": s2_id, } ], } @@ -385,5 +396,5 @@ def test_import_to_db(db_session): dependencies = db_session.exec(select(TopologyServiceDependency)).all() assert len(dependencies) == 1 - assert dependencies[0].service_id == 1 - assert dependencies[0].depends_on_service_id == 2 + assert str(dependencies[0].service_id) == s1_id + assert str(dependencies[0].depends_on_service_id) == s2_id