From 8e10316a8eed5e234794ecd44710125c6ede36a4 Mon Sep 17 00:00:00 2001 From: Pavel Mackarichev Date: Tue, 24 Oct 2023 13:32:22 +0300 Subject: [PATCH] tests: clean up tests --- .../adapters/databricks/mappers/models.py | 2 +- .../adapters/databricks/mappers/table.py | 2 +- .../adapters/superset/tests/__init__.py | 0 .../adapters/superset/tests/raw_data.py | 89 ------------------- .../superset/tests/test_map_column.py | 28 ------ .../superset/tests/test_map_dashboard.py | 55 ------------ .../superset/tests/test_map_database.py | 55 ------------ .../adapters/superset/tests/test_map_table.py | 53 ----------- tests/adapters/mode/mock_rest_client.py | 2 +- tests/integration/test_postgres.py | 4 +- 10 files changed, 5 insertions(+), 285 deletions(-) delete mode 100644 odd_collector/adapters/superset/tests/__init__.py delete mode 100644 odd_collector/adapters/superset/tests/raw_data.py delete mode 100644 odd_collector/adapters/superset/tests/test_map_column.py delete mode 100644 odd_collector/adapters/superset/tests/test_map_dashboard.py delete mode 100644 odd_collector/adapters/superset/tests/test_map_database.py delete mode 100644 odd_collector/adapters/superset/tests/test_map_table.py diff --git a/odd_collector/adapters/databricks/mappers/models.py b/odd_collector/adapters/databricks/mappers/models.py index 2c2e507b..bd1f6d4c 100644 --- a/odd_collector/adapters/databricks/mappers/models.py +++ b/odd_collector/adapters/databricks/mappers/models.py @@ -6,7 +6,7 @@ @dataclass(frozen=True) class DatabricksColumn(Column): - odd_metadata: dict[str, Any] = field(default_factory=dict) + ... @dataclass() diff --git a/odd_collector/adapters/databricks/mappers/table.py b/odd_collector/adapters/databricks/mappers/table.py index c62d6043..5ffaad8e 100644 --- a/odd_collector/adapters/databricks/mappers/table.py +++ b/odd_collector/adapters/databricks/mappers/table.py @@ -34,7 +34,7 @@ def get_column(table_info: dict, column_raw: dict) -> DatabricksColumn: name=column_raw.pop("name"), type=column_raw.get("type_text"), is_nullable=column_raw.get("nullable"), - odd_metadata=column_raw, + metadata=column_raw, ) return column diff --git a/odd_collector/adapters/superset/tests/__init__.py b/odd_collector/adapters/superset/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/odd_collector/adapters/superset/tests/raw_data.py b/odd_collector/adapters/superset/tests/raw_data.py deleted file mode 100644 index 6964227a..00000000 --- a/odd_collector/adapters/superset/tests/raw_data.py +++ /dev/null @@ -1,89 +0,0 @@ -datasets_nodes = [ - { - "changed_by": None, - "changed_by_name": "", - "changed_by_url": "", - "changed_on_delta_humanized": "an hour ago", - "changed_on_utc": "2022-08-31T21:50:35.320735+0000", - "database": {"database_name": "examples", "id": 1}, - "datasource_type": "table", - "default_endpoint": None, - "description": None, - "explore_url": "/explore/?dataset_type=table&dataset_id=20", - "extra": None, - "id": 20, - "kind": "physical", - "owners": [], - "schema": "public", - "sql": None, - "table_name": "threads", - }, - { - "changed_by": None, - "changed_by_name": "", - "changed_by_url": "", - "changed_on_delta_humanized": "an hour ago", - "changed_on_utc": "2022-08-31T21:50:31.009869+0000", - "database": {"database_name": "jj", "id": 2}, - "datasource_type": "table", - "default_endpoint": None, - "description": None, - "explore_url": "/explore/?dataset_type=table&dataset_id=14", - "extra": None, - "id": 14, - "kind": "physical", - "owners": [], - "schema": "public", - "sql": None, - "table_name": "channels", - }, - { - "changed_by": None, - "changed_by_name": "", - "changed_by_url": "", - "changed_on_delta_humanized": "an hour ago", - "changed_on_utc": "2022-08-31T21:50:31.009869+0000", - "database": {"database_name": "ppp", "id": 7}, - "datasource_type": "table", - "default_endpoint": None, - "description": None, - "explore_url": "/explore/?dataset_type=table&dataset_id=14", - "extra": None, - "id": 999, - "kind": "virtual", - "owners": [], - "schema": "public", - "sql": "SELECT 1", - "table_name": "channels", - }, - { - "changed_by": None, - "changed_by_name": "", - "changed_by_url": "", - "changed_on_delta_humanized": "an hour ago", - "changed_on_utc": "2022-08-31T21:50:35.320735+0000", - "database": {"database_name": "examples", "id": 1}, - "datasource_type": "table", - "default_endpoint": None, - "description": None, - "explore_url": "/explore/?dataset_type=table&dataset_id=20", - "extra": None, - "id": 7, - "kind": "physical", - "owners": [], - "schema": "public", - "sql": None, - "table_name": "pppp", - }, -] - -""" -{chart_id: {dashboard_id: dashboard_name}} -""" -nodes_with_chart_ids = {1: {10: "dash_10"}, 3: {13: "dash_13"}, 14: {10: "dash_10"}} - -chart_nodes = [ - {"datasource_id": 20, "id": 1}, - {"datasource_id": 14, "id": 14}, - {"datasource_id": 7, "id": 3}, -] diff --git a/odd_collector/adapters/superset/tests/test_map_column.py b/odd_collector/adapters/superset/tests/test_map_column.py deleted file mode 100644 index e87a9269..00000000 --- a/odd_collector/adapters/superset/tests/test_map_column.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest -from odd_models.models import Type -from oddrn_generator.generators import SupersetGenerator - -from odd_collector.adapters.superset.domain.column import Column -from odd_collector.adapters.superset.mappers.columns import map_column - - -@pytest.fixture -def generator(): - return SupersetGenerator(host_settings="host", databases="db", datasets="dataset") - - -def test_map_column(generator): - column = Column(id=1, name="Age", remote_type="TEXT") - data_entity = map_column(generator, column) - - assert ( - data_entity.oddrn - == "//superset/host/host/databases/db/datasets/dataset/columns/Age" - ) - - assert data_entity.name == "Age" - data_entity_type = data_entity.type - - assert data_entity_type.type == Type.TYPE_UNKNOWN - assert data_entity_type.logical_type == "TEXT" - assert data_entity_type.is_nullable == False diff --git a/odd_collector/adapters/superset/tests/test_map_dashboard.py b/odd_collector/adapters/superset/tests/test_map_dashboard.py deleted file mode 100644 index 13fa91b2..00000000 --- a/odd_collector/adapters/superset/tests/test_map_dashboard.py +++ /dev/null @@ -1,55 +0,0 @@ -import pytest -from oddrn_generator.generators import SupersetGenerator - -from odd_collector.adapters.superset.client import SupersetClient -from odd_collector.adapters.superset.domain.chart import Chart -from odd_collector.adapters.superset.domain.dataset import Dataset -from odd_collector.adapters.superset.mappers.dashboard import map_dashboard - -from .raw_data import chart_nodes, datasets_nodes, nodes_with_chart_ids - - -@pytest.fixture -def generator(): - return SupersetGenerator(host_settings="host") - - -def test_create_dashboards_entities(generator): - charts = [ - Chart( - id=chart_node["id"], - dataset_id=chart_node["datasource_id"], - dashboards_ids_names=nodes_with_chart_ids[chart_node["id"]], - ) - for chart_node in chart_nodes - ] - datasets = { - dataset[ - "id" - ]: f"//superset/host/host/databases/{dataset['database']['database_name']}/datasets/{dataset['table_name']}" - for dataset in datasets_nodes - } - dashboards = SupersetClient.extract_dashboards_from_charts(charts) - - dashboards_entities = [ - map_dashboard(generator, datasets, dashboard) for dashboard in dashboards - ] - assert len(dashboards_entities) == 2 - assert dashboards_entities[0].oddrn == "//superset/host/host/dashboards/dash_10" - assert len(dashboards_entities[0].data_consumer.inputs) == 2 - assert len(dashboards_entities[1].data_consumer.inputs) == 1 - - assert ( - dashboards_entities[0].data_consumer.inputs[0] - == "//superset/host/host/databases/examples/datasets/threads" - ) - assert ( - dashboards_entities[0].data_consumer.inputs[1] - == "//superset/host/host/databases/jj/datasets/channels" - ) - assert ( - dashboards_entities[1].data_consumer.inputs[0] - == "//superset/host/host/databases/examples/datasets/pppp" - ) - - pass diff --git a/odd_collector/adapters/superset/tests/test_map_database.py b/odd_collector/adapters/superset/tests/test_map_database.py deleted file mode 100644 index 7fcb1c69..00000000 --- a/odd_collector/adapters/superset/tests/test_map_database.py +++ /dev/null @@ -1,55 +0,0 @@ -from typing import Dict - -import pytest -from oddrn_generator.generators import SupersetGenerator - -from odd_collector.adapters.superset.domain.dataset import Dataset -from odd_collector.adapters.superset.mappers.databases import map_database - -from .raw_data import datasets_nodes - - -@pytest.fixture -def generator(): - return SupersetGenerator(host_settings="host") - - -def test_create_databases_entities(generator): - datasets = [ - Dataset( - id=dataset["id"], - name=dataset["table_name"], - db_id=dataset.get("database", {})["id"], - db_name=dataset.get("database", {})["database_name"], - kind=dataset["kind"], - schema="schema", - ) - for dataset in datasets_nodes - ] - - databases_ids_names: dict[int, str] = { - dataset.database_id: dataset.database_name for dataset in datasets - } - databases_entities = [ - map_database(generator, datasets, database_id, database_name) - for database_id, database_name in databases_ids_names.items() - ] - assert len(databases_entities) == 3 - assert databases_entities[0].oddrn == "//superset/host/host/databases/examples" - assert databases_entities[1].oddrn == "//superset/host/host/databases/jj" - assert databases_entities[2].oddrn == "//superset/host/host/databases/ppp" - - assert ( - databases_entities[0].data_entity_group.entities_list[0] - == "//superset/host/host/databases/examples/datasets/threads" - ) - - assert ( - databases_entities[0].data_entity_group.entities_list[1] - == "//superset/host/host/databases/examples/datasets/pppp" - ) - - assert ( - databases_entities[2].data_entity_group.entities_list[0] - == "//superset/host/host/databases/ppp/datasets/channels" - ) diff --git a/odd_collector/adapters/superset/tests/test_map_table.py b/odd_collector/adapters/superset/tests/test_map_table.py deleted file mode 100644 index d962e50d..00000000 --- a/odd_collector/adapters/superset/tests/test_map_table.py +++ /dev/null @@ -1,53 +0,0 @@ -import pytest -from odd_models.models import DataEntityType -from oddrn_generator.generators import SupersetGenerator - -from odd_collector.adapters.superset.domain.column import Column -from odd_collector.adapters.superset.domain.dataset import Dataset -from odd_collector.adapters.superset.mappers.datasets import map_table - - -@pytest.fixture -def generator(): - return SupersetGenerator(host_settings="host") - - -def test_table_to_data_entity(generator): - table = Dataset( - id=1, - name="table-name", - db_id=1, - db_name="db-name", - kind="virtual", - columns=[ - Column(id=1, name="Age", remote_type="TEXT"), - Column(id=3, name="Height", remote_type="INTEGER"), - ], - schema="schema", - metadata=[], - ) - - data_entity = map_table(generator, table) - - assert ( - data_entity.oddrn - == "//superset/host/host/databases/db-name/datasets/table-name" - ) - assert data_entity.name == "table-name" - assert data_entity.type == DataEntityType.VIEW - - assert data_entity.dataset is not None - - dataset = data_entity.dataset - assert ( - dataset.parent_oddrn - == "//superset/host/host/databases/db-name/datasets/table-name" - ) - assert len(data_entity.dataset.field_list) == 2 - - age_field = data_entity.dataset.field_list[0] - assert age_field.name == "Age" - assert ( - age_field.oddrn - == "//superset/host/host/databases/db-name/datasets/table-name/columns/Age" - ) diff --git a/tests/adapters/mode/mock_rest_client.py b/tests/adapters/mode/mock_rest_client.py index b8d7e3f9..756537d7 100644 --- a/tests/adapters/mode/mock_rest_client.py +++ b/tests/adapters/mode/mock_rest_client.py @@ -487,7 +487,7 @@ def __init__(self): }, } - async def fetch_async_response(self, session, request_args: RequestArgs): + async def fetch(self, session, request_args: RequestArgs): response = None if "/api/account/data_sources" in request_args.url: response = self.data_sources diff --git a/tests/integration/test_postgres.py b/tests/integration/test_postgres.py index 8461d506..b047d8b4 100644 --- a/tests/integration/test_postgres.py +++ b/tests/integration/test_postgres.py @@ -107,8 +107,8 @@ def test_data_base_service(data_entity_list: odd_models.DataEntityList): other_schema = find_by_name(data_entity_list, "other_schema") assert database is not None - public_schema is not None - other_schema is not None + assert public_schema is not None + assert other_schema is not None assert database.data_entity_group is not None assert len(database.data_entity_group.entities_list) == 2