From 3818dfb4558bfac86b4c190223915b229eb21d72 Mon Sep 17 00:00:00 2001 From: Pavel Makarichev Date: Mon, 2 Oct 2023 15:43:28 +0300 Subject: [PATCH] fix(mysql): add schema as params to columns query (#232) --- odd_collector/adapters/mysql/repository.py | 68 ++++---- tests/integration/test_mysql.py | 178 ++++++++++++++------- 2 files changed, 158 insertions(+), 88 deletions(-) diff --git a/odd_collector/adapters/mysql/repository.py b/odd_collector/adapters/mysql/repository.py index 30094ba7..1b1d842f 100644 --- a/odd_collector/adapters/mysql/repository.py +++ b/odd_collector/adapters/mysql/repository.py @@ -1,4 +1,5 @@ from dataclasses import asdict, dataclass +from typing import Optional import mysql.connector from odd_collector_sdk.errors import DataSourceConnectionError @@ -17,7 +18,7 @@ class ConnectionParams: port: int database: str user: str - password: str + password: Optional[str] ssl_disabled: bool @classmethod @@ -27,7 +28,7 @@ def from_config(cls, config: MySQLPlugin): port=config.port, database=config.database, user=config.user, - password=config.password.get_secret_value(), + password=config.password.get_secret_value() if config.password else None, ssl_disabled=config.ssl_disabled, ) @@ -83,7 +84,7 @@ def get_tables(self, database: str) -> list[Table]: def get_columns(self) -> list[Column]: with self.conn.cursor(dictionary=True) as cursor: - cursor.execute(self.columns_query) + cursor.execute(self.columns_query, (self.conn_params.database,)) columns = [] for raw in cursor.fetchall(): @@ -105,36 +106,36 @@ def get_columns(self) -> list[Column]: @property def tables_query(self): - return f""" - select t.table_catalog, - t.table_schema, - t.table_name, - t.table_type, - t.engine, - t.version, - t.row_format, - t.table_rows, - t.avg_row_length, - t.data_length, - t.max_data_length, - t.index_length, - t.data_free, - t.auto_increment, - t.create_time, - t.update_time, - t.check_time, - t.table_collation, - t.checksum, - t.create_options, - t.table_comment, - v.view_definition - from information_schema.tables t - left join information_schema.views v - on t.TABLE_CATALOG = v.TABLE_CATALOG and - t.TABLE_SCHEMA = v.TABLE_SCHEMA and - t.TABLE_NAME = v.TABLE_NAME - where t.table_schema = %s - order by t.table_catalog, t.table_schema, t.table_name + return """ + select t.table_catalog, + t.table_schema, + t.table_name, + t.table_type, + t.engine, + t.version, + t.row_format, + t.table_rows, + t.avg_row_length, + t.data_length, + t.max_data_length, + t.index_length, + t.data_free, + t.auto_increment, + t.create_time, + t.update_time, + t.check_time, + t.table_collation, + t.checksum, + t.create_options, + t.table_comment, + v.view_definition + from information_schema.tables t + left join information_schema.views v + on t.TABLE_CATALOG = v.TABLE_CATALOG and + t.TABLE_SCHEMA = v.TABLE_SCHEMA and + t.TABLE_NAME = v.TABLE_NAME + where t.table_schema = %s + order by t.table_catalog, t.table_schema, t.table_name """ @property @@ -164,5 +165,6 @@ def columns_query(self): generation_expression from information_schema.columns where table_schema not in ('information_schema', 'mysql', 'performance_schema', 'sys') + and table_schema = %s order by table_catalog, table_schema, table_name, ordinal_position; """ diff --git a/tests/integration/test_mysql.py b/tests/integration/test_mysql.py index 62c1e4a4..04020603 100644 --- a/tests/integration/test_mysql.py +++ b/tests/integration/test_mysql.py @@ -1,82 +1,150 @@ +import odd_models import pytest import sqlalchemy -from odd_models import DataEntity -from odd_models.models import DataEntityType from pydantic import SecretStr from testcontainers.mysql import MySqlContainer +from odd_collector.adapters.mysql.adapter import Adapter +from odd_collector.domain.plugin import MySQLPlugin from tests.integration.helpers import find_by_name, find_by_type -create_tables = """ -CREATE TABLE Persons ( - PersonID int, - LastName varchar(255), - FirstName varchar(255), - Address varchar(255), - City varchar(255) -);""" - -create_view = """ -CREATE VIEW persons_names AS -SELECT LastName, FirstName -FROM Persons -WHERE City = 'Sandnes'; -""" - -create_view_from_view = """ -CREATE VIEW persons_last_names AS -SELECT LastName -FROM persons_names; -""" -from odd_collector.adapters.mysql.adapter import Adapter -from odd_collector.domain.plugin import MySQLPlugin +def create_primary_schema(connection: sqlalchemy.engine.Connection): + create_tables = """ + CREATE TABLE Persons ( + PersonID int, + LastName varchar(255), + FirstName varchar(255), + Address varchar(255), + City varchar(255) + );""" + + create_view = """ + CREATE VIEW persons_names AS + SELECT LastName, FirstName + FROM Persons; + """ + + create_view_from_view = """ + CREATE VIEW persons_last_names AS + SELECT LastName + FROM persons_names; + """ + + connection.exec_driver_sql(create_tables) + connection.exec_driver_sql(create_view) + connection.exec_driver_sql(create_view_from_view) + + +def create_other_schema(connection: sqlalchemy.engine.Connection): + create_other_schema = """ + CREATE DATABASE `other_schema`; + """ + + create_tables = """ + CREATE TABLE `other_schema`.`Persons` ( + PersonID int, + LastName varchar(255), + FirstName varchar(255), + Address varchar(255), + City varchar(255) + );""" + + create_view = """ + CREATE VIEW `other_schema`.`persons_names` AS + SELECT LastName, FirstName + FROM `other_schema`.`Persons`; + """ + + create_view_from_view = """ + CREATE VIEW `other_schema`.`persons_last_names` AS + SELECT LastName + FROM `other_schema`.`persons_names`; + """ + + connection.exec_driver_sql(create_other_schema) + connection.exec_driver_sql(create_tables) + connection.exec_driver_sql(create_view) + connection.exec_driver_sql(create_view_from_view) + + +def entities_are_unique(entities: list[odd_models.DataEntity]): + return len(entities) == len({e.oddrn for e in entities}) -@pytest.mark.integration -def test_mysql(): - with MySqlContainer() as mysql: +@pytest.fixture(scope="module") +def data_entities() -> odd_models.DataEntityList: + with MySqlContainer(MYSQL_USER="root") as mysql: engine = sqlalchemy.create_engine(mysql.get_connection_url()) with engine.connect() as connection: - connection.exec_driver_sql(create_tables) - connection.exec_driver_sql(create_view) - connection.exec_driver_sql(create_view_from_view) + create_primary_schema(connection) + create_other_schema(connection) config = MySQLPlugin( type="mysql", name="test_mysql", database="test", password=SecretStr("test"), - user="test", + user="root", host=mysql.get_container_host_ip(), - port=mysql.get_exposed_port(3306), + port=int(mysql.get_exposed_port(3306)), ) - data_entities = Adapter(config).get_data_entity_list() - database_services: list[DataEntity] = find_by_type( - data_entities, DataEntityType.DATABASE_SERVICE - ) - assert len(database_services) == 1 - database_service = database_services[0] - assert len(database_service.data_entity_group.entities_list) == 3 + return Adapter(config).get_data_entity_list() + + +def test_entities_are_unique(data_entities: odd_models.DataEntityList): + assert entities_are_unique(data_entities.items) + + +def test_fetch_one_database_from_config(data_entities: odd_models.DataEntityList): + databases: list[odd_models.DataEntity] = find_by_type( + data_entities, odd_models.DataEntityType.DATABASE_SERVICE + ) + assert len(databases) == 1 + database = databases[0] + assert database.data_entity_group is not None + assert len(database.data_entity_group.entities_list) == 3 + + entities = database.data_entity_group.entities_list + assert len(entities) == len(set(entities)) + + +def test_fetch_only_one_table(data_entities: odd_models.DataEntityList): + tables = find_by_type(data_entities, odd_models.DataEntityType.TABLE) + + assert entities_are_unique(tables) + + table = tables[0] + + assert len(tables) == 1 + assert table.dataset is not None + assert len(table.dataset.field_list) == 5 + assert entities_are_unique(table.dataset.field_list) + + +def test_fetch_two_views(data_entities: odd_models.DataEntityList): + views = find_by_type(data_entities, odd_models.DataEntityType.VIEW) + assert len(views) == 2 + assert entities_are_unique(views) + + +def test_view_depends_on_table(data_entities: odd_models.DataEntityList): + table_entity = find_by_name(data_entities, "Persons") + entity = find_by_name(data_entities, "persons_names") - tables = find_by_type(data_entities, DataEntityType.TABLE) - assert len(tables) == 1 - table = tables[0] - assert len(table.dataset.field_list) == 5 + assert len(entity.dataset.field_list) == 2 + assert len(entity.data_transformer.inputs) == 1 + assert entity.data_transformer.inputs[0] == table_entity.oddrn - views = find_by_type(data_entities, DataEntityType.VIEW) - assert len(views) == 2 - persons_view = find_by_name(data_entities, "persons_names") - assert len(persons_view.dataset.field_list) == 2 - assert len(persons_view.data_transformer.inputs) == 1 - assert persons_view.data_transformer.inputs[0] == table.oddrn +def test_view_depends_on_view(data_entities: odd_models.DataEntityList): + view_entity = find_by_name(data_entities, "persons_names") + entity = find_by_name(data_entities, "persons_last_names") + assert len(entity.data_transformer.inputs) == 1 + assert entity.data_transformer.inputs[0] == view_entity.oddrn - last_names_view = find_by_name(data_entities, "persons_last_names") - assert len(last_names_view.dataset.field_list) == 1 - assert len(last_names_view.data_transformer.inputs) == 1 - assert last_names_view.data_transformer.inputs[0] == persons_view.oddrn - assert data_entities.json() +def test_decoding_data_entities(data_entities: odd_models.DataEntityList): + assert data_entities.json()