From 646b74cb8ed223c458b7ecdfd0b576865162af87 Mon Sep 17 00:00:00 2001 From: JulesHuisman Date: Sun, 12 Jun 2022 15:00:43 +0000 Subject: [PATCH 01/16] Add a max string length to migrations --- ...3e8639c6d2b_add_state_edit_to_job_state_enum.py | 4 ++-- ...3ea52e6d784_add_resource_type_to_embed_token.py | 2 +- .../versions/367228df6a43_add_trigger_to_job.py | 2 +- src/meltano/migrations/versions/6ef30ab7b8e5_.py | 14 +++++++------- .../versions/87d9638f6ac6_add_subscription.py | 9 +++++---- src/meltano/migrations/versions/b4c05e463b53_.py | 11 ++++++----- .../ceb00d7ff3bd_create_the_embedtoken_table.py | 2 +- 7 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py b/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py index c3bab8187e..081e065ce3 100644 --- a/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py +++ b/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py @@ -37,7 +37,7 @@ def upgrade(): op.alter_column( table_name="job", column_name="state", - type_=sa.types.String, + type_=sa.types.String(128), existing_type=sa.Enum(State, name="job_state"), ) @@ -56,5 +56,5 @@ def downgrade(): table_name="job", column_name="state", _type=sa.Enum(State, name="job_state"), - existing_type=sa.types.String, + existing_type=sa.types.String(128), ) diff --git a/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py b/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py index bc392b31f5..c270a3c214 100644 --- a/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py +++ b/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py @@ -18,7 +18,7 @@ def upgrade(): - op.add_column("embed_tokens", sa.Column("resource_type", sa.String())) + op.add_column("embed_tokens", sa.Column("resource_type", sa.String(128))) metadata = sa.MetaData(bind=op.get_bind()) Embed_Tokens = sa.Table("embed_tokens", metadata, autoload=True) diff --git a/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py b/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py index 53a2b77955..ff7640e24a 100644 --- a/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py +++ b/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py @@ -16,7 +16,7 @@ def upgrade(): - op.add_column("job", sa.Column("trigger", sa.String())) + op.add_column("job", sa.Column("trigger", sa.String(128))) def downgrade(): diff --git a/src/meltano/migrations/versions/6ef30ab7b8e5_.py b/src/meltano/migrations/versions/6ef30ab7b8e5_.py index 41a0c37464..2574aeffe1 100644 --- a/src/meltano/migrations/versions/6ef30ab7b8e5_.py +++ b/src/meltano/migrations/versions/6ef30ab7b8e5_.py @@ -19,9 +19,9 @@ def upgrade(): op.create_table( "user", sa.Column("id", sa.Integer, nullable=False), - sa.Column("username", sa.String), - sa.Column("email", sa.String), - sa.Column("password", sa.String), + sa.Column("username", sa.String(128)), + sa.Column("email", sa.String(128)), + sa.Column("password", sa.String(128)), sa.Column("active", sa.Boolean), sa.Column("confirmed_at", sa.DateTime, nullable=True), sa.PrimaryKeyConstraint("id"), @@ -33,7 +33,7 @@ def upgrade(): "role", sa.Column("id", sa.Integer, nullable=False), sa.Column("name", sa.String(80)), - sa.Column("description", sa.String, nullable=True), + sa.Column("description", sa.String(128), nullable=True), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("name"), ) @@ -52,8 +52,8 @@ def upgrade(): "role_permissions", sa.Column("id", sa.Integer, nullable=False), sa.Column("role_id", sa.Integer), - sa.Column("type", sa.String), - sa.Column("context", sa.String, nullable=True), + sa.Column("type", sa.String(128)), + sa.Column("context", sa.String(128), nullable=True), sa.ForeignKeyConstraint(["role_id"], ["role.id"]), sa.PrimaryKeyConstraint("id"), ) @@ -66,7 +66,7 @@ def upgrade(): sa.Column("provider_user_id", sa.Integer, nullable=True), sa.Column("access_token", sa.String(255)), sa.Column("created_at", sa.DateTime, nullable=True), - sa.Column("id_token", sa.String), + sa.Column("id_token", sa.String(128)), sa.ForeignKeyConstraint(["user_id"], ["user.id"]), sa.PrimaryKeyConstraint("id"), ) diff --git a/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py b/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py index 53f71476b6..93363cd28c 100644 --- a/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py +++ b/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py @@ -7,6 +7,7 @@ """ import sqlalchemy as sa from alembic import op + from meltano.migrations import GUID # revision identifiers, used by Alembic. @@ -20,10 +21,10 @@ def upgrade(): op.create_table( "subscriptions", sa.Column("id", GUID, primary_key=True), - sa.Column("recipient", sa.String(), nullable=False), - sa.Column("event_type", sa.String(), nullable=False), - sa.Column("source_type", sa.String(), nullable=True), - sa.Column("source_id", sa.String(), nullable=True), + sa.Column("recipient", sa.String(128), nullable=False), + sa.Column("event_type", sa.String(128), nullable=False), + sa.Column("source_type", sa.String(128), nullable=True), + sa.Column("source_id", sa.String(128), nullable=True), sa.Column("created_at", sa.DateTime), sa.UniqueConstraint("recipient", "event_type", "source_type", "source_id"), ) diff --git a/src/meltano/migrations/versions/b4c05e463b53_.py b/src/meltano/migrations/versions/b4c05e463b53_.py index f9cb73e1fd..17ae4f04ba 100644 --- a/src/meltano/migrations/versions/b4c05e463b53_.py +++ b/src/meltano/migrations/versions/b4c05e463b53_.py @@ -9,9 +9,10 @@ import sqlalchemy as sa from alembic import op -from meltano.migrations import IntFlag, JSONEncodedDict from sqlalchemy.ext.mutable import MutableDict +from meltano.migrations import IntFlag, JSONEncodedDict + # revision identifiers, used by Alembic. revision = "b4c05e463b53" down_revision = None @@ -32,7 +33,7 @@ def upgrade(): op.create_table( "job", sa.Column("id", sa.Integer, primary_key=True), - sa.Column("job_id", sa.String), + sa.Column("job_id", sa.String(128)), sa.Column("state", sa.Enum(State, name="job_state")), sa.Column("started_at", sa.DateTime), sa.Column("ended_at", sa.DateTime), @@ -42,10 +43,10 @@ def upgrade(): op.create_table( "plugin_settings", - sa.Column("label", sa.String(), nullable=True), + sa.Column("label", sa.String(128), nullable=True), sa.Column("description", sa.Text(), nullable=True), - sa.Column("name", sa.String(), nullable=False), - sa.Column("namespace", sa.String(), nullable=True), + sa.Column("name", sa.String(128), nullable=False), + sa.Column("namespace", sa.String(128), nullable=True), sa.Column("value", sa.PickleType(), nullable=True), sa.Column("enabled", sa.Boolean(), nullable=True), sa.PrimaryKeyConstraint("name", "namespace"), diff --git a/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py b/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py index 97d1767665..d26234d561 100644 --- a/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py +++ b/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py @@ -20,7 +20,7 @@ def upgrade(): "embed_tokens", sa.Column("id", sa.Integer, primary_key=True), sa.Column("token", sa.String(64), unique=True), - sa.Column("resource_id", sa.String(), nullable=False), + sa.Column("resource_id", sa.String(128), nullable=False), sa.Column("created_at", sa.DateTime), ) From 58355e53558c5079596bd8ed9374c4aaf741770f Mon Sep 17 00:00:00 2001 From: JulesHuisman Date: Sun, 12 Jun 2022 15:04:23 +0000 Subject: [PATCH 02/16] Added a string limit to JSONEncodedDict --- src/meltano/migrations/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meltano/migrations/__init__.py b/src/meltano/migrations/__init__.py index 9673cc397b..183d0b931e 100644 --- a/src/meltano/migrations/__init__.py +++ b/src/meltano/migrations/__init__.py @@ -12,5 +12,5 @@ # create a version for this type for all migrations that # would follow. GUID = types.GUID -JSONEncodedDict = types.JSONEncodedDict +JSONEncodedDict = types.JSONEncodedDict(length=128) IntFlag = types.IntFlag From 14bf82bb33580f5ca4f2d9609219da4f0dddeec5 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 20:46:39 +0000 Subject: [PATCH 03/16] Fixed typing to work with MSSQL backends --- src/meltano/migrations/__init__.py | 2 +- src/meltano/migrations/dialect_typing.py | 25 +++++++ ...9c6d2b_add_state_edit_to_job_state_enum.py | 4 +- ...e6d784_add_resource_type_to_embed_token.py | 2 +- .../367228df6a43_add_trigger_to_job.py | 2 +- .../migrations/versions/6ef30ab7b8e5_.py | 19 ++++-- .../versions/87d9638f6ac6_add_subscription.py | 14 ++-- .../a3e2b0a4937d_add_login_audit_columns.py | 7 +- .../migrations/versions/b4c05e463b53_.py | 17 +++-- ...eb00d7ff3bd_create_the_embedtoken_table.py | 9 ++- ...f49_add_last_heartbeat_at_column_to_job.py | 7 +- ...4fbabc3fed6_add_last_activity_at_column.py | 7 +- tests/conftest.py | 4 +- tests/fixtures/db/mssql.py | 68 +++++++++++++++++++ tests/fixtures/db/postgresql.py | 2 +- 15 files changed, 161 insertions(+), 28 deletions(-) create mode 100644 src/meltano/migrations/dialect_typing.py create mode 100644 tests/fixtures/db/mssql.py diff --git a/src/meltano/migrations/__init__.py b/src/meltano/migrations/__init__.py index 183d0b931e..9673cc397b 100644 --- a/src/meltano/migrations/__init__.py +++ b/src/meltano/migrations/__init__.py @@ -12,5 +12,5 @@ # create a version for this type for all migrations that # would follow. GUID = types.GUID -JSONEncodedDict = types.JSONEncodedDict(length=128) +JSONEncodedDict = types.JSONEncodedDict IntFlag = types.IntFlag diff --git a/src/meltano/migrations/dialect_typing.py b/src/meltano/migrations/dialect_typing.py new file mode 100644 index 0000000000..5b96f6d70c --- /dev/null +++ b/src/meltano/migrations/dialect_typing.py @@ -0,0 +1,25 @@ +from typing import Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.mssql import DATETIME2 +from sqlalchemy.engine import Connection + + +def get_dialect_name(op: op) -> str: + """ + Get the dialect name from the Alembic op. + """ + database_connection: Connection = op.get_bind() + return database_connection.dialect.name + +def datetime_for_dialect(dialect_name: str) -> Union[DATETIME2, sa.DateTime]: + """ + Get the datetime type for the given dialect. + """ + # We need to use the DATETIME2 type for MSSQL, because the + # default DATETIME type does not go back to the year 1. + if dialect_name is "mssql": + return DATETIME2 + + return sa.DateTime diff --git a/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py b/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py index 081e065ce3..c3bab8187e 100644 --- a/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py +++ b/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py @@ -37,7 +37,7 @@ def upgrade(): op.alter_column( table_name="job", column_name="state", - type_=sa.types.String(128), + type_=sa.types.String, existing_type=sa.Enum(State, name="job_state"), ) @@ -56,5 +56,5 @@ def downgrade(): table_name="job", column_name="state", _type=sa.Enum(State, name="job_state"), - existing_type=sa.types.String(128), + existing_type=sa.types.String, ) diff --git a/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py b/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py index c270a3c214..de0e34b4b9 100644 --- a/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py +++ b/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py @@ -18,7 +18,7 @@ def upgrade(): - op.add_column("embed_tokens", sa.Column("resource_type", sa.String(128))) + op.add_column("embed_tokens", sa.Column("resource_type", sa.String)) metadata = sa.MetaData(bind=op.get_bind()) Embed_Tokens = sa.Table("embed_tokens", metadata, autoload=True) diff --git a/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py b/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py index ff7640e24a..c31f7c2046 100644 --- a/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py +++ b/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py @@ -16,7 +16,7 @@ def upgrade(): - op.add_column("job", sa.Column("trigger", sa.String(128))) + op.add_column("job", sa.Column("trigger", sa.String)) def downgrade(): diff --git a/src/meltano/migrations/versions/6ef30ab7b8e5_.py b/src/meltano/migrations/versions/6ef30ab7b8e5_.py index 2574aeffe1..4fd28845de 100644 --- a/src/meltano/migrations/versions/6ef30ab7b8e5_.py +++ b/src/meltano/migrations/versions/6ef30ab7b8e5_.py @@ -8,6 +8,8 @@ import sqlalchemy as sa from alembic import op +from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name + # revision identifiers, used by Alembic. revision = "6ef30ab7b8e5" down_revision = "b4c05e463b53" @@ -16,12 +18,19 @@ def upgrade(): + conn = op.get_bind() + dialect_name = get_dialect_name(op) + datetime_type = datetime_for_dialect(dialect_name) + + # max_string_length = 128 if conn.dialect.name in {"mssql", "mysql"} else None + # max_string_length = 128 + op.create_table( "user", sa.Column("id", sa.Integer, nullable=False), sa.Column("username", sa.String(128)), sa.Column("email", sa.String(128)), - sa.Column("password", sa.String(128)), + sa.Column("password", sa.String), sa.Column("active", sa.Boolean), sa.Column("confirmed_at", sa.DateTime, nullable=True), sa.PrimaryKeyConstraint("id"), @@ -33,7 +42,7 @@ def upgrade(): "role", sa.Column("id", sa.Integer, nullable=False), sa.Column("name", sa.String(80)), - sa.Column("description", sa.String(128), nullable=True), + sa.Column("description", sa.String, nullable=True), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("name"), ) @@ -52,8 +61,8 @@ def upgrade(): "role_permissions", sa.Column("id", sa.Integer, nullable=False), sa.Column("role_id", sa.Integer), - sa.Column("type", sa.String(128)), - sa.Column("context", sa.String(128), nullable=True), + sa.Column("type", sa.String), + sa.Column("context", sa.String, nullable=True), sa.ForeignKeyConstraint(["role_id"], ["role.id"]), sa.PrimaryKeyConstraint("id"), ) @@ -66,7 +75,7 @@ def upgrade(): sa.Column("provider_user_id", sa.Integer, nullable=True), sa.Column("access_token", sa.String(255)), sa.Column("created_at", sa.DateTime, nullable=True), - sa.Column("id_token", sa.String(128)), + sa.Column("id_token", sa.String), sa.ForeignKeyConstraint(["user_id"], ["user.id"]), sa.PrimaryKeyConstraint("id"), ) diff --git a/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py b/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py index 93363cd28c..0af673e94b 100644 --- a/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py +++ b/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py @@ -9,6 +9,7 @@ from alembic import op from meltano.migrations import GUID +from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name # revision identifiers, used by Alembic. revision = "87d9638f6ac6" @@ -18,14 +19,17 @@ def upgrade(): + dialect_name = get_dialect_name(op) + datetime_type = datetime_for_dialect(dialect_name) + op.create_table( "subscriptions", sa.Column("id", GUID, primary_key=True), - sa.Column("recipient", sa.String(128), nullable=False), - sa.Column("event_type", sa.String(128), nullable=False), - sa.Column("source_type", sa.String(128), nullable=True), - sa.Column("source_id", sa.String(128), nullable=True), - sa.Column("created_at", sa.DateTime), + sa.Column("recipient", sa.String(255), nullable=False), + sa.Column("event_type", sa.String(255), nullable=False), + sa.Column("source_type", sa.String(255), nullable=True), + sa.Column("source_id", sa.String(255), nullable=True), + sa.Column("created_at", datetime_type), sa.UniqueConstraint("recipient", "event_type", "source_type", "source_id"), ) diff --git a/src/meltano/migrations/versions/a3e2b0a4937d_add_login_audit_columns.py b/src/meltano/migrations/versions/a3e2b0a4937d_add_login_audit_columns.py index 26a6d5a038..db551d78b5 100644 --- a/src/meltano/migrations/versions/a3e2b0a4937d_add_login_audit_columns.py +++ b/src/meltano/migrations/versions/a3e2b0a4937d_add_login_audit_columns.py @@ -8,6 +8,8 @@ import sqlalchemy as sa from alembic import op +from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name + # revision identifiers, used by Alembic. revision = "a3e2b0a4937d" down_revision = "53e97221d99f" @@ -16,7 +18,10 @@ def upgrade(): - op.add_column("user", sa.Column("last_login_at", sa.DateTime(), nullable=True)) + dialect_name = get_dialect_name(op) + datetime_type = datetime_for_dialect(dialect_name) + + op.add_column("user", sa.Column("last_login_at", datetime_type, nullable=True)) op.add_column("user", sa.Column("login_count", sa.Integer, default=0)) diff --git a/src/meltano/migrations/versions/b4c05e463b53_.py b/src/meltano/migrations/versions/b4c05e463b53_.py index 17ae4f04ba..4a8b5a8d3a 100644 --- a/src/meltano/migrations/versions/b4c05e463b53_.py +++ b/src/meltano/migrations/versions/b4c05e463b53_.py @@ -9,9 +9,11 @@ import sqlalchemy as sa from alembic import op +from sqlalchemy.dialects.mssql import DATETIME2 from sqlalchemy.ext.mutable import MutableDict from meltano.migrations import IntFlag, JSONEncodedDict +from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name # revision identifiers, used by Alembic. revision = "b4c05e463b53" @@ -30,23 +32,26 @@ class State(Enum): def upgrade(): + dialect_name = get_dialect_name(op) + datetime_type = datetime_for_dialect(dialect_name) + op.create_table( "job", sa.Column("id", sa.Integer, primary_key=True), - sa.Column("job_id", sa.String(128)), + sa.Column("job_id", sa.String), sa.Column("state", sa.Enum(State, name="job_state")), - sa.Column("started_at", sa.DateTime), - sa.Column("ended_at", sa.DateTime), + sa.Column("started_at", datetime_type), + sa.Column("ended_at", datetime_type), sa.Column("payload", MutableDict.as_mutable(JSONEncodedDict)), sa.Column("payload_flags", IntFlag, default=0), ) op.create_table( "plugin_settings", - sa.Column("label", sa.String(128), nullable=True), + sa.Column("label", sa.String, nullable=True), sa.Column("description", sa.Text(), nullable=True), - sa.Column("name", sa.String(128), nullable=False), - sa.Column("namespace", sa.String(128), nullable=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("namespace", sa.String(255), nullable=True), sa.Column("value", sa.PickleType(), nullable=True), sa.Column("enabled", sa.Boolean(), nullable=True), sa.PrimaryKeyConstraint("name", "namespace"), diff --git a/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py b/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py index d26234d561..dc49c84d02 100644 --- a/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py +++ b/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py @@ -8,6 +8,8 @@ import sqlalchemy as sa from alembic import op +from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name + # revision identifiers, used by Alembic. revision = "ceb00d7ff3bd" down_revision = "990c0665f3ce" @@ -16,12 +18,15 @@ def upgrade(): + dialect_name = get_dialect_name(op) + datetime_type = datetime_for_dialect(dialect_name) + op.create_table( "embed_tokens", sa.Column("id", sa.Integer, primary_key=True), sa.Column("token", sa.String(64), unique=True), - sa.Column("resource_id", sa.String(128), nullable=False), - sa.Column("created_at", sa.DateTime), + sa.Column("resource_id", sa.String, nullable=False), + sa.Column("created_at", datetime_type), ) diff --git a/src/meltano/migrations/versions/d135f52a6f49_add_last_heartbeat_at_column_to_job.py b/src/meltano/migrations/versions/d135f52a6f49_add_last_heartbeat_at_column_to_job.py index 2c66e3090e..a9715585d7 100644 --- a/src/meltano/migrations/versions/d135f52a6f49_add_last_heartbeat_at_column_to_job.py +++ b/src/meltano/migrations/versions/d135f52a6f49_add_last_heartbeat_at_column_to_job.py @@ -8,6 +8,8 @@ import sqlalchemy as sa from alembic import op +from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name + # revision identifiers, used by Alembic. revision = "d135f52a6f49" down_revision = "e4fbabc3fed6" @@ -16,7 +18,10 @@ def upgrade(): - op.add_column("job", sa.Column("last_heartbeat_at", sa.DateTime(), nullable=True)) + dialect_name = get_dialect_name(op) + datetime_type = datetime_for_dialect(dialect_name) + + op.add_column("job", sa.Column("last_heartbeat_at", datetime_type, nullable=True)) def downgrade(): diff --git a/src/meltano/migrations/versions/e4fbabc3fed6_add_last_activity_at_column.py b/src/meltano/migrations/versions/e4fbabc3fed6_add_last_activity_at_column.py index 3fcffc49d7..a04e6c3517 100644 --- a/src/meltano/migrations/versions/e4fbabc3fed6_add_last_activity_at_column.py +++ b/src/meltano/migrations/versions/e4fbabc3fed6_add_last_activity_at_column.py @@ -8,6 +8,8 @@ import sqlalchemy as sa from alembic import op +from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name + # revision identifiers, used by Alembic. revision = "e4fbabc3fed6" down_revision = "367228df6a43" @@ -16,7 +18,10 @@ def upgrade(): - op.add_column("user", sa.Column("last_activity_at", sa.DateTime(), nullable=True)) + dialect_name = get_dialect_name(op) + datetime_type = datetime_for_dialect(dialect_name) + + op.add_column("user", sa.Column("last_activity_at", datetime_type, nullable=True)) def downgrade(): diff --git a/tests/conftest.py b/tests/conftest.py index b6fb753d3f..ee7eeaa76d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,10 +31,12 @@ pytest_plugins.append("fixtures.db.sqlite") elif PYTEST_BACKEND == "postgresql": pytest_plugins.append("fixtures.db.postgresql") +elif PYTEST_BACKEND == "mssql": + pytest_plugins.append("fixtures.db.mssql") else: raise Exception(f"Unsuported backend: {PYTEST_BACKEND}.") -BACKEND = ["sqlite", "postgresql"] +BACKEND = ["sqlite", "postgresql", "mssql"] def pytest_runtest_setup(item): diff --git a/tests/fixtures/db/mssql.py b/tests/fixtures/db/mssql.py new file mode 100644 index 0000000000..063a94f227 --- /dev/null +++ b/tests/fixtures/db/mssql.py @@ -0,0 +1,68 @@ +import contextlib +import logging +import os + +import pytest +import sqlalchemy as sa +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL + + +def recreate_database(engine, db_name): + """ + Drop & Create a new database. We need to use the master connection to do this. + """ + with contextlib.suppress(sa.exc.ProgrammingError): + engine.execute(text(f""" + USE master; + ALTER DATABASE [{db_name}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; + DROP DATABASE [{db_name}]; + """)) + + with contextlib.suppress(sa.exc.ProgrammingError): + engine.execute(text(f"CREATE DATABASE {db_name}")) + +def create_connection_url(host: str, port: int, user: str, password: str, database: str) -> URL: + """ + Create a MSSQL connection URL for the given parameters. + """ + connection_url = sa.engine.URL.create( + "mssql+pyodbc", + username=user, + password=password, + host=host, + port=port, + database=database, + query={ + "driver": "ODBC Driver 18 for SQL Server", + "LongAsMax": "Yes", + "TrustServerCertificate": "Yes", + }, + ) + + return connection_url + +@pytest.fixture(scope="session") +def engine_uri(): + host = os.getenv("MSSQL_ADDRESS") + port = os.getenv("MSSQL_PORT", 1433) + user = os.getenv("MSSQL_USER") + password = os.getenv("MSSQL_PASSWORD") + database = os.getenv("MSSQL_DB", "pytest_meltano") + + # create the database + engine_uri = create_connection_url(host, port, user, password, database) + engine = create_engine(engine_uri, isolation_level="AUTOCOMMIT") + recreate_database(engine, database) + + return str(engine_uri) + + +@pytest.fixture() +def pg_stats(request, session): + yield + + from meltano.core.job import Job + + jobs = session.query(Job).all() + print(f"{request.node.name} created {len(jobs)} Job.") diff --git a/tests/fixtures/db/postgresql.py b/tests/fixtures/db/postgresql.py index 924891dffc..a9ae990b5c 100644 --- a/tests/fixtures/db/postgresql.py +++ b/tests/fixtures/db/postgresql.py @@ -24,7 +24,7 @@ def engine_uri(): port = os.getenv("POSTGRES_PORT", 5432) user = os.getenv("POSTGRES_USER") password = os.getenv("POSTGRES_PASSWORD") - database = "pytest_meltano" + database = os.getenv("POSTGRES_DB", "pytest_meltano") # create the database engine_uri = f"postgresql://{user}:{password}@{host}:{port}/postgres" From fe63148092dab064380389d0edcd69ca6f8674e0 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 20:55:19 +0000 Subject: [PATCH 04/16] Added two more dynamic datetime types --- src/meltano/migrations/versions/6ef30ab7b8e5_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/meltano/migrations/versions/6ef30ab7b8e5_.py b/src/meltano/migrations/versions/6ef30ab7b8e5_.py index 4fd28845de..894507c66a 100644 --- a/src/meltano/migrations/versions/6ef30ab7b8e5_.py +++ b/src/meltano/migrations/versions/6ef30ab7b8e5_.py @@ -32,7 +32,7 @@ def upgrade(): sa.Column("email", sa.String(128)), sa.Column("password", sa.String), sa.Column("active", sa.Boolean), - sa.Column("confirmed_at", sa.DateTime, nullable=True), + sa.Column("confirmed_at", datetime_type, nullable=True), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("email"), ) @@ -74,7 +74,7 @@ def upgrade(): sa.Column("provider_id", sa.String(255)), sa.Column("provider_user_id", sa.Integer, nullable=True), sa.Column("access_token", sa.String(255)), - sa.Column("created_at", sa.DateTime, nullable=True), + sa.Column("created_at", datetime_type, nullable=True), sa.Column("id_token", sa.String), sa.ForeignKeyConstraint(["user_id"], ["user.id"]), sa.PrimaryKeyConstraint("id"), From 42c46911dd4a9abf4e68433e87b93be8ee78fe05 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 21:21:14 +0000 Subject: [PATCH 05/16] Altered migrations to work with MYSQL --- .../migrations/{ => utils}/dialect_typing.py | 14 ++++- .../migrations/versions/6ef30ab7b8e5_.py | 24 ++++---- .../versions/87d9638f6ac6_add_subscription.py | 5 +- .../a3e2b0a4937d_add_login_audit_columns.py | 5 +- .../migrations/versions/b4c05e463b53_.py | 13 ++-- ...eb00d7ff3bd_create_the_embedtoken_table.py | 5 +- ...f49_add_last_heartbeat_at_column_to_job.py | 5 +- ...4fbabc3fed6_add_last_activity_at_column.py | 5 +- tests/conftest.py | 4 +- tests/fixtures/db/mysql.py | 59 +++++++++++++++++++ 10 files changed, 116 insertions(+), 23 deletions(-) rename src/meltano/migrations/{ => utils}/dialect_typing.py (64%) create mode 100644 tests/fixtures/db/mysql.py diff --git a/src/meltano/migrations/dialect_typing.py b/src/meltano/migrations/utils/dialect_typing.py similarity index 64% rename from src/meltano/migrations/dialect_typing.py rename to src/meltano/migrations/utils/dialect_typing.py index 5b96f6d70c..daa37c416c 100644 --- a/src/meltano/migrations/dialect_typing.py +++ b/src/meltano/migrations/utils/dialect_typing.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import Optional, Union import sqlalchemy as sa from alembic import op @@ -19,7 +19,17 @@ def datetime_for_dialect(dialect_name: str) -> Union[DATETIME2, sa.DateTime]: """ # We need to use the DATETIME2 type for MSSQL, because the # default DATETIME type does not go back to the year 1. - if dialect_name is "mssql": + if dialect_name == "mssql": return DATETIME2 return sa.DateTime + +def max_string_length_for_dialect(dialect_name: str) -> Optional[int]: + """ + Get the maximum string length for the given dialect. + We need to limit the size of the string to avoid MySQL throwing an error. + """ + if dialect_name == "mysql": + return 1024 + + return None diff --git a/src/meltano/migrations/versions/6ef30ab7b8e5_.py b/src/meltano/migrations/versions/6ef30ab7b8e5_.py index 894507c66a..3a4a2cb2ee 100644 --- a/src/meltano/migrations/versions/6ef30ab7b8e5_.py +++ b/src/meltano/migrations/versions/6ef30ab7b8e5_.py @@ -8,7 +8,11 @@ import sqlalchemy as sa from alembic import op -from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name +from meltano.migrations.utils.dialect_typing import ( + datetime_for_dialect, + get_dialect_name, + max_string_length_for_dialect, +) # revision identifiers, used by Alembic. revision = "6ef30ab7b8e5" @@ -21,16 +25,14 @@ def upgrade(): conn = op.get_bind() dialect_name = get_dialect_name(op) datetime_type = datetime_for_dialect(dialect_name) - - # max_string_length = 128 if conn.dialect.name in {"mssql", "mysql"} else None - # max_string_length = 128 + max_string_length = max_string_length_for_dialect(dialect_name) op.create_table( "user", sa.Column("id", sa.Integer, nullable=False), - sa.Column("username", sa.String(128)), - sa.Column("email", sa.String(128)), - sa.Column("password", sa.String), + sa.Column("username", sa.String(255)), + sa.Column("email", sa.String(255)), + sa.Column("password", sa.String(max_string_length)), sa.Column("active", sa.Boolean), sa.Column("confirmed_at", datetime_type, nullable=True), sa.PrimaryKeyConstraint("id"), @@ -42,7 +44,7 @@ def upgrade(): "role", sa.Column("id", sa.Integer, nullable=False), sa.Column("name", sa.String(80)), - sa.Column("description", sa.String, nullable=True), + sa.Column("description", sa.String(max_string_length), nullable=True), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("name"), ) @@ -61,8 +63,8 @@ def upgrade(): "role_permissions", sa.Column("id", sa.Integer, nullable=False), sa.Column("role_id", sa.Integer), - sa.Column("type", sa.String), - sa.Column("context", sa.String, nullable=True), + sa.Column("type", sa.String(max_string_length)), + sa.Column("context", sa.String(max_string_length), nullable=True), sa.ForeignKeyConstraint(["role_id"], ["role.id"]), sa.PrimaryKeyConstraint("id"), ) @@ -75,7 +77,7 @@ def upgrade(): sa.Column("provider_user_id", sa.Integer, nullable=True), sa.Column("access_token", sa.String(255)), sa.Column("created_at", datetime_type, nullable=True), - sa.Column("id_token", sa.String), + sa.Column("id_token", sa.String(max_string_length)), sa.ForeignKeyConstraint(["user_id"], ["user.id"]), sa.PrimaryKeyConstraint("id"), ) diff --git a/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py b/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py index 0af673e94b..8d8eb4299c 100644 --- a/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py +++ b/src/meltano/migrations/versions/87d9638f6ac6_add_subscription.py @@ -9,7 +9,10 @@ from alembic import op from meltano.migrations import GUID -from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name +from meltano.migrations.utils.dialect_typing import ( + datetime_for_dialect, + get_dialect_name, +) # revision identifiers, used by Alembic. revision = "87d9638f6ac6" diff --git a/src/meltano/migrations/versions/a3e2b0a4937d_add_login_audit_columns.py b/src/meltano/migrations/versions/a3e2b0a4937d_add_login_audit_columns.py index db551d78b5..75cad840f8 100644 --- a/src/meltano/migrations/versions/a3e2b0a4937d_add_login_audit_columns.py +++ b/src/meltano/migrations/versions/a3e2b0a4937d_add_login_audit_columns.py @@ -8,7 +8,10 @@ import sqlalchemy as sa from alembic import op -from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name +from meltano.migrations.utils.dialect_typing import ( + datetime_for_dialect, + get_dialect_name, +) # revision identifiers, used by Alembic. revision = "a3e2b0a4937d" diff --git a/src/meltano/migrations/versions/b4c05e463b53_.py b/src/meltano/migrations/versions/b4c05e463b53_.py index 4a8b5a8d3a..6f8a2c6ec3 100644 --- a/src/meltano/migrations/versions/b4c05e463b53_.py +++ b/src/meltano/migrations/versions/b4c05e463b53_.py @@ -13,7 +13,11 @@ from sqlalchemy.ext.mutable import MutableDict from meltano.migrations import IntFlag, JSONEncodedDict -from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name +from meltano.migrations.utils.dialect_typing import ( + datetime_for_dialect, + get_dialect_name, + max_string_length_for_dialect, +) # revision identifiers, used by Alembic. revision = "b4c05e463b53" @@ -34,21 +38,22 @@ class State(Enum): def upgrade(): dialect_name = get_dialect_name(op) datetime_type = datetime_for_dialect(dialect_name) + max_string_length = max_string_length_for_dialect(dialect_name) op.create_table( "job", sa.Column("id", sa.Integer, primary_key=True), - sa.Column("job_id", sa.String), + sa.Column("job_id", sa.String(max_string_length)), sa.Column("state", sa.Enum(State, name="job_state")), sa.Column("started_at", datetime_type), sa.Column("ended_at", datetime_type), - sa.Column("payload", MutableDict.as_mutable(JSONEncodedDict)), + sa.Column("payload", MutableDict.as_mutable(JSONEncodedDict(max_string_length))), sa.Column("payload_flags", IntFlag, default=0), ) op.create_table( "plugin_settings", - sa.Column("label", sa.String, nullable=True), + sa.Column("label", datetime_type, nullable=True), sa.Column("description", sa.Text(), nullable=True), sa.Column("name", sa.String(255), nullable=False), sa.Column("namespace", sa.String(255), nullable=True), diff --git a/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py b/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py index dc49c84d02..2733848608 100644 --- a/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py +++ b/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py @@ -8,7 +8,10 @@ import sqlalchemy as sa from alembic import op -from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name +from meltano.migrations.utils.dialect_typing import ( + datetime_for_dialect, + get_dialect_name, +) # revision identifiers, used by Alembic. revision = "ceb00d7ff3bd" diff --git a/src/meltano/migrations/versions/d135f52a6f49_add_last_heartbeat_at_column_to_job.py b/src/meltano/migrations/versions/d135f52a6f49_add_last_heartbeat_at_column_to_job.py index a9715585d7..b4cba0bf4c 100644 --- a/src/meltano/migrations/versions/d135f52a6f49_add_last_heartbeat_at_column_to_job.py +++ b/src/meltano/migrations/versions/d135f52a6f49_add_last_heartbeat_at_column_to_job.py @@ -8,7 +8,10 @@ import sqlalchemy as sa from alembic import op -from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name +from meltano.migrations.utils.dialect_typing import ( + datetime_for_dialect, + get_dialect_name, +) # revision identifiers, used by Alembic. revision = "d135f52a6f49" diff --git a/src/meltano/migrations/versions/e4fbabc3fed6_add_last_activity_at_column.py b/src/meltano/migrations/versions/e4fbabc3fed6_add_last_activity_at_column.py index a04e6c3517..9cd6398df6 100644 --- a/src/meltano/migrations/versions/e4fbabc3fed6_add_last_activity_at_column.py +++ b/src/meltano/migrations/versions/e4fbabc3fed6_add_last_activity_at_column.py @@ -8,7 +8,10 @@ import sqlalchemy as sa from alembic import op -from meltano.migrations.dialect_typing import datetime_for_dialect, get_dialect_name +from meltano.migrations.utils.dialect_typing import ( + datetime_for_dialect, + get_dialect_name, +) # revision identifiers, used by Alembic. revision = "e4fbabc3fed6" diff --git a/tests/conftest.py b/tests/conftest.py index ee7eeaa76d..d9d36e7f50 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,10 +33,12 @@ pytest_plugins.append("fixtures.db.postgresql") elif PYTEST_BACKEND == "mssql": pytest_plugins.append("fixtures.db.mssql") +elif PYTEST_BACKEND == "mysql": + pytest_plugins.append("fixtures.db.mysql") else: raise Exception(f"Unsuported backend: {PYTEST_BACKEND}.") -BACKEND = ["sqlite", "postgresql", "mssql"] +BACKEND = ["sqlite", "postgresql", "mssql", "mysql"] def pytest_runtest_setup(item): diff --git a/tests/fixtures/db/mysql.py b/tests/fixtures/db/mysql.py new file mode 100644 index 0000000000..1798d89cb5 --- /dev/null +++ b/tests/fixtures/db/mysql.py @@ -0,0 +1,59 @@ +import contextlib +import logging +import os + +import pytest +import sqlalchemy as sa +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL + + +def recreate_database(engine, db_name): + """ + Drop & Create a new database. + """ + with contextlib.suppress(sa.exc.ProgrammingError): + engine.execute(text(f"DROP DATABASE {db_name}")) + + with contextlib.suppress(sa.exc.ProgrammingError): + engine.execute(text(f"CREATE DATABASE {db_name}")) + +def create_connection_url(host: str, port: int, user: str, password: str, database: str) -> URL: + """ + Create a MSSQL connection URL for the given parameters. + """ + connection_url = sa.engine.URL.create( + "mysql+pymysql", + username=user, + password=password, + host=host, + port=port, + database=database, + ) + + return connection_url + +@pytest.fixture(scope="session") +def engine_uri(): + host = os.getenv("MYSQL_ADDRESS") + port = os.getenv("MYSQL_PORT", 3306) + user = os.getenv("MYSQL_USER") + password = os.getenv("MYSQL_PASSWORD") + database = os.getenv("MYSQL_DB", "pytest_meltano") + + # create the database + engine_uri = create_connection_url(host, port, user, password, database) + engine = create_engine(engine_uri, isolation_level="AUTOCOMMIT") + recreate_database(engine, database) + + return str(engine_uri) + + +@pytest.fixture() +def pg_stats(request, session): + yield + + from meltano.core.job import Job + + jobs = session.query(Job).all() + print(f"{request.node.name} created {len(jobs)} Job.") From 4778d4808d643d3c20d6641e403299b999da99f7 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 21:30:55 +0000 Subject: [PATCH 06/16] Added MSSQL testing to Github workflow --- .github/workflows/test.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f61718d266..a0bb3c02cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,6 +18,9 @@ jobs: - { python-version: "3.7", os: "ubuntu-latest", backend-db: postgresql } - { python-version: "3.8", os: "ubuntu-latest", backend-db: postgresql } - { python-version: "3.9", os: "ubuntu-latest", backend-db: postgresql } + - { python-version: "3.7", os: "ubuntu-latest", backend-db: mssql } + - { python-version: "3.8", os: "ubuntu-latest", backend-db: mssql } + - { python-version: "3.9", os: "ubuntu-latest", backend-db: mssql } fail-fast: false name: "Pytest on py${{ matrix.python-version }} (OS: ${{ matrix.os }}, DB: ${{ matrix.backend-db }})" @@ -42,6 +45,22 @@ jobs: ports: # Maps tcp port 5432 on service container to the host - 5432:5432 + mssql: + # SQL Server image + image: mcr.microsoft.com/mssql/server:2019-latest + # Provide the password for mssql + env: + SA_PASSWORD: Meltan0admin + ACCEPT_EULA: "Y" + # Set health checks to wait until mssql has started + options: >- + --health-cmd /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Meltan0admin -Q "SELECT 1" -b -o /dev/null + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 1433 on service container to the host + - 1433:1433 env: PYTEST_MARKERS: not concurrent @@ -86,6 +105,13 @@ jobs: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: pytest_warehouse + + # MSSQL backend + MSSQL_ADDRESS: localhost + MSSQL_PORT: 1433 + MSSQL_USER: sa + MSSQL_PASSWORD: Meltan0admin + MSSQL_DB: pytest_warehouse run: | poetry run coverage run --parallel -m pytest -m "$PYTEST_MARKERS" From 99e4d49cd4e345b57c52d72ed0f477102db5e85a Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 21:35:37 +0000 Subject: [PATCH 07/16] Added MySQL to the Github workflow --- .github/workflows/test.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0bb3c02cc..25766d8783 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,9 @@ jobs: - { python-version: "3.7", os: "ubuntu-latest", backend-db: mssql } - { python-version: "3.8", os: "ubuntu-latest", backend-db: mssql } - { python-version: "3.9", os: "ubuntu-latest", backend-db: mssql } + - { python-version: "3.7", os: "ubuntu-latest", backend-db: mysql } + - { python-version: "3.8", os: "ubuntu-latest", backend-db: mysql } + - { python-version: "3.9", os: "ubuntu-latest", backend-db: mysql } fail-fast: false name: "Pytest on py${{ matrix.python-version }} (OS: ${{ matrix.os }}, DB: ${{ matrix.backend-db }})" @@ -54,13 +57,28 @@ jobs: ACCEPT_EULA: "Y" # Set health checks to wait until mssql has started options: >- - --health-cmd /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Meltan0admin -Q "SELECT 1" -b -o /dev/null + --health-cmd /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $$SA_PASSWORD -Q "SELECT 1" -b -o /dev/null --health-interval 10s --health-timeout 5s --health-retries 5 ports: # Maps tcp port 1433 on service container to the host - 1433:1433 + mysql: + # Mysql image + image: mysql:8-oracle + # Provide the password for mysql + env: + MYSQL_ROOT_PASSWORD: mysql + # Set health checks to wait until mysql has started + options: >- + --health-cmd mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 3306 on service container to the host + - 3306:3306 env: PYTEST_MARKERS: not concurrent @@ -112,6 +130,13 @@ jobs: MSSQL_USER: sa MSSQL_PASSWORD: Meltan0admin MSSQL_DB: pytest_warehouse + + # MySQL backend + MYSQL_ADDRESS: localhost + MYSQL_PORT: 3306 + MYSQL_USER: root + MYSQL_PASSWORD: mysql + MYSQL_DB: pytest_warehouse run: | poetry run coverage run --parallel -m pytest -m "$PYTEST_MARKERS" From def2f987ab338d4ad41a98c18b529c886e98464d Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 21:44:32 +0000 Subject: [PATCH 08/16] Hard coded string lengths in migrations --- src/meltano/migrations/versions/6ef30ab7b8e5_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/meltano/migrations/versions/6ef30ab7b8e5_.py b/src/meltano/migrations/versions/6ef30ab7b8e5_.py index 3a4a2cb2ee..90dffea810 100644 --- a/src/meltano/migrations/versions/6ef30ab7b8e5_.py +++ b/src/meltano/migrations/versions/6ef30ab7b8e5_.py @@ -32,7 +32,7 @@ def upgrade(): sa.Column("id", sa.Integer, nullable=False), sa.Column("username", sa.String(255)), sa.Column("email", sa.String(255)), - sa.Column("password", sa.String(max_string_length)), + sa.Column("password", sa.String(255)), sa.Column("active", sa.Boolean), sa.Column("confirmed_at", datetime_type, nullable=True), sa.PrimaryKeyConstraint("id"), @@ -44,7 +44,7 @@ def upgrade(): "role", sa.Column("id", sa.Integer, nullable=False), sa.Column("name", sa.String(80)), - sa.Column("description", sa.String(max_string_length), nullable=True), + sa.Column("description", sa.String(255), nullable=True), sa.PrimaryKeyConstraint("id"), sa.UniqueConstraint("name"), ) From 5f2e9d704cd622259e99dbf4a9222527f46ef3f1 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 22:02:20 +0000 Subject: [PATCH 09/16] Moved MSSQL testing to github action run --- .github/workflows/test.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dda92b5bd3..d72067b36c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,6 +125,20 @@ jobs: --health-retries 5 postgres:11 + - name: Start MSSQL Container + if: always() && (matrix.backend-db == 'mssql') + run: > + docker run -d + -p "1433:1433" + -e "SA_PASSWORD=Meltan0admin" + -e "ACCEPT_EULA=Y" + --name mssql + --health-cmd "/opt/mssql-tools/bin/sqlcmd -U -P Meltan0admin -Q 'select 1' -b -o /dev/null" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + mcr.microsoft.com/mssql/server:2019-latest + - name: Check running containers run: | docker ps -a From 8e302b0c42b543619973b2ccf0b0107d11f66644 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 22:04:11 +0000 Subject: [PATCH 10/16] Moved Mysql to Github run action --- .github/workflows/test.yml | 64 ++++++++------------------------------ 1 file changed, 13 insertions(+), 51 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d72067b36c..c4835d3e11 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,57 +28,6 @@ jobs: name: "Pytest on py${{ matrix.python-version }} (OS: ${{ matrix.os }}, DB: ${{ matrix.backend-db }})" runs-on: ${{ matrix.os }} - services: - # Running a Postgres service even when it's not needed - # (in tests using SQLite as backend) is wasteful but currently - # there doesn't seem to be any way around that: - # https://github.community/t/conditional-services-in-a-job/135301 - postgres: - # Docker Hub image - image: postgres:11 - # Provide the password for postgres - env: - POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 - mssql: - # SQL Server image - image: mcr.microsoft.com/mssql/server:2019-latest - # Provide the password for mssql - env: - SA_PASSWORD: Meltan0admin - ACCEPT_EULA: "Y" - # Set health checks to wait until mssql has started - options: >- - --health-cmd /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $$SA_PASSWORD -Q "SELECT 1" -b -o /dev/null - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 1433 on service container to the host - - 1433:1433 - mysql: - # Mysql image - image: mysql:8-oracle - # Provide the password for mysql - env: - MYSQL_ROOT_PASSWORD: mysql - # Set health checks to wait until mysql has started - options: >- - --health-cmd mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - # Maps tcp port 3306 on service container to the host - - 3306:3306 env: PYTEST_MARKERS: not concurrent @@ -139,6 +88,19 @@ jobs: --health-retries 5 mcr.microsoft.com/mssql/server:2019-latest + - name: Start MYSQL Container + if: always() && (matrix.backend-db == 'mysql') + run: > + docker run -d + -p "3306:3306" + -e "MYSQL_ROOT_PASSWORD=mysql" + --name mysql + --health-cmd "mysqladmin ping -h 127.0.0.1 -u root --password=mysql" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + mysql:8-oracle + - name: Check running containers run: | docker ps -a From da62cdf59c529ece2691f246aaea97f0c2e7a812 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Wed, 22 Jun 2022 22:20:51 +0000 Subject: [PATCH 11/16] Added MSSQL user to health check --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c4835d3e11..e2abfc5421 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,7 +82,7 @@ jobs: -e "SA_PASSWORD=Meltan0admin" -e "ACCEPT_EULA=Y" --name mssql - --health-cmd "/opt/mssql-tools/bin/sqlcmd -U -P Meltan0admin -Q 'select 1' -b -o /dev/null" + --health-cmd "/opt/mssql-tools/bin/sqlcmd -U sa -P Meltan0admin -Q 'select 1' -b -o /dev/null" --health-interval 10s --health-timeout 5s --health-retries 5 From eab9b148bddfbecc0e4b7f0301a66d9e79e49dca Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Thu, 23 Jun 2022 08:25:55 +0000 Subject: [PATCH 12/16] Install extra database drivers in testing action --- .github/workflows/test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2abfc5421..7605b0f955 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,6 +87,11 @@ jobs: --health-timeout 5s --health-retries 5 mcr.microsoft.com/mssql/server:2019-latest + + - name: Install MSSQL Driver + if: always() && (matrix.backend-db == 'mssql') + run: | + pip install 'pyodbc==4.0.*' - name: Start MYSQL Container if: always() && (matrix.backend-db == 'mysql') @@ -100,6 +105,11 @@ jobs: --health-timeout 5s --health-retries 5 mysql:8-oracle + + - name: Install MYSQL Driver + if: always() && (matrix.backend-db == 'mysql') + run: | + pip install 'pymysql==1.0.*' - name: Check running containers run: | From 5809c2a05c22ae8c8b837dd98d3adc0df4ca0566 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Thu, 23 Jun 2022 19:50:45 +0000 Subject: [PATCH 13/16] Added extras for database drivers --- .github/workflows/test.yml | 12 +------ poetry.lock | 65 +++++++++++++++++++++++++++++++++++++- pyproject.toml | 4 +++ 3 files changed, 69 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7605b0f955..25376721ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,7 +59,7 @@ jobs: - name: Install Dependencies run: | poetry env use "${{ matrix.python-version }}" - poetry install + poetry install -E mysql -E mssql - name: Start Postgres Container if: always() && (matrix.backend-db == 'postgresql') @@ -87,11 +87,6 @@ jobs: --health-timeout 5s --health-retries 5 mcr.microsoft.com/mssql/server:2019-latest - - - name: Install MSSQL Driver - if: always() && (matrix.backend-db == 'mssql') - run: | - pip install 'pyodbc==4.0.*' - name: Start MYSQL Container if: always() && (matrix.backend-db == 'mysql') @@ -105,11 +100,6 @@ jobs: --health-timeout 5s --health-retries 5 mysql:8-oracle - - - name: Install MYSQL Driver - if: always() && (matrix.backend-db == 'mysql') - run: | - pip install 'pymysql==1.0.*' - name: Check running containers run: | diff --git a/poetry.lock b/poetry.lock index 165fa9608a..df8f623798 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1578,6 +1578,26 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" [package.extras] testutil = ["gitpython (>3)"] +[[package]] +name = "pymysql" +version = "1.0.2" +description = "Pure Python MySQL Driver" +category = "main" +optional = true +python-versions = ">=3.6" + +[package.extras] +ed25519 = ["PyNaCl (>=1.4.0)"] +rsa = ["cryptography"] + +[[package]] +name = "pyodbc" +version = "4.0.32" +description = "DB API Module for ODBC" +category = "main" +optional = true +python-versions = "*" + [[package]] name = "pyopenssl" version = "22.0.0" @@ -2282,11 +2302,13 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [extras] infra = [] +mssql = ["pyodbc"] +mysql = ["PyMySQL"] [metadata] lock-version = "1.1" python-versions = ">=3.7,<3.10" -content-hash = "4fe6485b819d0e0179d22507eb50637484ef1c094f7057c8931efbb82ec7df85" +content-hash = "a05cffe6894567e505569567e4ce3020f84d1dcb5133441af6fda0844e18b2eb" [metadata.files] aenum = [ @@ -2758,6 +2780,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1"}, {file = "greenlet-1.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58"}, + {file = "greenlet-1.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b336501a05e13b616ef81ce329c0e09ac5ed8c732d9ba7e3e983fcc1a9e86965"}, {file = "greenlet-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708"}, {file = "greenlet-1.1.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23"}, {file = "greenlet-1.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee"}, @@ -2770,6 +2793,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08"}, {file = "greenlet-1.1.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168"}, + {file = "greenlet-1.1.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b8c008de9d0daba7b6666aa5bbfdc23dcd78cafc33997c9b7741ff6353bafb7f"}, {file = "greenlet-1.1.2-cp36-cp36m-win32.whl", hash = "sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa"}, {file = "greenlet-1.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d"}, {file = "greenlet-1.1.2-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4"}, @@ -2778,6 +2802,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28"}, {file = "greenlet-1.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5"}, + {file = "greenlet-1.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c5d5b35f789a030ebb95bff352f1d27a93d81069f2adb3182d99882e095cefe"}, {file = "greenlet-1.1.2-cp37-cp37m-win32.whl", hash = "sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc"}, {file = "greenlet-1.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06"}, {file = "greenlet-1.1.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0"}, @@ -2786,6 +2811,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711"}, {file = "greenlet-1.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b"}, + {file = "greenlet-1.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2bde6792f313f4e918caabc46532aa64aa27a0db05d75b20edfc5c6f46479de2"}, {file = "greenlet-1.1.2-cp38-cp38-win32.whl", hash = "sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd"}, {file = "greenlet-1.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3"}, {file = "greenlet-1.1.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67"}, @@ -2794,6 +2820,7 @@ greenlet = [ {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b"}, {file = "greenlet-1.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3"}, + {file = "greenlet-1.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0051c6f1f27cb756ffc0ffbac7d2cd48cb0362ac1736871399a739b2885134d3"}, {file = "greenlet-1.1.2-cp39-cp39-win32.whl", hash = "sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf"}, {file = "greenlet-1.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd"}, {file = "greenlet-1.1.2.tar.gz", hash = "sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a"}, @@ -2935,6 +2962,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad"}, {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d"}, {file = "MarkupSafe-2.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a"}, + {file = "MarkupSafe-2.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a"}, {file = "MarkupSafe-2.0.1-cp310-cp310-win32.whl", hash = "sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28"}, {file = "MarkupSafe-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51"}, @@ -2946,6 +2976,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f"}, + {file = "MarkupSafe-2.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win32.whl", hash = "sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d"}, {file = "MarkupSafe-2.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567"}, @@ -2957,6 +2990,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9"}, + {file = "MarkupSafe-2.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415"}, {file = "MarkupSafe-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914"}, {file = "MarkupSafe-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9"}, @@ -2969,6 +3005,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a"}, {file = "MarkupSafe-2.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194"}, + {file = "MarkupSafe-2.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win32.whl", hash = "sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64"}, {file = "MarkupSafe-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833"}, {file = "MarkupSafe-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26"}, @@ -2981,6 +3020,9 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac"}, {file = "MarkupSafe-2.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e"}, + {file = "MarkupSafe-2.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win32.whl", hash = "sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74"}, {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, @@ -3325,6 +3367,26 @@ pylint = [ {file = "pylint-2.13.9-py3-none-any.whl", hash = "sha256:705c620d388035bdd9ff8b44c5bcdd235bfb49d276d488dd2c8ff1736aa42526"}, {file = "pylint-2.13.9.tar.gz", hash = "sha256:095567c96e19e6f57b5b907e67d265ff535e588fe26b12b5ebe1fc5645b2c731"}, ] +pymysql = [ + {file = "PyMySQL-1.0.2-py3-none-any.whl", hash = "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641"}, + {file = "PyMySQL-1.0.2.tar.gz", hash = "sha256:816927a350f38d56072aeca5dfb10221fe1dc653745853d30a216637f5d7ad36"}, +] +pyodbc = [ + {file = "pyodbc-4.0.32-cp27-cp27m-win32.whl", hash = "sha256:2152ce6d5131d769ff5839aa762e12d844c95e9ec4bb2f666e8cd9dfa1ae2240"}, + {file = "pyodbc-4.0.32-cp27-cp27m-win_amd64.whl", hash = "sha256:56ec4974096d40d6c62a228799122dbc2ade6c4045cc5d31860212a32cae95b1"}, + {file = "pyodbc-4.0.32-cp310-cp310-win_amd64.whl", hash = "sha256:b6d24b99face3a2d055b0dc7345083be2b70d02e7b45861a9afb7c21e0f5db13"}, + {file = "pyodbc-4.0.32-cp36-cp36m-win32.whl", hash = "sha256:699c080b1c1f7b4afc368b3521fd1161f46a10223443692a249cb01d90949b31"}, + {file = "pyodbc-4.0.32-cp36-cp36m-win_amd64.whl", hash = "sha256:0d4e14adb149cae45da37fa87aa297055156dae6e89ca3c75493d3d62d78e543"}, + {file = "pyodbc-4.0.32-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6c1e1c1fe747b0f6419e8df0b5c43161e7437dbf72f93f9fcfb9b7358fad3e12"}, + {file = "pyodbc-4.0.32-cp37-cp37m-win32.whl", hash = "sha256:bbc07517f339e019ee9f1fe679c4241251d11ca2124567616f67d62e73c29fc0"}, + {file = "pyodbc-4.0.32-cp37-cp37m-win_amd64.whl", hash = "sha256:e81ebf9cab80a6eaba7922dea02036e9f8a507a7b818856b8008a02d6fc0d2ab"}, + {file = "pyodbc-4.0.32-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0e4178e9b93329bbba17555882008e36a114179d06033b813a13b254dcd755d0"}, + {file = "pyodbc-4.0.32-cp38-cp38-win32.whl", hash = "sha256:c066f032e69fd71e9fadb3a380dfe8ecd1728b40a2bf38f76054d284f8523b29"}, + {file = "pyodbc-4.0.32-cp38-cp38-win_amd64.whl", hash = "sha256:736acad1b264ddb7313058dfe37265b0c5160c1c2a9d1ffd391347c025eb5dd1"}, + {file = "pyodbc-4.0.32-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:339d8aa633b0c65be5149c3378c7e3b5bead94dc8bb023a715b416bd047a008e"}, + {file = "pyodbc-4.0.32-cp39-cp39-win_amd64.whl", hash = "sha256:cda790bdc25bfad12d4fb9ba93368275802f7f9ecfa4c9c65e982d3a7fc35f2e"}, + {file = "pyodbc-4.0.32.tar.gz", hash = "sha256:9be5f0c3590655e1968488410fe3528bb8023d527e7ccec1f663d64245071a6b"}, +] pyopenssl = [ {file = "pyOpenSSL-22.0.0-py2.py3-none-any.whl", hash = "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0"}, {file = "pyOpenSSL-22.0.0.tar.gz", hash = "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf"}, @@ -3553,6 +3615,7 @@ sqlalchemy = [ {file = "SQLAlchemy-1.4.37-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:d9050b0c4a7f5538650c74aaba5c80cd64450e41c206f43ea6d194ae6d060ff9"}, {file = "SQLAlchemy-1.4.37-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b4c92823889cf9846b972ee6db30c0e3a92c0ddfc76c6060a6cda467aa5fb694"}, {file = "SQLAlchemy-1.4.37-cp27-cp27m-win32.whl", hash = "sha256:b55932fd0e81b43f4aff397c8ad0b3c038f540af37930423ab8f47a20b117e4c"}, + {file = "SQLAlchemy-1.4.37-cp27-cp27m-win_amd64.whl", hash = "sha256:4a17c1a1152ca4c29d992714aa9df3054da3af1598e02134f2e7314a32ef69d8"}, {file = "SQLAlchemy-1.4.37-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ffe487570f47536b96eff5ef2b84034a8ba4e19aab5ab7647e677d94a119ea55"}, {file = "SQLAlchemy-1.4.37-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:78363f400fbda80f866e8e91d37d36fe6313ff847ded08674e272873c1377ea5"}, {file = "SQLAlchemy-1.4.37-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ee34c85cbda7779d66abac392c306ec78c13f5c73a1f01b8b767916d4895d23"}, diff --git a/pyproject.toml b/pyproject.toml index 20b55324ce..a76ffccd6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,9 +69,13 @@ tzlocal = "^4.2.0" uvicorn = {extras = ["standard"], version = "^0.17.6"} watchdog = "^0.9.0" werkzeug = "^1" +PyMySQL = {version = "^1.0.2", optional = true} +pyodbc = {version = "^4.0.32", optional = true} [tool.poetry.extras] infra = ["ansible"] +mysql = ["pymysql"] +mssql = ["pyodbc"] [tool.poetry.dev-dependencies] asynctest = "^0.12.2" From 6c1cc8bf1537015d0d62409e2d93a3d0804b5ff7 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Thu, 23 Jun 2022 20:48:02 +0000 Subject: [PATCH 14/16] Updated mysql auth in Github action --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 25376721ff..789bca3b91 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,7 +99,8 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - mysql:8-oracle + mysql + --default-authentication-plugin=mysql_native_password - name: Check running containers run: | From 4a4939d25d71057efd44d69f40229c87bc61cb02 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Fri, 24 Jun 2022 11:53:57 +0000 Subject: [PATCH 15/16] Changed MSSQL driver to pymssql --- poetry.lock | 89 +++++++++++++++++++++++++++----------- pyproject.toml | 4 +- tests/fixtures/db/mssql.py | 24 +++++----- tests/fixtures/db/mysql.py | 16 ++++--- 4 files changed, 87 insertions(+), 46 deletions(-) diff --git a/poetry.lock b/poetry.lock index df8f623798..444e175204 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1578,6 +1578,14 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" [package.extras] testutil = ["gitpython (>3)"] +[[package]] +name = "pymssql" +version = "2.2.5" +description = "DB-API interface to Microsoft SQL Server for Python. (new Cython-based version)" +category = "main" +optional = true +python-versions = "*" + [[package]] name = "pymysql" version = "1.0.2" @@ -1590,14 +1598,6 @@ python-versions = ">=3.6" ed25519 = ["PyNaCl (>=1.4.0)"] rsa = ["cryptography"] -[[package]] -name = "pyodbc" -version = "4.0.32" -description = "DB API Module for ODBC" -category = "main" -optional = true -python-versions = "*" - [[package]] name = "pyopenssl" version = "22.0.0" @@ -2302,13 +2302,13 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [extras] infra = [] -mssql = ["pyodbc"] +mssql = ["pymssql"] mysql = ["PyMySQL"] [metadata] lock-version = "1.1" python-versions = ">=3.7,<3.10" -content-hash = "a05cffe6894567e505569567e4ce3020f84d1dcb5133441af6fda0844e18b2eb" +content-hash = "fe3f3e1afcf21ba7cd28201cee4d7ebe50bcc3718aa58fe60c828652b149bc7f" [metadata.files] aenum = [ @@ -3367,26 +3367,63 @@ pylint = [ {file = "pylint-2.13.9-py3-none-any.whl", hash = "sha256:705c620d388035bdd9ff8b44c5bcdd235bfb49d276d488dd2c8ff1736aa42526"}, {file = "pylint-2.13.9.tar.gz", hash = "sha256:095567c96e19e6f57b5b907e67d265ff535e588fe26b12b5ebe1fc5645b2c731"}, ] +pymssql = [ + {file = "pymssql-2.2.5-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6462017183f05ae231c3f84efce4e9a8d085b4a2e9e3ed5c407ee643494a0842"}, + {file = "pymssql-2.2.5-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6a013c82f7320c92039ac20db935e897a3fcd7423435705cef177f863dbb32e4"}, + {file = "pymssql-2.2.5-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:45e9bba4870103363c865d2b867de8de082745dd753083f27927ded3d15df587"}, + {file = "pymssql-2.2.5-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20de2f718f3c99040637a0e7d3b81f22e62211bcac01fb65641b964e70f26e20"}, + {file = "pymssql-2.2.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18eb4fcb55b67aaa2811e124eb8750fd84eeba1668a695908c6cc13682f6be5d"}, + {file = "pymssql-2.2.5-cp310-cp310-manylinux_2_24_i686.whl", hash = "sha256:9969971117401096a8a7c1e08d84ea3d5d3f598ee482822583a44e2e6d3791d0"}, + {file = "pymssql-2.2.5-cp310-cp310-manylinux_2_24_x86_64.whl", hash = "sha256:bb858c8a64990dd0145d874ef01af4b6fe39e9202ee0a74a8dff8285b801db21"}, + {file = "pymssql-2.2.5-cp310-cp310-win32.whl", hash = "sha256:da78a94908d42aa0f8af1b917c49c4dea38273035f81ea168b095e0b3ba8e486"}, + {file = "pymssql-2.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:b691779a450ebbbb98cd2ad89ae9f17799b0bf0c9dd2e65b316efcc74246716e"}, + {file = "pymssql-2.2.5-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:92262bf5d21b6d80e887d6fd0f03ab6337dad39b642b1887416cc67b1bd04cbf"}, + {file = "pymssql-2.2.5-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00f5b7e22bf7eb0faa5c6693e02c00f10ecc89fe505bbe84d553352681c2a749"}, + {file = "pymssql-2.2.5-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac80d1d11703120b986c604ddf4993c3dcbcbe907b04e34729c5aae2891399c0"}, + {file = "pymssql-2.2.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bf813ea0e4a0bfbb53a3469125eed06635222ea729339403585fd45f18a7812"}, + {file = "pymssql-2.2.5-cp36-cp36m-manylinux_2_24_i686.whl", hash = "sha256:73b747261cdf8d69395823cf650cf143d618ada412cbec90a88e4d449e7fa7f8"}, + {file = "pymssql-2.2.5-cp36-cp36m-manylinux_2_24_x86_64.whl", hash = "sha256:ce9e7ed16793c72d28e80506abd59a06b594fc55fe24f6256b7007b5b5e26120"}, + {file = "pymssql-2.2.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b5dcb4e4ceb16592a5d9ea3868162f21e7c215dde5c5e7f992709696c84a69b0"}, + {file = "pymssql-2.2.5-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9c90bc134a0c5c60e9acf651d87275c7e6dc6682981f62d574fa3dd441acb0f2"}, + {file = "pymssql-2.2.5-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:29d647ef61641cebc0e16d4a549b3f793f32bab88e7ba7c7550772a7aba9dea3"}, + {file = "pymssql-2.2.5-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:633079e248cdefef2207c0a55608637d0f3410f53e5679c589b37ea7965a2cca"}, + {file = "pymssql-2.2.5-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e373764cc81719f830b113ce6fee6849f2c4e5dfe1037804864b7dfb9946817e"}, + {file = "pymssql-2.2.5-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b47c05da0fffd6dd77141c1c37975a8469334f63e6faf03f5eab6d83c9001cc"}, + {file = "pymssql-2.2.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc245011aad9c018105193585469461c35fb35e7255b8e66fa27479f835742aa"}, + {file = "pymssql-2.2.5-cp37-cp37m-manylinux_2_24_i686.whl", hash = "sha256:72bbccdaf4938db7ff99db1f82ecdf6dc78778d8b7bbe6080590dd01024988de"}, + {file = "pymssql-2.2.5-cp37-cp37m-manylinux_2_24_x86_64.whl", hash = "sha256:395ec9e512a8d5b707aea8f516eebf51291b6fb6bc18fe4da88b222843bd4d46"}, + {file = "pymssql-2.2.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bb912b4681d55dce63f940becb7f4533661cfdb99cd6cd3daa7d7b5e285e977"}, + {file = "pymssql-2.2.5-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:14dee37da84653d1e483236d51bc0545e6442e0642ee362526db6599fd3c1733"}, + {file = "pymssql-2.2.5-cp37-cp37m-win32.whl", hash = "sha256:5e2cd5a6da441aa3e8ee13e8811f884791d77a4cb17028381543bf50d010c919"}, + {file = "pymssql-2.2.5-cp37-cp37m-win_amd64.whl", hash = "sha256:ba43efd1057018ab404c8603bbce643347b9165c08364df8a8c03d54dffc4453"}, + {file = "pymssql-2.2.5-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ac8d6b7e0dc97a9261b7500f086772ff5cf2bf9a0340d0aaee4dff5b217dd526"}, + {file = "pymssql-2.2.5-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ed7c5e58cd0ffcf3448cc38dfd2a6011f657759fcae9509748320cd89a0b8d32"}, + {file = "pymssql-2.2.5-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:30c4ab08cff619c7fe4d309fa6a05219b343082a1f46ca368deab5841632b8bf"}, + {file = "pymssql-2.2.5-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49806c343a6dea5221e98cabed1532fdd14535c5e5bb183001e0ffad31e11032"}, + {file = "pymssql-2.2.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15cc71762ec75a66bcb3751c2534ba4151b8512252b0377b842efec7cae6d953"}, + {file = "pymssql-2.2.5-cp38-cp38-manylinux_2_24_i686.whl", hash = "sha256:e42c7043c472d45db686ae67601c6680596a2f12ceaa99fb136e647d3d8cd643"}, + {file = "pymssql-2.2.5-cp38-cp38-manylinux_2_24_x86_64.whl", hash = "sha256:c738d51181ab4b3332631644ef7b561543837d1d942a291f8bc36a9e2dfb65db"}, + {file = "pymssql-2.2.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e33770ad113a646a45e4fc75f30683e92c9b929431e12977f824d6c810f28162"}, + {file = "pymssql-2.2.5-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8c5ad8d09f9035009ccb6f20bc9829ef60c03c436efb4b5c33af395c4030ea3d"}, + {file = "pymssql-2.2.5-cp38-cp38-win32.whl", hash = "sha256:c0611a3ea2a7ac496df30506fd28e4f7389808c59743b2eac0069e89af7289b8"}, + {file = "pymssql-2.2.5-cp38-cp38-win_amd64.whl", hash = "sha256:45c1519c94f9915911075c95fc63d62ba612e152248093aa12b7e902499c6416"}, + {file = "pymssql-2.2.5-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:1089cd5871daedc9918f8958605bbd2621006c29c06b571b419be53ea46c113a"}, + {file = "pymssql-2.2.5-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8ac2b3213462f078671806702dfadb8e459923f0f5013d371febebf4caf598ba"}, + {file = "pymssql-2.2.5-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9b707e5867bb49ad9d8945231112dfa3e9588b98f6f1248714d1f1af88321230"}, + {file = "pymssql-2.2.5-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:538a8e7ce255cabf093b25b72f8037b1d5fd3ba28b13a34c761a029840dc1880"}, + {file = "pymssql-2.2.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b9f5ec7896fd2f19393cd564d8ef562b26692218a84088ce1b10e9fe1215032"}, + {file = "pymssql-2.2.5-cp39-cp39-manylinux_2_24_i686.whl", hash = "sha256:6086f0b695b7ceb4603cdc2f66356a58e367a3f1828d1ca99255c4d92d1f0932"}, + {file = "pymssql-2.2.5-cp39-cp39-manylinux_2_24_x86_64.whl", hash = "sha256:e0b1bcb8465d787d5141bfc02d98e1e507d02155848b1b5622528402760f5dbf"}, + {file = "pymssql-2.2.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ebb7867207cf4cd497ea1d01b42590ed1db3561ef3ab8e135239301b6ee2695e"}, + {file = "pymssql-2.2.5-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:8b32ad436f293c8cbadc1ba3aaf2621ab82303d169b1ab6f30f55053640ab5b1"}, + {file = "pymssql-2.2.5-cp39-cp39-win32.whl", hash = "sha256:cc523bfd26671346faccb1273b16a74dcdf1a8ac81c93c6bfebdb240e4d5a042"}, + {file = "pymssql-2.2.5-cp39-cp39-win_amd64.whl", hash = "sha256:e9cce9c47e6d550992936f91f8f2e97a8f5132749bad1cd4f5f7923bf8732d1f"}, + {file = "pymssql-2.2.5.tar.gz", hash = "sha256:857411c308ecb584a3ca633be30f1971d2a8cc0bcd978709b1abf96ff43767ad"}, +] pymysql = [ {file = "PyMySQL-1.0.2-py3-none-any.whl", hash = "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641"}, {file = "PyMySQL-1.0.2.tar.gz", hash = "sha256:816927a350f38d56072aeca5dfb10221fe1dc653745853d30a216637f5d7ad36"}, ] -pyodbc = [ - {file = "pyodbc-4.0.32-cp27-cp27m-win32.whl", hash = "sha256:2152ce6d5131d769ff5839aa762e12d844c95e9ec4bb2f666e8cd9dfa1ae2240"}, - {file = "pyodbc-4.0.32-cp27-cp27m-win_amd64.whl", hash = "sha256:56ec4974096d40d6c62a228799122dbc2ade6c4045cc5d31860212a32cae95b1"}, - {file = "pyodbc-4.0.32-cp310-cp310-win_amd64.whl", hash = "sha256:b6d24b99face3a2d055b0dc7345083be2b70d02e7b45861a9afb7c21e0f5db13"}, - {file = "pyodbc-4.0.32-cp36-cp36m-win32.whl", hash = "sha256:699c080b1c1f7b4afc368b3521fd1161f46a10223443692a249cb01d90949b31"}, - {file = "pyodbc-4.0.32-cp36-cp36m-win_amd64.whl", hash = "sha256:0d4e14adb149cae45da37fa87aa297055156dae6e89ca3c75493d3d62d78e543"}, - {file = "pyodbc-4.0.32-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6c1e1c1fe747b0f6419e8df0b5c43161e7437dbf72f93f9fcfb9b7358fad3e12"}, - {file = "pyodbc-4.0.32-cp37-cp37m-win32.whl", hash = "sha256:bbc07517f339e019ee9f1fe679c4241251d11ca2124567616f67d62e73c29fc0"}, - {file = "pyodbc-4.0.32-cp37-cp37m-win_amd64.whl", hash = "sha256:e81ebf9cab80a6eaba7922dea02036e9f8a507a7b818856b8008a02d6fc0d2ab"}, - {file = "pyodbc-4.0.32-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0e4178e9b93329bbba17555882008e36a114179d06033b813a13b254dcd755d0"}, - {file = "pyodbc-4.0.32-cp38-cp38-win32.whl", hash = "sha256:c066f032e69fd71e9fadb3a380dfe8ecd1728b40a2bf38f76054d284f8523b29"}, - {file = "pyodbc-4.0.32-cp38-cp38-win_amd64.whl", hash = "sha256:736acad1b264ddb7313058dfe37265b0c5160c1c2a9d1ffd391347c025eb5dd1"}, - {file = "pyodbc-4.0.32-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:339d8aa633b0c65be5149c3378c7e3b5bead94dc8bb023a715b416bd047a008e"}, - {file = "pyodbc-4.0.32-cp39-cp39-win_amd64.whl", hash = "sha256:cda790bdc25bfad12d4fb9ba93368275802f7f9ecfa4c9c65e982d3a7fc35f2e"}, - {file = "pyodbc-4.0.32.tar.gz", hash = "sha256:9be5f0c3590655e1968488410fe3528bb8023d527e7ccec1f663d64245071a6b"}, -] pyopenssl = [ {file = "pyOpenSSL-22.0.0-py2.py3-none-any.whl", hash = "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0"}, {file = "pyOpenSSL-22.0.0.tar.gz", hash = "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf"}, diff --git a/pyproject.toml b/pyproject.toml index a76ffccd6e..ffe7c09b00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,12 +70,12 @@ uvicorn = {extras = ["standard"], version = "^0.17.6"} watchdog = "^0.9.0" werkzeug = "^1" PyMySQL = {version = "^1.0.2", optional = true} -pyodbc = {version = "^4.0.32", optional = true} +pymssql = {version = "^2.2.5", optional = true} [tool.poetry.extras] infra = ["ansible"] mysql = ["pymysql"] -mssql = ["pyodbc"] +mssql = ["pymssql"] [tool.poetry.dev-dependencies] asynctest = "^0.12.2" diff --git a/tests/fixtures/db/mssql.py b/tests/fixtures/db/mssql.py index 063a94f227..caab0b8cd3 100644 --- a/tests/fixtures/db/mssql.py +++ b/tests/fixtures/db/mssql.py @@ -14,9 +14,9 @@ def recreate_database(engine, db_name): """ with contextlib.suppress(sa.exc.ProgrammingError): engine.execute(text(f""" - USE master; - ALTER DATABASE [{db_name}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; - DROP DATABASE [{db_name}]; + USE master + IF EXISTS(select * from sys.databases where name='{db_name}') + DROP DATABASE {db_name} """)) with contextlib.suppress(sa.exc.ProgrammingError): @@ -27,17 +27,12 @@ def create_connection_url(host: str, port: int, user: str, password: str, databa Create a MSSQL connection URL for the given parameters. """ connection_url = sa.engine.URL.create( - "mssql+pyodbc", + "mssql+pymssql", username=user, password=password, host=host, port=port, database=database, - query={ - "driver": "ODBC Driver 18 for SQL Server", - "LongAsMax": "Yes", - "TrustServerCertificate": "Yes", - }, ) return connection_url @@ -50,12 +45,15 @@ def engine_uri(): password = os.getenv("MSSQL_PASSWORD") database = os.getenv("MSSQL_DB", "pytest_meltano") - # create the database - engine_uri = create_connection_url(host, port, user, password, database) - engine = create_engine(engine_uri, isolation_level="AUTOCOMMIT") + # Recreate the database using the master database + master_engine_uri = create_connection_url(host, port, user, password, "master") + engine = create_engine(master_engine_uri, isolation_level="AUTOCOMMIT") recreate_database(engine, database) - return str(engine_uri) + # Connect to the database where the tests will be run + testing_engine_uri = create_connection_url(host, port, user, password, database) + + return str(testing_engine_uri) @pytest.fixture() diff --git a/tests/fixtures/db/mysql.py b/tests/fixtures/db/mysql.py index 1798d89cb5..fce002a7c1 100644 --- a/tests/fixtures/db/mysql.py +++ b/tests/fixtures/db/mysql.py @@ -13,7 +13,7 @@ def recreate_database(engine, db_name): Drop & Create a new database. """ with contextlib.suppress(sa.exc.ProgrammingError): - engine.execute(text(f"DROP DATABASE {db_name}")) + engine.execute(text(f"DROP DATABASE IF EXISTS {db_name}")) with contextlib.suppress(sa.exc.ProgrammingError): engine.execute(text(f"CREATE DATABASE {db_name}")) @@ -29,6 +29,9 @@ def create_connection_url(host: str, port: int, user: str, password: str, databa host=host, port=port, database=database, + query={ + "charset": "utf8" + } ) return connection_url @@ -41,12 +44,15 @@ def engine_uri(): password = os.getenv("MYSQL_PASSWORD") database = os.getenv("MYSQL_DB", "pytest_meltano") - # create the database - engine_uri = create_connection_url(host, port, user, password, database) - engine = create_engine(engine_uri, isolation_level="AUTOCOMMIT") + # Recreate the database using the master database + master_engine_uri = create_connection_url(host, port, user, password, "mysql") + engine = create_engine(master_engine_uri, isolation_level="AUTOCOMMIT") recreate_database(engine, database) - return str(engine_uri) + # Connect to the database where the tests will be run + testing_engine_uri = create_connection_url(host, port, user, password, database) + + return str(testing_engine_uri) @pytest.fixture() From f36213549aeabbeb44378ead9bcb63f770b22553 Mon Sep 17 00:00:00 2001 From: Jules Huisman Date: Fri, 24 Jun 2022 11:54:20 +0000 Subject: [PATCH 16/16] Added more string limits for MySQL --- .github/workflows/test.yml | 6 ++++++ .../13e8639c6d2b_add_state_edit_to_job_state_enum.py | 10 +++++++++- .../23ea52e6d784_add_resource_type_to_embed_token.py | 10 +++++++++- .../versions/367228df6a43_add_trigger_to_job.py | 10 +++++++++- src/meltano/migrations/versions/6ef30ab7b8e5_.py | 1 - .../ceb00d7ff3bd_create_the_embedtoken_table.py | 4 +++- 6 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 789bca3b91..e2f05a83c0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,6 +101,12 @@ jobs: --health-retries 5 mysql --default-authentication-plugin=mysql_native_password + --collation_server=utf8_general_ci + --character_set_server=utf8 + + - name: Wait for MYSQL Container + if: always() && (matrix.backend-db == 'mysql') + run: sleep 15 - name: Check running containers run: | diff --git a/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py b/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py index c3bab8187e..b9a11cdf1f 100644 --- a/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py +++ b/src/meltano/migrations/versions/13e8639c6d2b_add_state_edit_to_job_state_enum.py @@ -10,6 +10,11 @@ import sqlalchemy as sa from alembic import op +from meltano.migrations.utils.dialect_typing import ( + get_dialect_name, + max_string_length_for_dialect, +) + # revision identifiers, used by Alembic. revision = "13e8639c6d2b" down_revision = "d135f52a6f49" @@ -30,6 +35,9 @@ class State(Enum): def upgrade(): + dialect_name = get_dialect_name(op) + max_string_length = max_string_length_for_dialect(dialect_name) + conn = op.get_bind() # In sqlite, the field is already a varchar. # "ALTER COLUMN" statements are also not supported. @@ -37,7 +45,7 @@ def upgrade(): op.alter_column( table_name="job", column_name="state", - type_=sa.types.String, + type_=sa.types.String(max_string_length), existing_type=sa.Enum(State, name="job_state"), ) diff --git a/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py b/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py index de0e34b4b9..82f22eee9c 100644 --- a/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py +++ b/src/meltano/migrations/versions/23ea52e6d784_add_resource_type_to_embed_token.py @@ -8,6 +8,11 @@ import sqlalchemy as sa from alembic import op +from meltano.migrations.utils.dialect_typing import ( + get_dialect_name, + max_string_length_for_dialect, +) + # revision identifiers, used by Alembic. revision = "23ea52e6d784" down_revision = "ceb00d7ff3bd" @@ -18,7 +23,10 @@ def upgrade(): - op.add_column("embed_tokens", sa.Column("resource_type", sa.String)) + dialect_name = get_dialect_name(op) + max_string_length = max_string_length_for_dialect(dialect_name) + + op.add_column("embed_tokens", sa.Column("resource_type", sa.String(max_string_length))) metadata = sa.MetaData(bind=op.get_bind()) Embed_Tokens = sa.Table("embed_tokens", metadata, autoload=True) diff --git a/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py b/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py index c31f7c2046..8750e9834b 100644 --- a/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py +++ b/src/meltano/migrations/versions/367228df6a43_add_trigger_to_job.py @@ -8,6 +8,11 @@ import sqlalchemy as sa from alembic import op +from meltano.migrations.utils.dialect_typing import ( + get_dialect_name, + max_string_length_for_dialect, +) + # revision identifiers, used by Alembic. revision = "367228df6a43" down_revision = "87d9638f6ac6" @@ -16,7 +21,10 @@ def upgrade(): - op.add_column("job", sa.Column("trigger", sa.String)) + dialect_name = get_dialect_name(op) + max_string_length = max_string_length_for_dialect(dialect_name) + + op.add_column("job", sa.Column("trigger", sa.String(max_string_length))) def downgrade(): diff --git a/src/meltano/migrations/versions/6ef30ab7b8e5_.py b/src/meltano/migrations/versions/6ef30ab7b8e5_.py index 90dffea810..45b6bf3506 100644 --- a/src/meltano/migrations/versions/6ef30ab7b8e5_.py +++ b/src/meltano/migrations/versions/6ef30ab7b8e5_.py @@ -22,7 +22,6 @@ def upgrade(): - conn = op.get_bind() dialect_name = get_dialect_name(op) datetime_type = datetime_for_dialect(dialect_name) max_string_length = max_string_length_for_dialect(dialect_name) diff --git a/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py b/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py index 2733848608..0b1e8c2235 100644 --- a/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py +++ b/src/meltano/migrations/versions/ceb00d7ff3bd_create_the_embedtoken_table.py @@ -11,6 +11,7 @@ from meltano.migrations.utils.dialect_typing import ( datetime_for_dialect, get_dialect_name, + max_string_length_for_dialect, ) # revision identifiers, used by Alembic. @@ -23,12 +24,13 @@ def upgrade(): dialect_name = get_dialect_name(op) datetime_type = datetime_for_dialect(dialect_name) + max_string_length = max_string_length_for_dialect(dialect_name) op.create_table( "embed_tokens", sa.Column("id", sa.Integer, primary_key=True), sa.Column("token", sa.String(64), unique=True), - sa.Column("resource_id", sa.String, nullable=False), + sa.Column("resource_id", sa.String(max_string_length), nullable=False), sa.Column("created_at", datetime_type), )