From 49aed1bbab74093a46bfc4d52ac8a28253d09f23 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Thu, 24 Jul 2025 14:17:40 +0100 Subject: [PATCH 1/3] Fix `column already exists` error by invalidating table cache when a new column is created --- target_snowflake/connector.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/target_snowflake/connector.py b/target_snowflake/connector.py index eead02c..3b5d0cf 100644 --- a/target_snowflake/connector.py +++ b/target_snowflake/connector.py @@ -272,6 +272,10 @@ def prepare_column( ) raise + def _create_empty_column(self, full_table_name, column_name, sql_type): + super()._create_empty_column(full_table_name, column_name, sql_type) + self.table_cache.pop(full_table_name, None) + @staticmethod def get_column_rename_ddl( table_name: str, From bdc03c4dc6f4c4f088f07db1c39babf9cf4daea5 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Thu, 24 Jul 2025 15:46:56 +0100 Subject: [PATCH 2/3] Make pre-commit happy --- target_snowflake/connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target_snowflake/connector.py b/target_snowflake/connector.py index 3b5d0cf..0338fca 100644 --- a/target_snowflake/connector.py +++ b/target_snowflake/connector.py @@ -272,7 +272,7 @@ def prepare_column( ) raise - def _create_empty_column(self, full_table_name, column_name, sql_type): + def _create_empty_column(self, full_table_name, column_name, sql_type) -> None: # noqa: ANN001 super()._create_empty_column(full_table_name, column_name, sql_type) self.table_cache.pop(full_table_name, None) From 1c71bf3bc3ee7b861b0fa2c318e3c95ce0dfd773 Mon Sep 17 00:00:00 2001 From: Reuben Frankel Date: Tue, 29 Jul 2025 20:23:30 +0100 Subject: [PATCH 3/3] Clear table cache on sink creation instead, to avoid degraded performance when many columns are added --- target_snowflake/connector.py | 4 ---- target_snowflake/sinks.py | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/target_snowflake/connector.py b/target_snowflake/connector.py index 0338fca..eead02c 100644 --- a/target_snowflake/connector.py +++ b/target_snowflake/connector.py @@ -272,10 +272,6 @@ def prepare_column( ) raise - def _create_empty_column(self, full_table_name, column_name, sql_type) -> None: # noqa: ANN001 - super()._create_empty_column(full_table_name, column_name, sql_type) - self.table_cache.pop(full_table_name, None) - @staticmethod def get_column_rename_ddl( table_name: str, diff --git a/target_snowflake/sinks.py b/target_snowflake/sinks.py index e08eebc..f84ec74 100644 --- a/target_snowflake/sinks.py +++ b/target_snowflake/sinks.py @@ -77,6 +77,9 @@ def setup(self) -> None: self.connector.prepare_schema( self.conform_name(self.schema_name, object_type="schema"), ) + + self.connector.table_cache.pop(self.full_table_name, None) + try: self.connector.prepare_table( full_table_name=self.full_table_name, @@ -94,6 +97,8 @@ def setup(self) -> None: ) raise + self.connector.table_cache.pop(self.full_table_name, None) + def conform_name( self, name: str,