[ELE-47] feature: allow glue adapter

macros/utils/cross_db_utils/file_format.sql

@@ -0,0 +1,11 @@
+{% macro get_default_file_format() %}
+  {% do return(adapter.dispatch("get_default_file_format", "elementary")()) %}
+{% endmacro %}
+
+{%- macro glue__get_default_file_format() %}
+  {% do return("delta") %}
+{% endmacro %}
+
+{% macro default__get_default_file_format() %}
+  {% do return(none) %}
+{% endmacro %}

macros/utils/cross_db_utils/incremental_strategy.sql

@@ -13,3 +13,7 @@
 {% macro default__get_default_incremental_strategy() %}
   {% do return(none) %}
 {% endmacro %}
+
+{%- macro glue__get_default_incremental_strategy() %}
+  {% do return("merge") %}
+{% endmacro %}

macros/utils/run_queries/run_query.sql

@@ -1,4 +1,8 @@
 {% macro run_query(query, lowercase_column_names=True) %}
+    {{ return(adapter.dispatch('run_query', 'elementary')(query, lowercase_column_names)) }}
+{% endmacro %}
+
+{% macro default__run_query(query, lowercase_column_names=True) %}
     {% set query_result = dbt.run_query(query) %}
     {% if lowercase_column_names %}
         {% set lowercased_column_names = {} %}
@@ -10,3 +14,44 @@
 
     {% do return(query_result) %}
 {% endmacro %}
+
+{% macro glue__run_query(query, lowercase_column_names=True) %}
+    -- Glue does not support running queries that does not return results through the `run_query` method
+    -- We need to check if the query is a DDL or DML statement and run it using the `run_query_statement` statement
+    -- There are other statements that should not get query results, but keeping it simple for now
+
+    {#{{ log("want to run "~query, info=true) }}#}
+
+    {% set should_not_get_query_results_statements = ["create table", "create or replace table", "insert into", "create view", "create temporary view", "create or replace temporary view"] %}
+
+    {% set should_not_get_query_results = [] %}
+    {% for statement in should_not_get_query_results_statements %}
+        {% if statement in query.lower() %}
+            {% do should_not_get_query_results.append(True) %}
+        {% endif %}
+    {% endfor %}
+
+    {% if should_not_get_query_results | length > 0 %}
+        -- Morover, Glue throws an error if the query contains escaped single quotes in single quotes strings
+        {% set curracted_query = query.replace("\\'","") %}
+
+        {% call statement("run_query_statement", fetch_result=false, auto_begin=false) %}
+            {{ curracted_query }}
+        {% endcall %}
+
+    {% else %}
+        {% set query_result = dbt.run_query(query) %}
+
+        {% if lowercase_column_names %}
+            {% set lowercased_column_names = {} %}
+            {% for column_name in query_result.column_names %}
+                {% do lowercased_column_names.update({column_name: column_name.lower()}) %}
+            {% endfor %}
+            {% set query_result = query_result.rename(lowercased_column_names) %}
+        {% endif %}
+
+        {% do return(query_result) %}
+        {% endif %}
+
+{% endmacro %}
+

macros/utils/table_operations/create_temp_table.sql

@@ -1,4 +1,8 @@
 {% macro create_temp_table(database_name, schema_name, table_name, sql_query) %}
+    {% do return(adapter.dispatch("create_temp_table", "elementary")(database_name, schema_name, table_name, sql_query)) %}
+{% endmacro %}
+
+{% macro default__create_temp_table(database_name, schema_name, table_name, sql_query) %}
     {% set temp_table_exists, temp_table_relation = dbt.get_or_create_relation(database=database_name,
                                                                                schema=schema_name,
                                                                                identifier=table_name,
@@ -11,4 +15,25 @@
         {% do elementary.run_query(dbt.create_table_as(True, temp_table_relation, sql_query)) %}
     {% endif %}
     {{ return(temp_table_relation) }}
-{% endmacro %}
+{% endmacro %}
+
+{% macro glue__create_temp_table(database_name, schema_name, table_name, sql_query) %}
+    -- Apparently, inside glue, cannot query temporary tables
+    -- Moroever, there is a problem inside edr_make_temp_relation, we need to enforce no suffix
+    -- It is a problem because for the moment, it tries to get relation without suffix 
+    -- and it puts none as suffix if not specified maybe the function is not well named
+    {% set temp_table_exists, temp_table_relation = dbt.get_or_create_relation(database=database_name,
+                                                                               schema=schema_name,
+                                                                               identifier=table_name,
+                                                                               type='table') -%}
+    {% set temp_table_relation = elementary.edr_make_temp_relation(temp_table_relation, suffix="") %}
+
+    {% if temp_table_exists %}
+        {% do adapter.drop_relation(temp_table_relation) %}
+        {% do elementary.run_query(dbt.create_table_as(False, temp_table_relation, sql_query)) %}
+    {% else %}
+        {% do elementary.run_query(dbt.create_table_as(False, temp_table_relation, sql_query)) %}
+    {% endif %}
+    {{ return(temp_table_relation) }}
+{% endmacro %}
+

macros/utils/table_operations/delete_and_insert.sql

@@ -87,6 +87,41 @@
     {% do return(queries) %}
 {% endmacro %}
 
+
+{% macro glue__get_delete_and_insert_queries(relation, insert_relation, delete_relation, delete_column_key) %}
+    {% set queries = [] %}
+
+    {# Calling `is_delta` raises an error if `metadata` is None - https://github.com/databricks/dbt-databricks/blob/33dca4b66b05f268741030b33659d34ff69591c1/dbt/adapters/databricks/relation.py#L71 #}
+    {{ log("get delete has delete_relation "~delete_relation~" relation.metadata "~relation.metadata~" relation.is_delta "~relation.is_delta, info=true) }}
+    {% if delete_relation and relation.is_delta %}
+        {% set delete_query %}
+            merge into {{ relation }} as source
+            using {{ delete_relation }} as target
+            on (source.{{ delete_column_key }} = target.{{ delete_column_key }}) or source.{{ delete_column_key }} is null
+            when matched then delete;
+        {% endset %}
+        {% do queries.append(delete_query) %}
+
+    {% elif delete_relation %}
+        {% set delete_query %}
+            delete from {{ relation }}
+            where
+            {{ delete_column_key }} is null
+            or {{ delete_column_key }} in (select {{ delete_column_key }} from {{ delete_relation }});
+        {% endset %}
+        {% do queries.append(delete_query) %}
+    {% endif %}
+
+    {% if insert_relation %}
+        {% set insert_query %}
+            insert into {{ relation }} select * from {{ insert_relation }};
+        {% endset %}
+        {% do queries.append(insert_query) %}
+    {% endif %}
+
+    {% do return(queries) %}
+{% endmacro %}
+
 {% macro athena__get_delete_and_insert_queries(relation, insert_relation, delete_relation, delete_column_key) %}
     {% set queries = [] %}
 

macros/utils/table_operations/make_temp_relation.sql

@@ -16,6 +16,10 @@
     {% do return(tmp_relation) %}
 {% endmacro %}
 
+{% macro glue__edr_make_temp_relation(base_relation, suffix) %}
+    {% do return(dbt.make_temp_relation(base_relation, suffix)) %}
+{% endmacro %}
+
 {% macro databricks__edr_make_temp_relation(base_relation, suffix) %}
     {% do return(dbt.make_temp_relation(base_relation, suffix)) %}
 {% endmacro %}

models/edr/data_monitoring/data_monitoring/data_monitoring_metrics.sql

@@ -10,6 +10,7 @@
       "prev_timestamp_column": "updated_at",
       },
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy(),
   )
 }}

models/edr/data_monitoring/schema_changes/schema_columns_snapshot.sql

@@ -9,6 +9,7 @@
       "prev_timestamp_column": "detected_at",
       },
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/dbt_artifacts/dbt_columns.sql

@@ -7,8 +7,9 @@
     on_schema_change='sync_all_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
-    )
+  )
 }}
 
 {{ elementary.get_dbt_columns_empty_table_query() }}

models/edr/dbt_artifacts/dbt_exposures.sql

@@ -7,8 +7,9 @@
     on_schema_change='sync_all_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
-    )
+  )
 }}
 
 {{ elementary.get_dbt_exposures_empty_table_query() }}

models/edr/dbt_artifacts/dbt_invocations.sql

@@ -10,6 +10,7 @@
       "prev_timestamp_column": "generated_at",
       },
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/dbt_artifacts/dbt_metrics.sql

@@ -7,8 +7,9 @@
     on_schema_change='sync_all_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
-    )
+  )
 }}
 
 {{ elementary.get_dbt_metrics_empty_table_query() }}

models/edr/dbt_artifacts/dbt_models.sql

@@ -7,6 +7,7 @@
     on_schema_change='sync_all_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/dbt_artifacts/dbt_run_results.sql

@@ -12,6 +12,7 @@
       "prev_timestamp_column": "generated_at",
       },
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/dbt_artifacts/dbt_seeds.sql

@@ -7,6 +7,7 @@
     on_schema_change='sync_all_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/dbt_artifacts/dbt_snapshots.sql

@@ -7,6 +7,7 @@
     on_schema_change='sync_all_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/dbt_artifacts/dbt_sources.sql

@@ -7,8 +7,9 @@
     on_schema_change='sync_all_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
-    )
+  )
 }}
 
 {{ elementary.get_dbt_sources_empty_table_query() }}

models/edr/dbt_artifacts/dbt_tests.sql

@@ -7,6 +7,7 @@
     on_schema_change='sync_all_columns',
     full_refresh=elementary.get_config_var('elementary_full_refresh'),
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/run_results/dbt_source_freshness_results.sql

@@ -9,6 +9,7 @@
       "prev_timestamp_column": "generated_at",
       },
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/run_results/elementary_test_results.sql

@@ -9,6 +9,7 @@
       "prev_timestamp_column": "detected_at",
       },
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}

models/edr/run_results/test_result_rows.sql

@@ -11,6 +11,7 @@
       "prev_timestamp_column": "detected_at",
       },
     table_type=elementary.get_default_table_type(),
+    file_format=elementary.get_default_file_format(),
     incremental_strategy=elementary.get_default_incremental_strategy()
   )
 }}