Merge pull request #2 from rjh336/fix/generate_select_logs-snowflake-schema

replace hard coded db with snowflake target

Author: rjh336

.circleci/config.yml

@@ -26,6 +26,16 @@ jobs:
             . venv/bin/activate
             pip install -U pip setuptools wheel
             pip install -r dev-requirements.txt
+      - run:
+          name: Rename dbt models
+          command: |
+            . venv/bin/activate
+            export GLOBAL_MODEL_SUFFIX=`date +%s`
+            touch .env && echo "GLOBAL_MODEL_SUFFIX=$GLOBAL_MODEL_SUFFIX" >> .env
+            mv ./integration_tests/models/test_model_table.sql ./integration_tests/models/test_model_table_`echo $GLOBAL_MODEL_SUFFIX`.sql
+            mv ./integration_tests/models/test_model_view.sql ./integration_tests/models/test_model_view_`echo $GLOBAL_MODEL_SUFFIX`.sql
+            sed -i -e "s/test_model_table/test_model_table_$GLOBAL_MODEL_SUFFIX/g" ./integration_tests/models/properties.yml
+            sed -i -e "s/test_model_view/test_model_view_$GLOBAL_MODEL_SUFFIX/g" ./integration_tests/models/properties.yml
       - run:
           name: Install dbt dependencies
           command: |

dev-requirements.txt

@@ -3,5 +3,6 @@ dbt-postgres
 dbt-bigquery
 dbt-snowflake
 pytest
+python-dotenv
 google-cloud-bigquery
-snowflake-connector-python==2.7.9
+snowflake-connector-python==2.8.1

macros/common/generate_select_logs_query.sql

@@ -66,7 +66,7 @@
             query_id,
             query_text
         from
-            table(dbt_model_usage.information_schema.query_history(result_limit => 10000))
+            table({{ target_model.database }}.information_schema.query_history(result_limit => 10000))
         where
             query_type = 'SELECT'
             and start_time between timestampadd({{ time_unit }}, -{{ num_units }}, current_timestamp()) and current_timestamp()

scripts/query_database.py

@@ -1,18 +1,20 @@
+from dotenv import load_dotenv
+load_dotenv()
 import os
 import sys
 import snowflake.connector
 from google.cloud import bigquery
 
 QUERIES = {
     'models_query': """
-        select * from {0}.{1}.test_model_view
+        select * from {0}.{1}.test_model_view_{2}
         union all
-        select * from {0}.{1}.test_model_table
+        select * from {0}.{1}.test_model_table_{2}
         """,
 
     'columns_query': """
         select string_field 
-        from {0}.{1}.test_model_view
+        from {0}.{1}.test_model_view_{2}
         """
 }
 
@@ -24,7 +26,8 @@ def main(target, query_name):
         bigquery_ctx = bigquery.Client()
         query = query_format_string.format(
             os.getenv('BIGQUERY_TEST_DATABASE'),
-            os.getenv('TEST_SCHEMA')
+            os.getenv('TEST_SCHEMA'),
+            os.getenv('GLOBAL_MODEL_SUFFIX')
         )
         print(f"{query}\n")
         _ = bigquery_ctx.query(query).result()
@@ -36,7 +39,9 @@ def main(target, query_name):
             account=os.getenv('SNOWFLAKE_ACCOUNT_ID'))
         query = query_format_string.format(
             os.getenv('SNOWFLAKE_TEST_DATABASE'),
-            os.getenv('TEST_SCHEMA'))
+            os.getenv('TEST_SCHEMA'),
+            os.getenv('GLOBAL_MODEL_SUFFIX')
+        )
         print(f"{query}\n")
         cur = snowflake_ctx.cursor()
         cur.execute(query)

scripts/run_tests.sh

@@ -18,14 +18,14 @@ printf "\n\nRunning integration test query:"
 python3 ./scripts/query_database.py $dbt_target models_query
 
 printf "\n\ndbt test ...\n"
-dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 60}' || true
+dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 30}' || true
 printf "\nRun results validation:\n"
 pytest -v scripts/test_run_results.py::test_build_failure_column_test
 
 printf "\n\nRunning integration test query:"
 python3 ./scripts/query_database.py $dbt_target columns_query
 
 printf "\n\ndbt test ...\n"
-dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: hour, num_units: 1}'
+dbt test -t $dbt_target --project-dir $DBT_PROJECT_DIR --vars '{time_unit: second, num_units: 40}'
 printf "\nRun results validation:\n"
 pytest -v scripts/test_run_results.py::test_build_success