Merge pull request #275 from fishtown-analytics/fix/bq-schema-pattern

Handle schema pattern on BQ

CHANGELOG.md

@@ -1,3 +1,8 @@
+# dbt-utils v0.6.1
+
+## Fixes
+- Fix the logic in `get_tables_by_pattern_sql` for matching a schema pattern on BigQuery ([#275](https://github.com/fishtown-analytics/dbt-utils/pull/275/))
+
 # dbt-utils v0.6.0
 
 ## Breaking changes

integration_tests/data/sql/data_events_20180103.csv

@@ -0,0 +1,3 @@
+user_id,event
+5,play
+6,pause

integration_tests/data/sql/data_union_events_expected.csv

@@ -0,0 +1,7 @@
+user_id,event
+1,play
+2,pause
+3,play
+4,pause
+5,play
+6,pause

integration_tests/dbt_project.yml

@@ -46,3 +46,7 @@ seeds:
           num_buckets: integer
           min_value: float
           max_value: float
+
+    sql:
+      data_events_20180103:
+        +schema: events

integration_tests/models/sql/schema.yml

@@ -50,7 +50,7 @@ models:
               values:
                 - '5'
 
-  - name: test_get_tables_by_prefix_and_union
+  - name: test_get_relations_by_prefix_and_union
     columns:
       - name: event
         tests:
@@ -117,3 +117,8 @@ models:
     tests:
       - dbt_utils.equality:
           compare_model: ref('data_union_expected')
+
+  - name: test_get_relations_by_pattern
+    tests:
+      - dbt_utils.equality:
+          compare_model: ref('data_union_events_expected')

integration_tests/models/sql/test_get_relations_by_pattern.sql

@@ -1,4 +1,16 @@
 {{ config(materialized = 'table') }}
 
-{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_%') %}
-{{ dbt_utils.union_relations(relations) }}
+{% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %}
+
+with unioned as (
+
+    {{ dbt_utils.union_relations(relations) }}
+
+)
+
+select
+
+    user_id,
+    event
+
+from unioned

macros/sql/get_tables_by_pattern_sql.sql

@@ -5,7 +5,7 @@
 
 {% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}
 
-        select distinct 
+        select distinct
             table_schema as "table_schema", table_name as "table_name"
         from {{database}}.information_schema.tables
         where table_schema ilike '{{ schema_pattern }}'
@@ -16,13 +16,51 @@
 
 
 {% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}
-
-        select distinct
-            table_schema, table_name
 
-        from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES
-        where table_schema = '{{schema_pattern}}'
-            and lower(table_name) like lower ('{{table_pattern}}')
-            and lower(table_name) not like lower ('{{exclude}}')
+    {% if '%' in schema_pattern %}
+        {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %}
+    {% else %}
+        {% set schemata=[schema_pattern] %}
+    {% endif %}
+
+    {% set sql %}
+        {% for schema in schemata %}
+            select distinct
+                table_schema, table_name
+
+            from {{ adapter.quote(database) }}.{{ schema }}.INFORMATION_SCHEMA.TABLES
+            where lower(table_name) like lower ('{{ table_pattern }}')
+                and lower(table_name) not like lower ('{{ exclude }}')
+
+            {% if not loop.last %} union all {% endif %}
+
+        {% endfor %}
+    {% endset %}
+
+    {{ return(sql) }}
+
+{% endmacro %}
+
+
+{% macro _bigquery__get_matching_schemata(schema_pattern, database) %}
+    {% if execute %}
+
+        {% set sql %}
+        select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA
+        where lower(schema_name) like lower('{{ schema_pattern }}')
+        {% endset %}
+
+        {% set results=run_query(sql) %}
+
+        {% set schemata=results.columns['schema_name'].values() %}
+
+        {{ return(schemata) }}
+
+    {% else %}
+
+        {{ return([]) }}
+
+    {% endif %}
+
 
 {% endmacro %}