pull from main

macros/_samples/override_naming_model_yaml.sql

@@ -0,0 +1,56 @@
+--this model override allows users to avoid dbt's "auto lowercasing" behavior when using codegen to generate model yaml. 
+--To actually use this macro, put it in the macros folder and name it generate_model_yaml.sql. 
+{% macro generate_column_yaml(column, model_yaml, column_desc_dict, parent_column_name="") %}
+    {% if parent_column_name %}
+        {% set column_name = parent_column_name ~ "." ~ column.name %}
+    {% else %}
+        {% set column_name = column.name %}
+    {% endif %}
+-- removing "lower" from whole macro
+    {% do model_yaml.append('      - name: ' ~ column.name) %}
+    {% do model_yaml.append('        description: "' ~ column_desc_dict.get(column.name) ~ '"') %}
+    {% do model_yaml.append('') %}
+
+    {% if column.fields|length > 0 %}
+        {% for child_column in column.fields %}
+            {% set model_yaml = codegen.generate_column_yaml(child_column, model_yaml, column_desc_dict, parent_column_name=column_name) %}
+        {% endfor %}
+    {% endif %}
+    {% do return(model_yaml) %}
+{% endmacro %}
+
+{% macro generate_model_yaml(model_names=[], upstream_descriptions=False) %}
+
+    {% set model_yaml=[] %}
+
+    {% do model_yaml.append('version: 2') %}
+    {% do model_yaml.append('') %}
+    {% do model_yaml.append('models:') %}
+
+    {% if model_names is string %}
+        {{ exceptions.raise_compiler_error("The `model_names` argument must always be a list, even if there is only one model.") }}
+    {% else %}
+        {% for model in model_names %}
+            {% do model_yaml.append('  - name: ' ~ model ) %}
+            {% do model_yaml.append('    description: ""') %}
+            {% do model_yaml.append('    columns:') %}
+
+            {% set relation=ref(model) %}
+            {%- set columns = adapter.get_columns_in_relation(relation) -%}
+            {% set column_desc_dict =  codegen.build_dict_column_descriptions(model) if upstream_descriptions else {} %}
+
+            {% for column in columns %}
+                {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %}
+            {% endfor %}
+        {% endfor %}
+    {% endif %}
+
+{% if execute %}
+
+    {% set joined = model_yaml | join ('\n') %}
+    {{ log(joined, info=True) }}
+    {% do return(joined) %}
+
+{% endif %}
+
+{% endmacro %}

models/_samples/model_governance/_model_governance__models.yml

@@ -17,7 +17,7 @@ models:
           # unique constraint is NOT enforced in snowflake
           - type: unique 
             warn_unenforced: false
-        tests:
+        data_tests:
           - unique
       - name: order_date
         data_type: date 

models/_samples/python/_python__models.yml

@@ -18,7 +18,7 @@ models:
     columns:
       - name: customer_id
         description: Primary id on the customers table
-        tests:
+        data_tests:
           - unique
           - not_null
 

models/_samples/snapshot/_snapshot__models.yml

@@ -4,5 +4,8 @@ models:
   - name: example_join_snapshots
     columns:
       - name: surrogate_key
-        tests:
+        data_tests:
+          - unique:
+              config:
+                where: valid_to = '{{ var("future_proof_date") }}'
           - not_null

models/_samples/snapshot/example_join_snapshots.sql

@@ -46,4 +46,4 @@ final as (
     from joined
 )
 
-select * from final
+select * from final

models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml

@@ -7,14 +7,14 @@ models:
           multi line description 
           of my staging orders model from
           jaffle shop
-    tests:
+    data_tests:
       - dbt_utils.recency:
           datepart: day
           field: _etl_loaded_at
           interval: 7
     columns:
       - name: order_id 
-        tests:
+        data_tests:
           - not_null:
               config:
                 where: _etl_loaded_at >= dateadd('day', -3, current_timestamp())
@@ -30,7 +30,7 @@ models:
           multi line description 
           of my order date column from my
           jaffle shop orders model
-        tests:
+        data_tests:
           - dbt_expectations.expect_column_values_to_be_of_type:
               column_type: date
 
@@ -39,7 +39,7 @@ models:
       multi line description 
       of my staging
       jaffle shop customers model
-    tests:
+    data_tests:
       - not_empty
       - dbt_utils.equal_rowcount:
           compare_model: source('jaffle_shop', 'customers')
@@ -59,12 +59,12 @@ models:
           multi line description 
           of my customer id column from my
           jaffle shop customers model
-        tests:
+        data_tests:
           - unique 
           - not_null
           - unique_and_not_null
       - name: first_name
-        tests:
+        data_tests:
           - dbt_utils.at_least_one
           - dbt_utils.not_constant
           - dbt_utils.not_empty_string

models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml

@@ -30,7 +30,7 @@ sources:
         freshness: null
         columns:
           - name: id 
-            tests:
+            data_tests:
               - not_null
       - name: orders
         identifier: orders
@@ -40,6 +40,6 @@ sources:
           identifier: false
         columns:
           - name: id 
-            tests:
+            data_tests:
               - not_null
 

models/aggregates/_agg__models.yml

@@ -6,7 +6,7 @@ models:
     columns:
       - name: customer_id
         description: Primary key for customers
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: total_sales
@@ -17,19 +17,19 @@ models:
     columns:
       - name: region
         description: One of the five global regions.
-        tests:
+        data_tests:
           - accepted_values:
               values: ['MIDDLE EAST','AFRICA','EUROPE','ASIA','AMERICA']
 
       - name: market_segment
         description: One of the five market segments.
-        tests:
+        data_tests:
           - accepted_values:
               values: ['HOUSEHOLD','AUTOMOBILE','FURNITURE','BUILDING','MACHINERY']
 
       - name: total_sales
         description: Total sales for the region and market segment.
-        tests:
+        data_tests:
           - not_null
 
   # These two models are just different ways of doing the same thing (pivot over categories) using jinja and the PIVOT operation in Snowflake

models/intermediate/finance/_int_finance__models.yml

@@ -6,7 +6,7 @@ models:
     columns:
       - name: order_item_id
         description: '{{ doc("order_item_id") }}'
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: order_id
@@ -59,7 +59,7 @@ models:
     columns:
       - name: order_item_id
         description: '{{ doc("order_item_id") }}'
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: order_id
@@ -122,7 +122,7 @@ models:
     columns:
       - name: part_supplier_id
         description: primary id of the models
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: part_id

models/marts/finance/_finance__models.yml

@@ -12,7 +12,7 @@ models:
     columns:
       - name: order_item_id
         description: '{{ doc("order_item_id") }}'
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: order_id
@@ -69,12 +69,12 @@ models:
     columns:
       - name: order_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: customer_id
         description: foreign id for customers
-        tests:
+        data_tests:
           - relationships:
               to: ref('dim_customers')
               field: customer_id

models/marts/marketing/_marketing__models.yml

@@ -12,12 +12,12 @@ models:
     columns:
       - name: customer_id
         description: Primary id on the customers table
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: region
         description: region name
-        tests:
+        data_tests:
           - accepted_values:
               values: ['AFRICA','MIDDLE EAST','ASIA','EUROPE','AMERICA']
               severity: warn

models/marts/operations/_operations__models.yml

@@ -12,7 +12,7 @@ models:
     columns:
       - name: part_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: manufacturer
@@ -35,7 +35,7 @@ models:
     columns:
       - name: supplier_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: supplier_name

models/staging/stripe/_stripe__sources.yml

@@ -9,14 +9,14 @@ sources:
       - name: payment
         columns:
           - name: id
-            tests:
+            data_tests:
               - not_null
               - unique
           - name: paymentmethod
-            tests:
+            data_tests:
               - accepted_values:
                   values: ['credit_card', 'bank_transfer', 'gift_card', 'coupon']
           - name: status
-            tests:
+            data_tests:
               - accepted_values:
                   values: ['success', 'fail']

models/staging/tpch/_tpch__models.yml

@@ -6,7 +6,7 @@ models:
     columns:
       - name: customer_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: name
@@ -29,7 +29,7 @@ models:
     columns:
       - name: order_item_id
         description: '{{ doc("order_item_id") }}'
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: order_id
@@ -70,7 +70,7 @@ models:
     columns:
       - name: nation_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: name
@@ -85,7 +85,7 @@ models:
     columns:
       - name: order_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: customer_id
@@ -110,7 +110,7 @@ models:
     columns:
       - name: part_supplier_id
         description: surrogate id for the model -- combo of ps_partid + ps_suppid
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: part_id
@@ -129,7 +129,7 @@ models:
     columns:
       - name: part_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: name
@@ -154,7 +154,7 @@ models:
     columns:
       - name: region_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: name
@@ -167,7 +167,7 @@ models:
     columns:
       - name: supplier_id
         description: primary id of the model
-        tests:
+        data_tests:
           - unique
           - not_null
       - name: supplier_name