diff --git a/macros/_samples/override_naming_model_yaml.sql b/macros/_samples/override_naming_model_yaml.sql new file mode 100644 index 0000000..09a8eb3 --- /dev/null +++ b/macros/_samples/override_naming_model_yaml.sql @@ -0,0 +1,56 @@ +--this model override allows users to avoid dbt's "auto lowercasing" behavior when using codegen to generate model yaml. +--To actually use this macro, put it in the macros folder and name it generate_model_yaml.sql. +{% macro generate_column_yaml(column, model_yaml, column_desc_dict, parent_column_name="") %} + {% if parent_column_name %} + {% set column_name = parent_column_name ~ "." ~ column.name %} + {% else %} + {% set column_name = column.name %} + {% endif %} +-- removing "lower" from whole macro + {% do model_yaml.append(' - name: ' ~ column.name) %} + {% do model_yaml.append(' description: "' ~ column_desc_dict.get(column.name) ~ '"') %} + {% do model_yaml.append('') %} + + {% if column.fields|length > 0 %} + {% for child_column in column.fields %} + {% set model_yaml = codegen.generate_column_yaml(child_column, model_yaml, column_desc_dict, parent_column_name=column_name) %} + {% endfor %} + {% endif %} + {% do return(model_yaml) %} +{% endmacro %} + +{% macro generate_model_yaml(model_names=[], upstream_descriptions=False) %} + + {% set model_yaml=[] %} + + {% do model_yaml.append('version: 2') %} + {% do model_yaml.append('') %} + {% do model_yaml.append('models:') %} + + {% if model_names is string %} + {{ exceptions.raise_compiler_error("The `model_names` argument must always be a list, even if there is only one model.") }} + {% else %} + {% for model in model_names %} + {% do model_yaml.append(' - name: ' ~ model ) %} + {% do model_yaml.append(' description: ""') %} + {% do model_yaml.append(' columns:') %} + + {% set relation=ref(model) %} + {%- set columns = adapter.get_columns_in_relation(relation) -%} + {% set column_desc_dict = codegen.build_dict_column_descriptions(model) if upstream_descriptions else {} %} + + {% for column in columns %} + {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %} + {% endfor %} + {% endfor %} + {% endif %} + +{% if execute %} + + {% set joined = model_yaml | join ('\n') %} + {{ log(joined, info=True) }} + {% do return(joined) %} + +{% endif %} + +{% endmacro %} \ No newline at end of file diff --git a/models/_samples/model_governance/_model_governance__models.yml b/models/_samples/model_governance/_model_governance__models.yml index 6c83505..4d515bf 100644 --- a/models/_samples/model_governance/_model_governance__models.yml +++ b/models/_samples/model_governance/_model_governance__models.yml @@ -17,7 +17,7 @@ models: # unique constraint is NOT enforced in snowflake - type: unique warn_unenforced: false - tests: + data_tests: - unique - name: order_date data_type: date diff --git a/models/_samples/python/_python__models.yml b/models/_samples/python/_python__models.yml index 9331595..d2d23d0 100644 --- a/models/_samples/python/_python__models.yml +++ b/models/_samples/python/_python__models.yml @@ -18,7 +18,7 @@ models: columns: - name: customer_id description: Primary id on the customers table - tests: + data_tests: - unique - not_null diff --git a/models/_samples/snapshot/_snapshot__models.yml b/models/_samples/snapshot/_snapshot__models.yml index a6d91e8..ea193d9 100644 --- a/models/_samples/snapshot/_snapshot__models.yml +++ b/models/_samples/snapshot/_snapshot__models.yml @@ -4,5 +4,8 @@ models: - name: example_join_snapshots columns: - name: surrogate_key - tests: + data_tests: + - unique: + config: + where: valid_to = '{{ var("future_proof_date") }}' - not_null \ No newline at end of file diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.sql index 0281922..201d1c7 100644 --- a/models/_samples/snapshot/example_join_snapshots.sql +++ b/models/_samples/snapshot/example_join_snapshots.sql @@ -46,4 +46,4 @@ final as ( from joined ) -select * from final \ No newline at end of file +select * from final diff --git a/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml b/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml index a745c31..838684c 100644 --- a/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml +++ b/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml @@ -7,14 +7,14 @@ models: multi line description of my staging orders model from jaffle shop - tests: + data_tests: - dbt_utils.recency: datepart: day field: _etl_loaded_at interval: 7 columns: - name: order_id - tests: + data_tests: - not_null: config: where: _etl_loaded_at >= dateadd('day', -3, current_timestamp()) @@ -30,7 +30,7 @@ models: multi line description of my order date column from my jaffle shop orders model - tests: + data_tests: - dbt_expectations.expect_column_values_to_be_of_type: column_type: date @@ -39,7 +39,7 @@ models: multi line description of my staging jaffle shop customers model - tests: + data_tests: - not_empty - dbt_utils.equal_rowcount: compare_model: source('jaffle_shop', 'customers') @@ -59,12 +59,12 @@ models: multi line description of my customer id column from my jaffle shop customers model - tests: + data_tests: - unique - not_null - unique_and_not_null - name: first_name - tests: + data_tests: - dbt_utils.at_least_one - dbt_utils.not_constant - dbt_utils.not_empty_string diff --git a/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml b/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml index be514df..e5341aa 100644 --- a/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml +++ b/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml @@ -30,7 +30,7 @@ sources: freshness: null columns: - name: id - tests: + data_tests: - not_null - name: orders identifier: orders @@ -40,6 +40,6 @@ sources: identifier: false columns: - name: id - tests: + data_tests: - not_null \ No newline at end of file diff --git a/models/aggregates/_agg__models.yml b/models/aggregates/_agg__models.yml index 8bd5f47..8bc7fb8 100644 --- a/models/aggregates/_agg__models.yml +++ b/models/aggregates/_agg__models.yml @@ -6,7 +6,7 @@ models: columns: - name: customer_id description: Primary key for customers - tests: + data_tests: - unique - not_null - name: total_sales @@ -17,19 +17,19 @@ models: columns: - name: region description: One of the five global regions. - tests: + data_tests: - accepted_values: values: ['MIDDLE EAST','AFRICA','EUROPE','ASIA','AMERICA'] - name: market_segment description: One of the five market segments. - tests: + data_tests: - accepted_values: values: ['HOUSEHOLD','AUTOMOBILE','FURNITURE','BUILDING','MACHINERY'] - name: total_sales description: Total sales for the region and market segment. - tests: + data_tests: - not_null # These two models are just different ways of doing the same thing (pivot over categories) using jinja and the PIVOT operation in Snowflake diff --git a/models/intermediate/finance/_int_finance__models.yml b/models/intermediate/finance/_int_finance__models.yml index 480cc11..06c3e1d 100644 --- a/models/intermediate/finance/_int_finance__models.yml +++ b/models/intermediate/finance/_int_finance__models.yml @@ -6,7 +6,7 @@ models: columns: - name: order_item_id description: '{{ doc("order_item_id") }}' - tests: + data_tests: - unique - not_null - name: order_id @@ -59,7 +59,7 @@ models: columns: - name: order_item_id description: '{{ doc("order_item_id") }}' - tests: + data_tests: - unique - not_null - name: order_id @@ -122,7 +122,7 @@ models: columns: - name: part_supplier_id description: primary id of the models - tests: + data_tests: - unique - not_null - name: part_id diff --git a/models/marts/finance/_finance__models.yml b/models/marts/finance/_finance__models.yml index 25d6f0d..8408fa6 100644 --- a/models/marts/finance/_finance__models.yml +++ b/models/marts/finance/_finance__models.yml @@ -12,7 +12,7 @@ models: columns: - name: order_item_id description: '{{ doc("order_item_id") }}' - tests: + data_tests: - unique - not_null - name: order_id @@ -69,12 +69,12 @@ models: columns: - name: order_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: customer_id description: foreign id for customers - tests: + data_tests: - relationships: to: ref('dim_customers') field: customer_id diff --git a/models/marts/marketing/_marketing__models.yml b/models/marts/marketing/_marketing__models.yml index a0d3eda..553c03a 100644 --- a/models/marts/marketing/_marketing__models.yml +++ b/models/marts/marketing/_marketing__models.yml @@ -12,12 +12,12 @@ models: columns: - name: customer_id description: Primary id on the customers table - tests: + data_tests: - unique - not_null - name: region description: region name - tests: + data_tests: - accepted_values: values: ['AFRICA','MIDDLE EAST','ASIA','EUROPE','AMERICA'] severity: warn diff --git a/models/marts/operations/_operations__models.yml b/models/marts/operations/_operations__models.yml index 1d46d11..3c8f176 100644 --- a/models/marts/operations/_operations__models.yml +++ b/models/marts/operations/_operations__models.yml @@ -12,7 +12,7 @@ models: columns: - name: part_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: manufacturer @@ -35,7 +35,7 @@ models: columns: - name: supplier_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: supplier_name diff --git a/models/staging/stripe/_stripe__sources.yml b/models/staging/stripe/_stripe__sources.yml index 1656e66..5bb236b 100644 --- a/models/staging/stripe/_stripe__sources.yml +++ b/models/staging/stripe/_stripe__sources.yml @@ -9,14 +9,14 @@ sources: - name: payment columns: - name: id - tests: + data_tests: - not_null - unique - name: paymentmethod - tests: + data_tests: - accepted_values: values: ['credit_card', 'bank_transfer', 'gift_card', 'coupon'] - name: status - tests: + data_tests: - accepted_values: values: ['success', 'fail'] \ No newline at end of file diff --git a/models/staging/tpch/_tpch__models.yml b/models/staging/tpch/_tpch__models.yml index 48bb0d4..c8d75fb 100644 --- a/models/staging/tpch/_tpch__models.yml +++ b/models/staging/tpch/_tpch__models.yml @@ -6,7 +6,7 @@ models: columns: - name: customer_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: name @@ -29,7 +29,7 @@ models: columns: - name: order_item_id description: '{{ doc("order_item_id") }}' - tests: + data_tests: - unique - not_null - name: order_id @@ -70,7 +70,7 @@ models: columns: - name: nation_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: name @@ -85,7 +85,7 @@ models: columns: - name: order_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: customer_id @@ -110,7 +110,7 @@ models: columns: - name: part_supplier_id description: surrogate id for the model -- combo of ps_partid + ps_suppid - tests: + data_tests: - unique - not_null - name: part_id @@ -129,7 +129,7 @@ models: columns: - name: part_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: name @@ -154,7 +154,7 @@ models: columns: - name: region_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: name @@ -167,7 +167,7 @@ models: columns: - name: supplier_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: supplier_name diff --git a/models/staging/tpch/_tpch__sources.yml b/models/staging/tpch/_tpch__sources.yml index 6b4fe66..c59f71a 100644 --- a/models/staging/tpch/_tpch__sources.yml +++ b/models/staging/tpch/_tpch__sources.yml @@ -16,13 +16,13 @@ sources: columns: - name: o_orderkey description: SF*1,500,000 are sparsely populated - tests: + data_tests: - unique - not_null - name: o_custkey description: Foreign key to C_custkey - tests: + data_tests: - relationships: to: source('tpch', 'customer') field: c_custkey @@ -46,7 +46,7 @@ sources: columns: - name: c_custkey description: unique customer key - tests: + data_tests: - unique: severity: warn - not_null: @@ -71,19 +71,19 @@ sources: columns: - name: l_orderkey description: Foreign key to O_ORDERkey - tests: + data_tests: - relationships: to: source('tpch', 'orders') field: o_orderkey - name: l_partkey description: Foreign key to P_PARTkey, first part of the compound Foreign key to (PS_PARTkey,PS_SUPPkey) with L_SUPPkey - tests: + data_tests: - relationships: to: source('tpch', 'part') field: p_partkey - name: l_suppkey description: Foreign key to S_SUPPkey, second part of the compound Foreign key to (PS_PARTkey, PS_SUPPkey) with L_PARTkey - tests: + data_tests: - relationships: to: source('tpch', 'supplier') field: s_suppkey @@ -119,7 +119,7 @@ sources: columns: - name: n_nationkey description: 25 nations are populated - tests: + data_tests: - unique: severity: warn - not_null: @@ -128,7 +128,7 @@ sources: description: nation name - name: n_regionkey description: Foreign key to R_REGIONkey - tests: + data_tests: - relationships: to: source('tpch', 'region') field: r_regionkey @@ -140,7 +140,7 @@ sources: columns: - name: p_partkey description: SF*200,000 are populated - tests: + data_tests: - unique: severity: warn - not_null: @@ -167,13 +167,13 @@ sources: columns: - name: ps_partkey description: Foreign key to P_PARTkey - tests: + data_tests: - relationships: to: source('tpch', 'part') field: p_partkey - name: ps_suppkey description: Foreign key to S_SUPPkey - tests: + data_tests: - relationships: to: source('tpch', 'supplier') field: s_suppkey @@ -189,7 +189,7 @@ sources: columns: - name: r_regionkey description: 5 regions are populated - tests: + data_tests: - unique: severity: warn - not_null: @@ -204,7 +204,7 @@ sources: columns: - name: s_suppkey description: SF*10,000 are populated - tests: + data_tests: - unique: severity: warn - not_null: @@ -215,7 +215,7 @@ sources: description: address of the supplier - name: s_nationkey description: Foreign key to N_NATIONkey - tests: + data_tests: - relationships: to: source('tpch', 'nation') field: n_nationkey