From 8c3db86e79b0c9f906009129c9843060c04b76ff Mon Sep 17 00:00:00 2001 From: Dan Horner Date: Fri, 28 Jun 2024 18:22:35 +0000 Subject: [PATCH 1/3] systematically replacing tests with data_tests in RO Exemplar --- models/marts/finance/_finance__models.yml | 6 +++--- models/marts/marketing/_marketing__models.yml | 4 ++-- models/marts/operations/_operations__models.yml | 4 ++-- models/staging/stripe/_stripe__sources.yml | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/models/marts/finance/_finance__models.yml b/models/marts/finance/_finance__models.yml index 25d6f0d..8408fa6 100644 --- a/models/marts/finance/_finance__models.yml +++ b/models/marts/finance/_finance__models.yml @@ -12,7 +12,7 @@ models: columns: - name: order_item_id description: '{{ doc("order_item_id") }}' - tests: + data_tests: - unique - not_null - name: order_id @@ -69,12 +69,12 @@ models: columns: - name: order_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: customer_id description: foreign id for customers - tests: + data_tests: - relationships: to: ref('dim_customers') field: customer_id diff --git a/models/marts/marketing/_marketing__models.yml b/models/marts/marketing/_marketing__models.yml index a0d3eda..553c03a 100644 --- a/models/marts/marketing/_marketing__models.yml +++ b/models/marts/marketing/_marketing__models.yml @@ -12,12 +12,12 @@ models: columns: - name: customer_id description: Primary id on the customers table - tests: + data_tests: - unique - not_null - name: region description: region name - tests: + data_tests: - accepted_values: values: ['AFRICA','MIDDLE EAST','ASIA','EUROPE','AMERICA'] severity: warn diff --git a/models/marts/operations/_operations__models.yml b/models/marts/operations/_operations__models.yml index 1d46d11..3c8f176 100644 --- a/models/marts/operations/_operations__models.yml +++ b/models/marts/operations/_operations__models.yml @@ -12,7 +12,7 @@ models: columns: - name: part_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: manufacturer @@ -35,7 +35,7 @@ models: columns: - name: supplier_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: supplier_name diff --git a/models/staging/stripe/_stripe__sources.yml b/models/staging/stripe/_stripe__sources.yml index 1656e66..5bb236b 100644 --- a/models/staging/stripe/_stripe__sources.yml +++ b/models/staging/stripe/_stripe__sources.yml @@ -9,14 +9,14 @@ sources: - name: payment columns: - name: id - tests: + data_tests: - not_null - unique - name: paymentmethod - tests: + data_tests: - accepted_values: values: ['credit_card', 'bank_transfer', 'gift_card', 'coupon'] - name: status - tests: + data_tests: - accepted_values: values: ['success', 'fail'] \ No newline at end of file From 59518cfe3765b1d6cd98383ab99271f1687aa678 Mon Sep 17 00:00:00 2001 From: Dan Horner Date: Fri, 28 Jun 2024 18:33:53 +0000 Subject: [PATCH 2/3] swapped remaining tests configs for data-test config (for v 1.8 and forward) --- .../_samples/override_naming_model_yaml.sql | 0 .../_model_governance__models.yml | 2 +- models/_samples/python/_python__models.yml | 2 +- .../_samples/snapshot/_snapshot__models.yml | 2 +- .../jaffle_shop/_jaffle_shop__models.yml | 12 ++++---- .../jaffle_shop/_jaffle_shop__sources.yml | 4 +-- models/aggregates/_agg__models.yml | 8 +++--- .../finance/_int_finance__models.yml | 6 ++-- models/staging/tpch/_tpch__models.yml | 16 +++++------ models/staging/tpch/_tpch__sources.yml | 28 +++++++++---------- 10 files changed, 40 insertions(+), 40 deletions(-) rename {models => macros}/_samples/override_naming_model_yaml.sql (100%) diff --git a/models/_samples/override_naming_model_yaml.sql b/macros/_samples/override_naming_model_yaml.sql similarity index 100% rename from models/_samples/override_naming_model_yaml.sql rename to macros/_samples/override_naming_model_yaml.sql diff --git a/models/_samples/model_governance/_model_governance__models.yml b/models/_samples/model_governance/_model_governance__models.yml index 6c83505..4d515bf 100644 --- a/models/_samples/model_governance/_model_governance__models.yml +++ b/models/_samples/model_governance/_model_governance__models.yml @@ -17,7 +17,7 @@ models: # unique constraint is NOT enforced in snowflake - type: unique warn_unenforced: false - tests: + data_tests: - unique - name: order_date data_type: date diff --git a/models/_samples/python/_python__models.yml b/models/_samples/python/_python__models.yml index 9331595..d2d23d0 100644 --- a/models/_samples/python/_python__models.yml +++ b/models/_samples/python/_python__models.yml @@ -18,7 +18,7 @@ models: columns: - name: customer_id description: Primary id on the customers table - tests: + data_tests: - unique - not_null diff --git a/models/_samples/snapshot/_snapshot__models.yml b/models/_samples/snapshot/_snapshot__models.yml index a6786f4..5b75ce9 100644 --- a/models/_samples/snapshot/_snapshot__models.yml +++ b/models/_samples/snapshot/_snapshot__models.yml @@ -4,6 +4,6 @@ models: - name: example_join_snapshots columns: - name: surrogate_key - tests: + data_tests: - unique - not_null \ No newline at end of file diff --git a/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml b/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml index a745c31..838684c 100644 --- a/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml +++ b/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml @@ -7,14 +7,14 @@ models: multi line description of my staging orders model from jaffle shop - tests: + data_tests: - dbt_utils.recency: datepart: day field: _etl_loaded_at interval: 7 columns: - name: order_id - tests: + data_tests: - not_null: config: where: _etl_loaded_at >= dateadd('day', -3, current_timestamp()) @@ -30,7 +30,7 @@ models: multi line description of my order date column from my jaffle shop orders model - tests: + data_tests: - dbt_expectations.expect_column_values_to_be_of_type: column_type: date @@ -39,7 +39,7 @@ models: multi line description of my staging jaffle shop customers model - tests: + data_tests: - not_empty - dbt_utils.equal_rowcount: compare_model: source('jaffle_shop', 'customers') @@ -59,12 +59,12 @@ models: multi line description of my customer id column from my jaffle shop customers model - tests: + data_tests: - unique - not_null - unique_and_not_null - name: first_name - tests: + data_tests: - dbt_utils.at_least_one - dbt_utils.not_constant - dbt_utils.not_empty_string diff --git a/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml b/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml index be514df..e5341aa 100644 --- a/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml +++ b/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml @@ -30,7 +30,7 @@ sources: freshness: null columns: - name: id - tests: + data_tests: - not_null - name: orders identifier: orders @@ -40,6 +40,6 @@ sources: identifier: false columns: - name: id - tests: + data_tests: - not_null \ No newline at end of file diff --git a/models/aggregates/_agg__models.yml b/models/aggregates/_agg__models.yml index 8bd5f47..8bc7fb8 100644 --- a/models/aggregates/_agg__models.yml +++ b/models/aggregates/_agg__models.yml @@ -6,7 +6,7 @@ models: columns: - name: customer_id description: Primary key for customers - tests: + data_tests: - unique - not_null - name: total_sales @@ -17,19 +17,19 @@ models: columns: - name: region description: One of the five global regions. - tests: + data_tests: - accepted_values: values: ['MIDDLE EAST','AFRICA','EUROPE','ASIA','AMERICA'] - name: market_segment description: One of the five market segments. - tests: + data_tests: - accepted_values: values: ['HOUSEHOLD','AUTOMOBILE','FURNITURE','BUILDING','MACHINERY'] - name: total_sales description: Total sales for the region and market segment. - tests: + data_tests: - not_null # These two models are just different ways of doing the same thing (pivot over categories) using jinja and the PIVOT operation in Snowflake diff --git a/models/intermediate/finance/_int_finance__models.yml b/models/intermediate/finance/_int_finance__models.yml index 480cc11..06c3e1d 100644 --- a/models/intermediate/finance/_int_finance__models.yml +++ b/models/intermediate/finance/_int_finance__models.yml @@ -6,7 +6,7 @@ models: columns: - name: order_item_id description: '{{ doc("order_item_id") }}' - tests: + data_tests: - unique - not_null - name: order_id @@ -59,7 +59,7 @@ models: columns: - name: order_item_id description: '{{ doc("order_item_id") }}' - tests: + data_tests: - unique - not_null - name: order_id @@ -122,7 +122,7 @@ models: columns: - name: part_supplier_id description: primary id of the models - tests: + data_tests: - unique - not_null - name: part_id diff --git a/models/staging/tpch/_tpch__models.yml b/models/staging/tpch/_tpch__models.yml index 48bb0d4..c8d75fb 100644 --- a/models/staging/tpch/_tpch__models.yml +++ b/models/staging/tpch/_tpch__models.yml @@ -6,7 +6,7 @@ models: columns: - name: customer_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: name @@ -29,7 +29,7 @@ models: columns: - name: order_item_id description: '{{ doc("order_item_id") }}' - tests: + data_tests: - unique - not_null - name: order_id @@ -70,7 +70,7 @@ models: columns: - name: nation_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: name @@ -85,7 +85,7 @@ models: columns: - name: order_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: customer_id @@ -110,7 +110,7 @@ models: columns: - name: part_supplier_id description: surrogate id for the model -- combo of ps_partid + ps_suppid - tests: + data_tests: - unique - not_null - name: part_id @@ -129,7 +129,7 @@ models: columns: - name: part_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: name @@ -154,7 +154,7 @@ models: columns: - name: region_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: name @@ -167,7 +167,7 @@ models: columns: - name: supplier_id description: primary id of the model - tests: + data_tests: - unique - not_null - name: supplier_name diff --git a/models/staging/tpch/_tpch__sources.yml b/models/staging/tpch/_tpch__sources.yml index 6b4fe66..c59f71a 100644 --- a/models/staging/tpch/_tpch__sources.yml +++ b/models/staging/tpch/_tpch__sources.yml @@ -16,13 +16,13 @@ sources: columns: - name: o_orderkey description: SF*1,500,000 are sparsely populated - tests: + data_tests: - unique - not_null - name: o_custkey description: Foreign key to C_custkey - tests: + data_tests: - relationships: to: source('tpch', 'customer') field: c_custkey @@ -46,7 +46,7 @@ sources: columns: - name: c_custkey description: unique customer key - tests: + data_tests: - unique: severity: warn - not_null: @@ -71,19 +71,19 @@ sources: columns: - name: l_orderkey description: Foreign key to O_ORDERkey - tests: + data_tests: - relationships: to: source('tpch', 'orders') field: o_orderkey - name: l_partkey description: Foreign key to P_PARTkey, first part of the compound Foreign key to (PS_PARTkey,PS_SUPPkey) with L_SUPPkey - tests: + data_tests: - relationships: to: source('tpch', 'part') field: p_partkey - name: l_suppkey description: Foreign key to S_SUPPkey, second part of the compound Foreign key to (PS_PARTkey, PS_SUPPkey) with L_PARTkey - tests: + data_tests: - relationships: to: source('tpch', 'supplier') field: s_suppkey @@ -119,7 +119,7 @@ sources: columns: - name: n_nationkey description: 25 nations are populated - tests: + data_tests: - unique: severity: warn - not_null: @@ -128,7 +128,7 @@ sources: description: nation name - name: n_regionkey description: Foreign key to R_REGIONkey - tests: + data_tests: - relationships: to: source('tpch', 'region') field: r_regionkey @@ -140,7 +140,7 @@ sources: columns: - name: p_partkey description: SF*200,000 are populated - tests: + data_tests: - unique: severity: warn - not_null: @@ -167,13 +167,13 @@ sources: columns: - name: ps_partkey description: Foreign key to P_PARTkey - tests: + data_tests: - relationships: to: source('tpch', 'part') field: p_partkey - name: ps_suppkey description: Foreign key to S_SUPPkey - tests: + data_tests: - relationships: to: source('tpch', 'supplier') field: s_suppkey @@ -189,7 +189,7 @@ sources: columns: - name: r_regionkey description: 5 regions are populated - tests: + data_tests: - unique: severity: warn - not_null: @@ -204,7 +204,7 @@ sources: columns: - name: s_suppkey description: SF*10,000 are populated - tests: + data_tests: - unique: severity: warn - not_null: @@ -215,7 +215,7 @@ sources: description: address of the supplier - name: s_nationkey description: Foreign key to N_NATIONkey - tests: + data_tests: - relationships: to: source('tpch', 'nation') field: n_nationkey From b15d482b6cdd55c2444831bd39699f90a09f0855 Mon Sep 17 00:00:00 2001 From: Dan Horner Date: Fri, 28 Jun 2024 18:47:02 +0000 Subject: [PATCH 3/3] final data-test addition --- models/_samples/snapshot/_snapshot__models.yml | 4 +++- models/_samples/snapshot/example_join_snapshots.sql | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/models/_samples/snapshot/_snapshot__models.yml b/models/_samples/snapshot/_snapshot__models.yml index 5b75ce9..ea193d9 100644 --- a/models/_samples/snapshot/_snapshot__models.yml +++ b/models/_samples/snapshot/_snapshot__models.yml @@ -5,5 +5,7 @@ models: columns: - name: surrogate_key data_tests: - - unique + - unique: + config: + where: valid_to = '{{ var("future_proof_date") }}' - not_null \ No newline at end of file diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.sql index 0281922..201d1c7 100644 --- a/models/_samples/snapshot/example_join_snapshots.sql +++ b/models/_samples/snapshot/example_join_snapshots.sql @@ -46,4 +46,4 @@ final as ( from joined ) -select * from final \ No newline at end of file +select * from final