From b8aac1ece1eb90995e961dff3b2888e2e9ac3b5a Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Mon, 16 Sep 2024 23:48:11 +0000 Subject: [PATCH 01/15] Anything with commit changes --- analysis/test_file.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 analysis/test_file.sql diff --git a/analysis/test_file.sql b/analysis/test_file.sql new file mode 100644 index 0000000..e69de29 From 673b5d12cd04b0b9b4f99af87d13b8e022aa87fd Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:48:24 +0000 Subject: [PATCH 02/15] refactoring snapshot_stg_payments and snapshot_stg_payments to match new snapshot strategy --- snapshots/jaffle_snapshots.yml | 24 ++++++++++++++++++++++++ snapshots/snapshot_int_line_items.sql | 22 ---------------------- snapshots/snapshot_stg_payments.sql | 16 ---------------- 3 files changed, 24 insertions(+), 38 deletions(-) create mode 100644 snapshots/jaffle_snapshots.yml delete mode 100644 snapshots/snapshot_int_line_items.sql delete mode 100644 snapshots/snapshot_stg_payments.sql diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml new file mode 100644 index 0000000..707111d --- /dev/null +++ b/snapshots/jaffle_snapshots.yml @@ -0,0 +1,24 @@ +version: 2 + +snapshots: + - name: snapshot_stg_payments + relation: source('stripe', 'payment') + config: + enabled: true + strategy: timestamp + updated_at: _BATCHED_AT + unique_key: ID + persist_docs: + relation: true + columns: true + - name: snapshot_int_line_items + relation: ref('int_line_items_amounts_calculated') + config: + enabled: true + strategy: check + check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] + unique_key: order_item_id + persist_docs: + relation: true + columns: true + diff --git a/snapshots/snapshot_int_line_items.sql b/snapshots/snapshot_int_line_items.sql deleted file mode 100644 index 90bc550..0000000 --- a/snapshots/snapshot_int_line_items.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% snapshot snapshot_int_line_items %} - --- Example showing check columns strategy and using custom schemas (this is rare) - --- if you don't want to write to the same schema in development and prod, --- use environment-aware variable from dbt_project.yml - --- if you do not have a reliable updated at timestamp, checek columns for changes. - -{{ - config( - target_database='analytics', - target_schema=var('example_target_snapshot_schema'), - unique_key='order_item_id', - strategy='check', - check_cols=['gross_item_sales_amount', 'net_item_sales_amount'], - ) -}} - -select * from {{ ref('int_line_items_amounts_calculated') }} - -{% endsnapshot %} \ No newline at end of file diff --git a/snapshots/snapshot_stg_payments.sql b/snapshots/snapshot_stg_payments.sql deleted file mode 100644 index adc61b2..0000000 --- a/snapshots/snapshot_stg_payments.sql +++ /dev/null @@ -1,16 +0,0 @@ -{% snapshot snapshot_stg_payments %} - -{{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='ID', - - strategy='timestamp', - updated_at='_BATCHED_AT', - ) -}} - -select * from {{ source('stripe', 'payment') }} - -{% endsnapshot %} \ No newline at end of file From 910cccc4e6de35d25db9cf91797270ad64ad244a Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:49:41 +0000 Subject: [PATCH 03/15] deleting generate schema snapshot--i don't think this is relevant anymore given that snapshots now build to your dev schema then whatever your target is in prod. --- .../_samples/example_generate_schema_snapshot.sql | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 snapshots/_samples/example_generate_schema_snapshot.sql diff --git a/snapshots/_samples/example_generate_schema_snapshot.sql b/snapshots/_samples/example_generate_schema_snapshot.sql deleted file mode 100644 index 2174d7b..0000000 --- a/snapshots/_samples/example_generate_schema_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_generate_schema_snapshot %} - {{ - config( - target_database='analytics', - target_schema=generate_schema_name('snapshots'), - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file From c11e92d4214ce783cee4d686f01d35efb8334553 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:55:03 +0000 Subject: [PATCH 04/15] deleting old orders_line_items snapshot, replacing it with new config. also commenting out example join snapshot. --- .../_samples/snapshot/example_join_snapshots.sql | 3 ++- .../example_orders_line_items_snapshot.sql | 14 -------------- snapshots/jaffle_snapshots.yml | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 15 deletions(-) delete mode 100644 snapshots/_samples/example_orders_line_items_snapshot.sql diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.sql index 201d1c7..9ea20d4 100644 --- a/models/_samples/snapshot/example_join_snapshots.sql +++ b/models/_samples/snapshot/example_join_snapshots.sql @@ -1,4 +1,4 @@ -with order_snapshot as ( +{# with order_snapshot as ( select * exclude dbt_valid_to, coalesce(dbt_valid_to, cast('{{ var("future_proof_date") }}' as timestamp)) as dbt_valid_to @@ -47,3 +47,4 @@ final as ( ) select * from final +#} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_line_items_snapshot.sql b/snapshots/_samples/example_orders_line_items_snapshot.sql deleted file mode 100644 index 63603f5..0000000 --- a/snapshots/_samples/example_orders_line_items_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_line_items_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index 707111d..eebfaba 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -1,6 +1,7 @@ version: 2 snapshots: +# This is a pretty standard snapshot - name: snapshot_stg_payments relation: source('stripe', 'payment') config: @@ -11,6 +12,7 @@ snapshots: persist_docs: relation: true columns: true +# This snapshot has the check_cols example - name: snapshot_int_line_items relation: ref('int_line_items_amounts_calculated') config: @@ -21,4 +23,16 @@ snapshots: persist_docs: relation: true columns: true +# This snapshot has invalidate hard deletes as an example + - name: snapshot_orders_line_items + relation: ref('example_orders_line_items_source_for_snapshot') + config: + enabled: true + strategy: timestamp + unique_key: id + updated_at: order_updated_at + invalidate_hard_deletes: true + persist_docs: + relation: true + columns: true From 080bb9468f9756b2c7b8cf84e4053db58911a8ae Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:58:33 +0000 Subject: [PATCH 05/15] refactoring example_orders_snapshot into new way --- snapshots/_samples/example_orders_snapshot.sql | 14 -------------- snapshots/jaffle_snapshots.yml | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 15 deletions(-) delete mode 100644 snapshots/_samples/example_orders_snapshot.sql diff --git a/snapshots/_samples/example_orders_snapshot.sql b/snapshots/_samples/example_orders_snapshot.sql deleted file mode 100644 index e4e7ad6..0000000 --- a/snapshots/_samples/example_orders_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='order_id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index eebfaba..afa84f2 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -12,6 +12,19 @@ snapshots: persist_docs: relation: true columns: true +# Another standard snapshot but this time with a different table. + - name: snapshot_example_orders + relation: ref('example_orders_source_for_snapshot') + config: + enabled: true + strategy: timestamp + updated_at: order_updated_at + unique_key: ID + persist_docs: + relation: true + columns: true + invalidate_hard_deletes: true + # This snapshot has the check_cols example - name: snapshot_int_line_items relation: ref('int_line_items_amounts_calculated') @@ -19,7 +32,7 @@ snapshots: enabled: true strategy: check check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] - unique_key: order_item_id + unique_key: order_id persist_docs: relation: true columns: true From 2c8532a0cd1f96a045b2df26035aaed1346885b3 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:02:37 +0000 Subject: [PATCH 06/15] convert example join snapshot to markdown till better fix avail --- .../{example_join_snapshots.sql => example_join_snapshots.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/_samples/snapshot/{example_join_snapshots.sql => example_join_snapshots.md} (100%) diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.md similarity index 100% rename from models/_samples/snapshot/example_join_snapshots.sql rename to models/_samples/snapshot/example_join_snapshots.md From ef2f5474aa2aa97d03bab1f439670e7cec68c571 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:04:18 +0000 Subject: [PATCH 07/15] fixing order snapshot --- snapshots/jaffle_snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index afa84f2..c7bd414 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -19,7 +19,7 @@ snapshots: enabled: true strategy: timestamp updated_at: order_updated_at - unique_key: ID + unique_key: order_id persist_docs: relation: true columns: true From 833d85cdf5cae3607cd121300de36108f2c7cdec Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:11:36 +0000 Subject: [PATCH 08/15] fixing snapshot line items --- snapshots/jaffle_snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index c7bd414..98b9695 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -32,7 +32,7 @@ snapshots: enabled: true strategy: check check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] - unique_key: order_id + unique_key: order_item_id persist_docs: relation: true columns: true From ba8118a332db57181a03ca010d185cc5bbdda41d Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Mon, 18 Nov 2024 19:22:31 +0000 Subject: [PATCH 09/15] Commit changes --- macros/test_macro.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 macros/test_macro.sql diff --git a/macros/test_macro.sql b/macros/test_macro.sql new file mode 100644 index 0000000..0db9069 --- /dev/null +++ b/macros/test_macro.sql @@ -0,0 +1,5 @@ +{% macro test_macro() %} + + {{ log("This is a message by log macro", info=True) }} + +{% endmacro %} \ No newline at end of file From 6f267b4ab12f1ecc1e68c3081740843e80fc4f56 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Tue, 26 Nov 2024 02:31:23 +0000 Subject: [PATCH 10/15] Commit changes for tests --- dbt_project.yml | 5 +-- .../orders/int_daily_orders_fact.sql | 36 ++++++++++++++++++ models/marts/dims/_dims__models.yml | 38 +++++++++++++++++++ models/marts/dims/_dims__unit_tests | 12 ++++++ models/marts/dims/dim_countries.sql | 20 ++++++++++ models/marts/dims/dim_customer_info.sql | 16 ++++++++ models/marts/dims/dim_dates.sql | 19 ++++++++++ models/marts/dims/dim_orders.sql | 21 ++++++++++ models/marts/facts/_facts__models.yml | 10 +++++ models/marts/facts/fct_daily_orders.sql | 25 ++++++++++++ models/test_model.sql | 18 +++++++++ models/test_query.sql | 0 seeds/_seeds_properties.yml | 7 ++++ seeds/test_seed.csv | 2 + 14 files changed, 225 insertions(+), 4 deletions(-) create mode 100644 models/intermediate/orders/int_daily_orders_fact.sql create mode 100644 models/marts/dims/_dims__models.yml create mode 100644 models/marts/dims/_dims__unit_tests create mode 100644 models/marts/dims/dim_countries.sql create mode 100644 models/marts/dims/dim_customer_info.sql create mode 100644 models/marts/dims/dim_dates.sql create mode 100644 models/marts/dims/dim_orders.sql create mode 100644 models/marts/facts/_facts__models.yml create mode 100644 models/marts/facts/fct_daily_orders.sql create mode 100644 models/test_model.sql create mode 100644 models/test_query.sql create mode 100644 seeds/_seeds_properties.yml create mode 100644 seeds/test_seed.csv diff --git a/dbt_project.yml b/dbt_project.yml index 0801a25..bc8f141 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -65,7 +65,4 @@ models: +materialized: ephemeral tests: - rapid_onboarding_exemplar: - _samples: - staging: - +store_failures: "{{ true if target.name == 'prod' else false }}" + +store_failures: false diff --git a/models/intermediate/orders/int_daily_orders_fact.sql b/models/intermediate/orders/int_daily_orders_fact.sql new file mode 100644 index 0000000..afe02ff --- /dev/null +++ b/models/intermediate/orders/int_daily_orders_fact.sql @@ -0,0 +1,36 @@ +with + +dates as ( + + select * from {{ ref('dim_dates') }} + +), + +orders as ( + + select * from {{ ref('dim_orders') }} + +), + +countries as ( + + select * from {{ ref('dim_countries') }} + +), + +join_sources as ( + + select + ordr.order_sk, + country.country_sk as order_country_origin_sk, + date.date_sk as order_date_sk, + ordr.order_quantity + from orders ordr + left join dates date + on ordr.order_date = date.calendar_date + left join countries country + on ordr.order_country_origin = country.country_name + +) + +select * from join_sources diff --git a/models/marts/dims/_dims__models.yml b/models/marts/dims/_dims__models.yml new file mode 100644 index 0000000..934e558 --- /dev/null +++ b/models/marts/dims/_dims__models.yml @@ -0,0 +1,38 @@ +version: 2 + +models: + - name: dim_orders + columns: + - name: order_sk + data_tests: + - unique + - not_null + + - name: dim_countries + columns: + - name: country_name + data_tests: + - accepted_values: + values: + - "United States" + - "France" + - "Italy" + - "Philippines" + +unit_tests: + - name: test_if_full_name_format_is_correct + description: "Checks if the format for customer full name is correct" + model: dim_customer_info + config: + tags: | + {%- if target.name == 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + given: + - input: ref('dim_orders') + rows: + - {"cust_fname": "Johnny", "cust_lname": "Bravo"} + + expect: + rows: + - {cust_fullname: "Bravo, Johnny"} \ No newline at end of file diff --git a/models/marts/dims/_dims__unit_tests b/models/marts/dims/_dims__unit_tests new file mode 100644 index 0000000..4ee8197 --- /dev/null +++ b/models/marts/dims/_dims__unit_tests @@ -0,0 +1,12 @@ +unit_tests: + - name: test_if_full_name_format_is_correct + description: "Checks if the format for customer full name is correct" + model: dim_customer_info + given: + - input: ref('dim_orders') + rows: + - {"cust_fname": "Johnny", "cust_lname": "Bravo"} + + expect: + rows: + - {cust_fullname: "Bravo, Johnny"} diff --git a/models/marts/dims/dim_countries.sql b/models/marts/dims/dim_countries.sql new file mode 100644 index 0000000..d70eda8 --- /dev/null +++ b/models/marts/dims/dim_countries.sql @@ -0,0 +1,20 @@ +{{ + config( + materialized='table' + ) +}} + +with + +countries as ( + + select 1 as country_sk, 'United States'::varchar(100) as country_name union all + select 2 as country_sk, 'France'::varchar(100) as country_name union all + select 3 as country_sk, 'Italy'::varchar(100) as country_name union all + select 4 as country_sk, 'Philippines'::varchar(100) as country_name + -- union all + -- select 5 as country_sk, 'Italia'::varchar(100) as country_name + +) + +select * from countries \ No newline at end of file diff --git a/models/marts/dims/dim_customer_info.sql b/models/marts/dims/dim_customer_info.sql new file mode 100644 index 0000000..a81f840 --- /dev/null +++ b/models/marts/dims/dim_customer_info.sql @@ -0,0 +1,16 @@ +with + +dim_orders as ( + + select * from {{ ref('dim_orders') }} + +), + +final as ( + + select + cust_lname || ', ' || cust_fname as cust_fullname, + from dim_orders +) + +select * from final diff --git a/models/marts/dims/dim_dates.sql b/models/marts/dims/dim_dates.sql new file mode 100644 index 0000000..af9975e --- /dev/null +++ b/models/marts/dims/dim_dates.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='table' + ) +}} + +with + +dates as ( + + select '20240101'::decimal(18,2) as date_sk, '2024-01-01'::date as calendar_date union all + select '20240102'::decimal(18,2) as date_sk, '2024-01-02'::date as calendar_date union all + select '20240103'::decimal(18,2) as date_sk, '2024-01-03'::date as calendar_date union all + select '20240104'::decimal(18,2) as date_sk, '2024-01-04'::date as calendar_date union all + select '20240105'::decimal(18,2) as date_sk, '2024-01-05'::date as calendar_date + +) + +select * from dates \ No newline at end of file diff --git a/models/marts/dims/dim_orders.sql b/models/marts/dims/dim_orders.sql new file mode 100644 index 0000000..1f0f147 --- /dev/null +++ b/models/marts/dims/dim_orders.sql @@ -0,0 +1,21 @@ +{{ + config( + materialized='table' + ) +}} + + +with + +orders as ( + + select '1'::decimal(18,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname, 1 as order_quantity union all + select '2'::decimal(18,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname, 2 as order_quantity union all + select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname, 3 as order_quantity + -- union all + -- select NULL::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Johnny'::varchar(10) as cust_fname, 'English'::varchar(10) as cust_lname, 3 as order_quantity union all + -- select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Mister'::varchar(10) as cust_fname, 'Bean'::varchar(10) as cust_lname, 3 as order_quantity + +) + +select * from orders \ No newline at end of file diff --git a/models/marts/facts/_facts__models.yml b/models/marts/facts/_facts__models.yml new file mode 100644 index 0000000..553f0f7 --- /dev/null +++ b/models/marts/facts/_facts__models.yml @@ -0,0 +1,10 @@ +version: 2 + +models: + - name: fct_daily_orders + columns: + - name: order_sk + tests: + - relationships: + to: ref('dim_orders') + field: order_sk \ No newline at end of file diff --git a/models/marts/facts/fct_daily_orders.sql b/models/marts/facts/fct_daily_orders.sql new file mode 100644 index 0000000..a3d5bd5 --- /dev/null +++ b/models/marts/facts/fct_daily_orders.sql @@ -0,0 +1,25 @@ +with + +int_daily_orders_fact as ( + + select * from {{ ref('int_daily_orders_fact') }} + +), + +agg_order_quantity as ( + + select + order_sk, + order_date_sk, + order_country_origin_sk, + sum(order_quantity) as total_order_quantity + from int_daily_orders_fact + group by + order_sk, + order_date_sk, + order_country_origin_sk + +) + +select * from agg_order_quantity + diff --git a/models/test_model.sql b/models/test_model.sql new file mode 100644 index 0000000..e36a24f --- /dev/null +++ b/models/test_model.sql @@ -0,0 +1,18 @@ +{{ + config( + materialized='table' + ) +}} + + +with + +src as ( + + select 1 as col_id + +) + +select * from src + +{{ log("This is a log inside the model", info=True) }} \ No newline at end of file diff --git a/models/test_query.sql b/models/test_query.sql new file mode 100644 index 0000000..e69de29 diff --git a/seeds/_seeds_properties.yml b/seeds/_seeds_properties.yml new file mode 100644 index 0000000..c4c5956 --- /dev/null +++ b/seeds/_seeds_properties.yml @@ -0,0 +1,7 @@ +version: 2 + +seeds: + - name: seeds_test + config: + database: analytics + schema: dbt_kfajardo \ No newline at end of file diff --git a/seeds/test_seed.csv b/seeds/test_seed.csv new file mode 100644 index 0000000..dbd888a --- /dev/null +++ b/seeds/test_seed.csv @@ -0,0 +1,2 @@ +col1,col2,col3 +1,2,3 \ No newline at end of file From 9d6fef6b9ba26065ceaaf041b585e54419f7daec Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Tue, 26 Nov 2024 02:41:33 +0000 Subject: [PATCH 11/15] Added singular test --- tests/assert_fct_daily_orders.sql | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/assert_fct_daily_orders.sql diff --git a/tests/assert_fct_daily_orders.sql b/tests/assert_fct_daily_orders.sql new file mode 100644 index 0000000..517e7cd --- /dev/null +++ b/tests/assert_fct_daily_orders.sql @@ -0,0 +1,21 @@ +with + + +source as ( + + select * from {{ ref('fct_daily_orders') }} + +), + +validations as ( + + select + order_sk, + count(*) as rec_cnt + from {{ ref('fct_daily_orders') }} + group by order_sk + having count(*) > 1 + +) + +select * from validations \ No newline at end of file From 08a594ffe0f4b7d4039c3cf56d5640b99613213b Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Tue, 26 Nov 2024 13:44:59 +0000 Subject: [PATCH 12/15] Commit changes. Added contracts. --- models/marts/dims/_dims__models.yml | 16 ++++++++++++++++ models/marts/dims/dim_orders.sql | 6 +++--- ...l => assert_duplicates__fct_daily_orders.sql} | 0 tests/assert_null__payment.sql | 12 ++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) rename tests/{assert_fct_daily_orders.sql => assert_duplicates__fct_daily_orders.sql} (100%) create mode 100644 tests/assert_null__payment.sql diff --git a/models/marts/dims/_dims__models.yml b/models/marts/dims/_dims__models.yml index 934e558..d2c69aa 100644 --- a/models/marts/dims/_dims__models.yml +++ b/models/marts/dims/_dims__models.yml @@ -2,11 +2,27 @@ version: 2 models: - name: dim_orders + config: + contract: + enforced: true columns: - name: order_sk + data_type: decimal(10,2) data_tests: - unique - not_null + - name: cust_fname + data_type: string + - name: cust_lname + data_type: string + - name: order_country_origin + data_type: string + - name: order_date + data_type: date + - name: order_id + data_type: string + - name: order_quantity + data_type: decimal(15,2) - name: dim_countries columns: diff --git a/models/marts/dims/dim_orders.sql b/models/marts/dims/dim_orders.sql index 1f0f147..6e72fbb 100644 --- a/models/marts/dims/dim_orders.sql +++ b/models/marts/dims/dim_orders.sql @@ -9,9 +9,9 @@ with orders as ( - select '1'::decimal(18,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname, 1 as order_quantity union all - select '2'::decimal(18,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname, 2 as order_quantity union all - select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname, 3 as order_quantity + select '1'::numeric(10,2) as order_sk, 'order-001' as order_id, '2024-01-01'::date as order_date, 'United States'::varchar(100) as order_country_origin,'John'::varchar(10) as cust_fname, 'Lennon'::varchar(10) as cust_lname, 1::decimal(17,3) as order_quantity union all + select '2'::numeric(10,2) as order_sk, 'order-002' as order_id, '2024-01-03'::date as order_date, 'France'::varchar(100) as order_country_origin,'Ringo'::varchar(10) as cust_fname, 'Starr'::varchar(10) as cust_lname, 2::decimal(17,2) as order_quantity union all + select '3'::numeric(10,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Paul'::varchar(10) as cust_fname, 'McCartney'::varchar(10) as cust_lname, 3::decimal(17,3) as order_quantity -- union all -- select NULL::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Johnny'::varchar(10) as cust_fname, 'English'::varchar(10) as cust_lname, 3 as order_quantity union all -- select '3'::decimal(18,2) as order_sk, 'order-003' as order_id, '2024-01-03'::date as order_date, 'Philippines'::varchar(100) as order_country_origin,'Mister'::varchar(10) as cust_fname, 'Bean'::varchar(10) as cust_lname, 3 as order_quantity diff --git a/tests/assert_fct_daily_orders.sql b/tests/assert_duplicates__fct_daily_orders.sql similarity index 100% rename from tests/assert_fct_daily_orders.sql rename to tests/assert_duplicates__fct_daily_orders.sql diff --git a/tests/assert_null__payment.sql b/tests/assert_null__payment.sql new file mode 100644 index 0000000..3627f80 --- /dev/null +++ b/tests/assert_null__payment.sql @@ -0,0 +1,12 @@ +with + +source as ( + + select * from {{ source('stripe', 'payment') }} + +) + +select +id +from source +where id is null \ No newline at end of file From 62a15a7a87ea76ed466305816eed2c4c73bd5a69 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Wed, 27 Nov 2024 23:41:15 +0000 Subject: [PATCH 13/15] Commit changes; unit test --- models/marts/dims/_dims__models.yml | 18 ------------------ ..._dims__unit_tests => _dims__unit_tests.yml} | 9 ++++++++- 2 files changed, 8 insertions(+), 19 deletions(-) rename models/marts/dims/{_dims__unit_tests => _dims__unit_tests.yml} (62%) diff --git a/models/marts/dims/_dims__models.yml b/models/marts/dims/_dims__models.yml index d2c69aa..e3c0a8c 100644 --- a/models/marts/dims/_dims__models.yml +++ b/models/marts/dims/_dims__models.yml @@ -34,21 +34,3 @@ models: - "France" - "Italy" - "Philippines" - -unit_tests: - - name: test_if_full_name_format_is_correct - description: "Checks if the format for customer full name is correct" - model: dim_customer_info - config: - tags: | - {%- if target.name == 'prod' -%} unit_test - {%- else -%} do_not_run - {% endif %} - given: - - input: ref('dim_orders') - rows: - - {"cust_fname": "Johnny", "cust_lname": "Bravo"} - - expect: - rows: - - {cust_fullname: "Bravo, Johnny"} \ No newline at end of file diff --git a/models/marts/dims/_dims__unit_tests b/models/marts/dims/_dims__unit_tests.yml similarity index 62% rename from models/marts/dims/_dims__unit_tests rename to models/marts/dims/_dims__unit_tests.yml index 4ee8197..ab82c25 100644 --- a/models/marts/dims/_dims__unit_tests +++ b/models/marts/dims/_dims__unit_tests.yml @@ -1,7 +1,14 @@ +version: 2 + unit_tests: - name: test_if_full_name_format_is_correct description: "Checks if the format for customer full name is correct" model: dim_customer_info + config: + tags: | + {%- if target.name == 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} given: - input: ref('dim_orders') rows: @@ -9,4 +16,4 @@ unit_tests: expect: rows: - - {cust_fullname: "Bravo, Johnny"} + - {cust_fullname: "Bravo, Johnny"} \ No newline at end of file From 5485517953cc5e68f512d959a70896b7cc1d651a Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 28 Nov 2024 00:54:29 +0000 Subject: [PATCH 14/15] Commit changes for unit test --- dbt_project.yml | 8 ++- models/intermediate/cfo/int_revenue_fct.sql | 67 +++++++++++++++++++ ...dims__models.yml => _dims__data_tests.yml} | 0 models/marts/dims/_dims__unit_tests.yml | 62 ++++++++++++++++- .../dims/cfo/dim_depreciation_accounts.sql | 16 +++++ .../marts/dims/cfo/dim_interests_accounts.sql | 16 +++++ models/marts/dims/cfo/dim_net_income.sql | 16 +++++ models/marts/dims/cfo/dim_tax_accounts.sql | 15 +++++ .../dims/{ => my_orders}/dim_countries.sql | 0 .../{ => my_orders}/dim_customer_info.sql | 0 .../marts/dims/{ => my_orders}/dim_dates.sql | 0 .../marts/dims/{ => my_orders}/dim_orders.sql | 0 ...cts__models.yml => _facts__data_tests.yml} | 0 models/marts/facts/cfo/fct_revenue.sql | 14 ++++ .../{ => my_orders}/fct_daily_orders.sql | 0 15 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 models/intermediate/cfo/int_revenue_fct.sql rename models/marts/dims/{_dims__models.yml => _dims__data_tests.yml} (100%) create mode 100644 models/marts/dims/cfo/dim_depreciation_accounts.sql create mode 100644 models/marts/dims/cfo/dim_interests_accounts.sql create mode 100644 models/marts/dims/cfo/dim_net_income.sql create mode 100644 models/marts/dims/cfo/dim_tax_accounts.sql rename models/marts/dims/{ => my_orders}/dim_countries.sql (100%) rename models/marts/dims/{ => my_orders}/dim_customer_info.sql (100%) rename models/marts/dims/{ => my_orders}/dim_dates.sql (100%) rename models/marts/dims/{ => my_orders}/dim_orders.sql (100%) rename models/marts/facts/{_facts__models.yml => _facts__data_tests.yml} (100%) create mode 100644 models/marts/facts/cfo/fct_revenue.sql rename models/marts/facts/{ => my_orders}/fct_daily_orders.sql (100%) diff --git a/dbt_project.yml b/dbt_project.yml index bc8f141..ebf91ff 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -31,8 +31,8 @@ vars: incremental_lookback_period: 'hour' future_proof_date: '9999-12-31' -on-run-end: - - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" +# on-run-end: +# - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" # Configuring models @@ -42,6 +42,10 @@ models: marts: +materialized: table +tags: ['tag'] + dims: + +tags: ["myorders"] + facts: + +tags: ["myorders"] finance: +group: finance marketing: diff --git a/models/intermediate/cfo/int_revenue_fct.sql b/models/intermediate/cfo/int_revenue_fct.sql new file mode 100644 index 0000000..22ebbec --- /dev/null +++ b/models/intermediate/cfo/int_revenue_fct.sql @@ -0,0 +1,67 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +depreciations as ( + + select * from {{ ref('dim_depreciation_accounts') }} + where account_type = 'Major' + +), + +interests as ( + + select * from {{ ref('dim_interests_accounts') }} + where account_type = 'Major' + +), + +net_income as ( + + select * from {{ ref('dim_net_income') }} + where account_type = 'Major' + +), + + +taxes as ( + + select * from {{ ref('dim_tax_accounts') }} + where account_type = 'Major' + +), + +join_sources as ( + + select + net_inc.account_id, + net_inc.net_income_usd, + deprec.depreciation_cost_usd, + ints.interests_amount_usd, + tax.tax_usd, + net_inc.account_type + from net_income net_inc + left join depreciations deprec + on net_inc.account_id = deprec.account_id + left join interests ints + on net_inc.account_id = ints.account_id + left join taxes tax + on net_inc.account_id = tax.account_id + +), + +final as ( + + select + account_id, + account_type, + (net_income_usd + interests_amount_usd + tax_usd + depreciation_cost_usd)::numeric(10, 3) as ebitda + from join_sources + +) + +select * from final + diff --git a/models/marts/dims/_dims__models.yml b/models/marts/dims/_dims__data_tests.yml similarity index 100% rename from models/marts/dims/_dims__models.yml rename to models/marts/dims/_dims__data_tests.yml diff --git a/models/marts/dims/_dims__unit_tests.yml b/models/marts/dims/_dims__unit_tests.yml index ab82c25..60b5477 100644 --- a/models/marts/dims/_dims__unit_tests.yml +++ b/models/marts/dims/_dims__unit_tests.yml @@ -16,4 +16,64 @@ unit_tests: expect: rows: - - {cust_fullname: "Bravo, Johnny"} \ No newline at end of file + - {cust_fullname: "Bravo, Johnny"} + + - name: test_accuracy_of_ebitda_calculation + description: "Test if the EBITDA calculation is correct" + model: int_revenue_fct + config: + tags: | + {%- if target.name != 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + + given: + - input: ref('dim_depreciation_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "depreciation_cost_usd": "-10"} + + - input: ref('dim_interests_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "interests_amount_usd": "10"} + + - input: ref('dim_net_income') + rows: + - {"account_id": "1", "account_type": "Major", "net_income_usd": "10"} + + - input: ref('dim_tax_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "tax_usd": "-10"} + + expect: + rows: + - {"account_id": "1", "account_type": "Major", "ebitda": "0"} + + - name: test_accuracy_of_ebitda_calculation__sql_format + description: "Test if the EBITDA calculation is correct using SQL format" + model: int_revenue_fct + config: + tags: | + {%- if target.name != 'prod' -%} unit_test + {%- else -%} do_not_run + {% endif %} + + given: + - input: ref('dim_depreciation_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "depreciation_cost_usd": "-10"} + + - input: ref('dim_interests_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "interests_amount_usd": "10"} + + - input: ref('dim_net_income') + rows: + - {"account_id": "1", "account_type": "Major", "net_income_usd": "10"} + + - input: ref('dim_tax_accounts') + rows: + - {"account_id": "1", "account_type": "Major", "tax_usd": "-10"} + + expect: + rows: + - {"account_id": "1", "account_type": "Major", "ebitda": "0"} \ No newline at end of file diff --git a/models/marts/dims/cfo/dim_depreciation_accounts.sql b/models/marts/dims/cfo/dim_depreciation_accounts.sql new file mode 100644 index 0000000..80bf71c --- /dev/null +++ b/models/marts/dims/cfo/dim_depreciation_accounts.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '-1000.00'::numeric(10, 3) depreciation_cost_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '-1000.00'::numeric(10, 3) depreciation_cost_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_interests_accounts.sql b/models/marts/dims/cfo/dim_interests_accounts.sql new file mode 100644 index 0000000..6fadd7f --- /dev/null +++ b/models/marts/dims/cfo/dim_interests_accounts.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) interests_amount_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) interests_amount_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_net_income.sql b/models/marts/dims/cfo/dim_net_income.sql new file mode 100644 index 0000000..b434f50 --- /dev/null +++ b/models/marts/dims/cfo/dim_net_income.sql @@ -0,0 +1,16 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) net_income_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '10000.00'::numeric(10, 3) net_income_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/cfo/dim_tax_accounts.sql b/models/marts/dims/cfo/dim_tax_accounts.sql new file mode 100644 index 0000000..92087d9 --- /dev/null +++ b/models/marts/dims/cfo/dim_tax_accounts.sql @@ -0,0 +1,15 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +source as ( + + select '1'::numeric(10, 3) as account_id, '-2000.00'::numeric(10, 3) tax_usd,'Major'::varchar as account_type, '2024-01-01'::date as record_updated_ts union all + select '2'::numeric(10, 3) as account_id, '-2000.00'::numeric(10, 3) tax_usd,'Minor'::varchar as account_type, '2024-01-01'::date as record_updated_ts + +) + +select * from source diff --git a/models/marts/dims/dim_countries.sql b/models/marts/dims/my_orders/dim_countries.sql similarity index 100% rename from models/marts/dims/dim_countries.sql rename to models/marts/dims/my_orders/dim_countries.sql diff --git a/models/marts/dims/dim_customer_info.sql b/models/marts/dims/my_orders/dim_customer_info.sql similarity index 100% rename from models/marts/dims/dim_customer_info.sql rename to models/marts/dims/my_orders/dim_customer_info.sql diff --git a/models/marts/dims/dim_dates.sql b/models/marts/dims/my_orders/dim_dates.sql similarity index 100% rename from models/marts/dims/dim_dates.sql rename to models/marts/dims/my_orders/dim_dates.sql diff --git a/models/marts/dims/dim_orders.sql b/models/marts/dims/my_orders/dim_orders.sql similarity index 100% rename from models/marts/dims/dim_orders.sql rename to models/marts/dims/my_orders/dim_orders.sql diff --git a/models/marts/facts/_facts__models.yml b/models/marts/facts/_facts__data_tests.yml similarity index 100% rename from models/marts/facts/_facts__models.yml rename to models/marts/facts/_facts__data_tests.yml diff --git a/models/marts/facts/cfo/fct_revenue.sql b/models/marts/facts/cfo/fct_revenue.sql new file mode 100644 index 0000000..3e7fe56 --- /dev/null +++ b/models/marts/facts/cfo/fct_revenue.sql @@ -0,0 +1,14 @@ +{{ config( + materialized='table', + tags=["cfo"] +) }} + +with + +sources as ( + + select * from {{ ref('int_revenue_fct') }} + +) + +select * from sources diff --git a/models/marts/facts/fct_daily_orders.sql b/models/marts/facts/my_orders/fct_daily_orders.sql similarity index 100% rename from models/marts/facts/fct_daily_orders.sql rename to models/marts/facts/my_orders/fct_daily_orders.sql From 2cbb34b6db3e80f0d1f62a7cbdead9fe694c1cd5 Mon Sep 17 00:00:00 2001 From: Keith Fajardo Date: Thu, 28 Nov 2024 01:10:02 +0000 Subject: [PATCH 15/15] commit changes for unit test --- models/marts/dims/_dims__unit_tests.yml | 60 ++++++++++++++++++++----- 1 file changed, 50 insertions(+), 10 deletions(-) diff --git a/models/marts/dims/_dims__unit_tests.yml b/models/marts/dims/_dims__unit_tests.yml index 60b5477..92b6551 100644 --- a/models/marts/dims/_dims__unit_tests.yml +++ b/models/marts/dims/_dims__unit_tests.yml @@ -59,21 +59,61 @@ unit_tests: given: - input: ref('dim_depreciation_accounts') - rows: - - {"account_id": "1", "account_type": "Major", "depreciation_cost_usd": "-10"} + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '-10'::numeric(10, 3) as depreciation_cost_usd + ) + + select * from main - input: ref('dim_interests_accounts') - rows: - - {"account_id": "1", "account_type": "Major", "interests_amount_usd": "10"} + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '10'::numeric(10, 3) as interests_amount_usd + ) + + select * from main - input: ref('dim_net_income') - rows: - - {"account_id": "1", "account_type": "Major", "net_income_usd": "10"} + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '10'::numeric(10, 3) as net_income_usd + ) + + select * from main - input: ref('dim_tax_accounts') - rows: - - {"account_id": "1", "account_type": "Major", "tax_usd": "-10"} + format: sql + rows: | + with + main as ( + select + '1'::numeric(10, 3) as account_id, + 'Major'::varchar as account_type, + '-10'::numeric(10, 3) as tax_usd + ) + + select * from main expect: - rows: - - {"account_id": "1", "account_type": "Major", "ebitda": "0"} \ No newline at end of file + format: sql + rows: | + select + 1::numeric(10, 3) as account_id, + 'Major' as account_type, + 0::numeric(10, 3) as ebitda \ No newline at end of file