From 673b5d12cd04b0b9b4f99af87d13b8e022aa87fd Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:48:24 +0000 Subject: [PATCH 1/7] refactoring snapshot_stg_payments and snapshot_stg_payments to match new snapshot strategy --- snapshots/jaffle_snapshots.yml | 24 ++++++++++++++++++++++++ snapshots/snapshot_int_line_items.sql | 22 ---------------------- snapshots/snapshot_stg_payments.sql | 16 ---------------- 3 files changed, 24 insertions(+), 38 deletions(-) create mode 100644 snapshots/jaffle_snapshots.yml delete mode 100644 snapshots/snapshot_int_line_items.sql delete mode 100644 snapshots/snapshot_stg_payments.sql diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml new file mode 100644 index 0000000..707111d --- /dev/null +++ b/snapshots/jaffle_snapshots.yml @@ -0,0 +1,24 @@ +version: 2 + +snapshots: + - name: snapshot_stg_payments + relation: source('stripe', 'payment') + config: + enabled: true + strategy: timestamp + updated_at: _BATCHED_AT + unique_key: ID + persist_docs: + relation: true + columns: true + - name: snapshot_int_line_items + relation: ref('int_line_items_amounts_calculated') + config: + enabled: true + strategy: check + check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] + unique_key: order_item_id + persist_docs: + relation: true + columns: true + diff --git a/snapshots/snapshot_int_line_items.sql b/snapshots/snapshot_int_line_items.sql deleted file mode 100644 index 90bc550..0000000 --- a/snapshots/snapshot_int_line_items.sql +++ /dev/null @@ -1,22 +0,0 @@ -{% snapshot snapshot_int_line_items %} - --- Example showing check columns strategy and using custom schemas (this is rare) - --- if you don't want to write to the same schema in development and prod, --- use environment-aware variable from dbt_project.yml - --- if you do not have a reliable updated at timestamp, checek columns for changes. - -{{ - config( - target_database='analytics', - target_schema=var('example_target_snapshot_schema'), - unique_key='order_item_id', - strategy='check', - check_cols=['gross_item_sales_amount', 'net_item_sales_amount'], - ) -}} - -select * from {{ ref('int_line_items_amounts_calculated') }} - -{% endsnapshot %} \ No newline at end of file diff --git a/snapshots/snapshot_stg_payments.sql b/snapshots/snapshot_stg_payments.sql deleted file mode 100644 index adc61b2..0000000 --- a/snapshots/snapshot_stg_payments.sql +++ /dev/null @@ -1,16 +0,0 @@ -{% snapshot snapshot_stg_payments %} - -{{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='ID', - - strategy='timestamp', - updated_at='_BATCHED_AT', - ) -}} - -select * from {{ source('stripe', 'payment') }} - -{% endsnapshot %} \ No newline at end of file From 910cccc4e6de35d25db9cf91797270ad64ad244a Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:49:41 +0000 Subject: [PATCH 2/7] deleting generate schema snapshot--i don't think this is relevant anymore given that snapshots now build to your dev schema then whatever your target is in prod. --- .../_samples/example_generate_schema_snapshot.sql | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 snapshots/_samples/example_generate_schema_snapshot.sql diff --git a/snapshots/_samples/example_generate_schema_snapshot.sql b/snapshots/_samples/example_generate_schema_snapshot.sql deleted file mode 100644 index 2174d7b..0000000 --- a/snapshots/_samples/example_generate_schema_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_generate_schema_snapshot %} - {{ - config( - target_database='analytics', - target_schema=generate_schema_name('snapshots'), - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file From c11e92d4214ce783cee4d686f01d35efb8334553 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:55:03 +0000 Subject: [PATCH 3/7] deleting old orders_line_items snapshot, replacing it with new config. also commenting out example join snapshot. --- .../_samples/snapshot/example_join_snapshots.sql | 3 ++- .../example_orders_line_items_snapshot.sql | 14 -------------- snapshots/jaffle_snapshots.yml | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 15 deletions(-) delete mode 100644 snapshots/_samples/example_orders_line_items_snapshot.sql diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.sql index 201d1c7..9ea20d4 100644 --- a/models/_samples/snapshot/example_join_snapshots.sql +++ b/models/_samples/snapshot/example_join_snapshots.sql @@ -1,4 +1,4 @@ -with order_snapshot as ( +{# with order_snapshot as ( select * exclude dbt_valid_to, coalesce(dbt_valid_to, cast('{{ var("future_proof_date") }}' as timestamp)) as dbt_valid_to @@ -47,3 +47,4 @@ final as ( ) select * from final +#} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_line_items_snapshot.sql b/snapshots/_samples/example_orders_line_items_snapshot.sql deleted file mode 100644 index 63603f5..0000000 --- a/snapshots/_samples/example_orders_line_items_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_line_items_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_line_items_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index 707111d..eebfaba 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -1,6 +1,7 @@ version: 2 snapshots: +# This is a pretty standard snapshot - name: snapshot_stg_payments relation: source('stripe', 'payment') config: @@ -11,6 +12,7 @@ snapshots: persist_docs: relation: true columns: true +# This snapshot has the check_cols example - name: snapshot_int_line_items relation: ref('int_line_items_amounts_calculated') config: @@ -21,4 +23,16 @@ snapshots: persist_docs: relation: true columns: true +# This snapshot has invalidate hard deletes as an example + - name: snapshot_orders_line_items + relation: ref('example_orders_line_items_source_for_snapshot') + config: + enabled: true + strategy: timestamp + unique_key: id + updated_at: order_updated_at + invalidate_hard_deletes: true + persist_docs: + relation: true + columns: true From 080bb9468f9756b2c7b8cf84e4053db58911a8ae Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 18:58:33 +0000 Subject: [PATCH 4/7] refactoring example_orders_snapshot into new way --- snapshots/_samples/example_orders_snapshot.sql | 14 -------------- snapshots/jaffle_snapshots.yml | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 15 deletions(-) delete mode 100644 snapshots/_samples/example_orders_snapshot.sql diff --git a/snapshots/_samples/example_orders_snapshot.sql b/snapshots/_samples/example_orders_snapshot.sql deleted file mode 100644 index e4e7ad6..0000000 --- a/snapshots/_samples/example_orders_snapshot.sql +++ /dev/null @@ -1,14 +0,0 @@ -{% snapshot example_orders_snapshot %} - {{ - config( - target_database='analytics', - target_schema='snapshots', - unique_key='order_id', - strategy='timestamp', - updated_at='order_updated_at', - invalidate_hard_deletes=True - ) - }} - - select * from {{ ref('example_orders_source_for_snapshot') }} - {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index eebfaba..afa84f2 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -12,6 +12,19 @@ snapshots: persist_docs: relation: true columns: true +# Another standard snapshot but this time with a different table. + - name: snapshot_example_orders + relation: ref('example_orders_source_for_snapshot') + config: + enabled: true + strategy: timestamp + updated_at: order_updated_at + unique_key: ID + persist_docs: + relation: true + columns: true + invalidate_hard_deletes: true + # This snapshot has the check_cols example - name: snapshot_int_line_items relation: ref('int_line_items_amounts_calculated') @@ -19,7 +32,7 @@ snapshots: enabled: true strategy: check check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] - unique_key: order_item_id + unique_key: order_id persist_docs: relation: true columns: true From 2c8532a0cd1f96a045b2df26035aaed1346885b3 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:02:37 +0000 Subject: [PATCH 5/7] convert example join snapshot to markdown till better fix avail --- .../{example_join_snapshots.sql => example_join_snapshots.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename models/_samples/snapshot/{example_join_snapshots.sql => example_join_snapshots.md} (100%) diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.md similarity index 100% rename from models/_samples/snapshot/example_join_snapshots.sql rename to models/_samples/snapshot/example_join_snapshots.md From ef2f5474aa2aa97d03bab1f439670e7cec68c571 Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:04:18 +0000 Subject: [PATCH 6/7] fixing order snapshot --- snapshots/jaffle_snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index afa84f2..c7bd414 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -19,7 +19,7 @@ snapshots: enabled: true strategy: timestamp updated_at: order_updated_at - unique_key: ID + unique_key: order_id persist_docs: relation: true columns: true From 833d85cdf5cae3607cd121300de36108f2c7cdec Mon Sep 17 00:00:00 2001 From: Faith Lierheimer Date: Fri, 8 Nov 2024 20:11:36 +0000 Subject: [PATCH 7/7] fixing snapshot line items --- snapshots/jaffle_snapshots.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snapshots/jaffle_snapshots.yml b/snapshots/jaffle_snapshots.yml index c7bd414..98b9695 100644 --- a/snapshots/jaffle_snapshots.yml +++ b/snapshots/jaffle_snapshots.yml @@ -32,7 +32,7 @@ snapshots: enabled: true strategy: check check_cols: ['gross_item_sales_amount', 'net_item_sales_amount'] - unique_key: order_id + unique_key: order_item_id persist_docs: relation: true columns: true