Bugfix/redshift constant expressions

.buildkite/scripts/run_models.sh

@@ -21,15 +21,15 @@ if [ "$db" = "databricks-sql" ]; then
 dbt seed --vars '{shopify_schema: shopify_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt run --vars '{shopify_schema: shopify_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt test --vars '{shopify_schema: shopify_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{shopify_schema: shopify_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01'}' --target "$db" --full-refresh
+dbt run --vars '{shopify_schema: shopify_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01', shopify_using_abandoned_checkout: false}' --target "$db" --full-refresh
 dbt test --vars '{shopify_schema: shopify_integrations_tests_sqlw}' --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 
 else
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01'}' --target "$db" --full-refresh
+dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_all_metafields: true, shopify__calendar_start_date: '2020-01-01', shopify_using_abandoned_checkout: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 fi

.quickstart/quickstart.yml

@@ -6,6 +6,10 @@ dbt_versions: ">=1.3.0 <2.0.0"
 table_variables:
   shopify_using_fulfillment_event:
     - fulfillment_event
+  shopify_using_abandoned_checkout:
+    - abandoned_checkout
+    - abandoned_checkout_discount_code
+    - abandoned_checkout_shipping_line
 
 destination_configurations:
   databricks:

CHANGELOG.md

@@ -1,3 +1,9 @@
+# dbt_shopify v0.14.0
+
+## Under the Hood
+- Adds enable config for the upstream `metadata` staging model (`stg_shopify__metafield`). For more information on how to enable/disable this table, refer to the [README](https://github.com/fivetran/dbt_shopify/blob/main/README.md#adding-metafields).
+- Adds disable config for the upstream `abandoned_checkout` staging models (including `stg_shopify__abandoned_checkout`, `stg_shopify__abandoned_checkout_discount_code`, and `stg_shopify__abandoned_checkout_shipping_line`). For more information on how to enable/disable these tables, refer to the [README](https://github.com/fivetran/dbt_shopify/blob/main/README.md#step-5-disable-models-for-non-existent-sources).
+
 # dbt_shopify v0.13.2
 [PR #89](https://github.com/fivetran/dbt_shopify/pull/89) includes the following changes:
 

README.md

@@ -120,7 +120,18 @@ vars:
     shopify_using_fulfillment_event: true # false by default
 ```
 
-### Step 5: Setting your timezone
+### Step 5: Disable models for non-existent sources
+This package considers that not every Shopify connector uses the `abandoned_checkout` tables (including `abandoned_checkout`, `abandoned_checkout_discount_code`, and `abandoned_checkout_shipping_line`). This package allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. To disable the `abandoned_checkout` tables, add the following variables:
+
+```yml
+# dbt_project.yml
+
+...
+vars:
+    shopify_using_abandoned_checkout: False  #True by default
+```
+
+### Step 6: Setting your timezone
 By default, the data in your Shopify schema is in UTC. However, you may want reporting to reflect a specific timezone for more realistic analysis or data validation.
 
 To convert the timezone of **all** timestamps in the package, update the `shopify_timezone` variable to your target zone in [IANA tz Database format](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones):
@@ -133,7 +144,7 @@ vars:
 
 > **Note**: This will only **numerically** convert timestamps to your target timezone. They will however have a "UTC" appended to them. This is a current limitation of the dbt-date `convert_timezone` [macro](https://github.com/calogica/dbt-date#convert_timezone-column-target_tznone-source_tznone) we leverage.
 
-### (Optional) Step 6: Additional configurations
+### (Optional) Step 7: Additional configurations
 <details open><summary>Expand/Collapse details</summary>
 
 #### Enabling Standardized Billing Model
@@ -143,7 +154,7 @@ This package contains the `shopify__line_item_enhanced` model which constructs a
 vars:
   shopify__standardized_billing_model_enabled: true # false by default.
 ```
-    
+
 #### Passing Through Additional Fields
 This package includes all source columns defined in the macros folder. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables:
 
@@ -172,7 +183,7 @@ vars:
 ```
 
 #### Adding Metafields
-In [May 2021](https://fivetran.com/docs/applications/shopify/changelog#may2021) the Shopify connector included support for the [metafield resource](https://shopify.dev/api/admin-rest/2023-01/resources/metafield). If you would like to take advantage of these metafields, this package offers corresponding mapping models which append these metafields to the respective source object for the following tables: collection, customer, order, product_image, product, product_variant, shop. If enabled, these models will materialize as `shopify__[object]_metafields` for each respective supported object. To enable these metafield mapping models, you may use the following configurations within your `dbt_project.yml`.
+In [May 2021](https://fivetran.com/docs/applications/shopify/changelog#may2021) the Shopify connector included support for the [metafield resource](https://shopify.dev/api/admin-rest/2023-01/resources/metafield). If you would like to take advantage of these metafields, this package offers corresponding mapping models which append these metafields to the respective source object for the following tables: collection, customer, order, product_image, product, product_variant, shop. Enabling any of the following variables will materialize the `stg_shopify__metafield` model, in addition to respective models that will materialize as `shopify__[object]_metafields` for each respective supported object. To enable these metafield mapping models, you may use the following configurations within your `dbt_project.yml`.
 >**Note**: These metafield models will contain all the same records as the corresponding staging models with the exception of the metafield columns being added.
 
 ```yml
@@ -235,7 +246,7 @@ vars:
 </details>
 
 
-### (Optional) Step 7: Orchestrate your models with Fivetran Transformations for dbt Core™
+### (Optional) Step 8: Orchestrate your models with Fivetran Transformations for dbt Core™
 <details><summary>Expand for details</summary>
 <br>
 
@@ -250,7 +261,7 @@ This dbt package is dependent on the following dbt packages. These dependencies
 ```yml
 packages:
     - package: fivetran/shopify_source
-      version: [">=0.12.0", "<0.13.0"]
+      version: [">=0.13.0", "<0.14.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify'
-version: '0.13.2'
+version: '0.14.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_integration_tests'
-version: '0.13.2'
+version: '0.14.0'
 profile: 'integration_tests'
 config-version: 2
 
@@ -10,11 +10,12 @@ clean-targets:         # directories to be removed by `dbt clean`
 
 vars:
   ## Uncomment when regenerating docs
-  # shopify_using_fulfillment_event: true
-  # shopify_using_all_metafields: true
-  # shopify__standardized_billing_model_enabled: true
-
+  shopify_using_fulfillment_event: true
+  shopify_using_all_metafields: true
+  shopify__standardized_billing_model_enabled: true
+  shopify_using_abandoned_checkout: true
   shopify_schema: shopify_integration_tests_11
+
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"
     shopify_order_line_refund_identifier: "shopify_order_line_refund_data"
@@ -56,7 +57,7 @@ dispatch:
     search_order: ['spark_utils', 'dbt_utils']
 
 models:
-  +schema: "{{ 'shopify_integrations_tests_sqlw' if target.name == 'databricks-sql' else 'shopify' }}"
+  +schema: "{{ 'shopify_integrations_tests_sqlw' if target.name == 'databricks-sql' else 'shopify_customers_dev' }}"
   # +schema: "shopify_{{ var('directed_schema','dev') }}"
 
 seeds:

models/intermediate/int_shopify__daily_abandoned_checkouts.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 with abandoned_checkout as (
 
     select *

models/intermediate/int_shopify__discounts__abandoned_checkouts.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 with abandoned_checkout as (
 
     select *

models/shopify__customer_emails.sql

@@ -10,6 +10,7 @@ with customer_emails as (
     from {{ ref('int_shopify__emails__order_aggregates' )}}
     where email is not null
 
+{% if var('shopify_using_abandoned_checkout', True) %}
 ), abandoned as (
 
     select 
@@ -19,12 +20,17 @@ with customer_emails as (
     from {{ var('shopify_abandoned_checkout' )}}
     where email is not null
     group by 1,2
+{% endif %}
 
 ), joined as (
 
     select 
         customer_emails.*,
+
+        {% if var('shopify_using_abandoned_checkout', True) %}
         coalesce(abandoned.lifetime_abandoned_checkouts, 0) as lifetime_abandoned_checkouts,
+        {% endif %}
+
         orders.first_order_timestamp,
         orders.most_recent_order_timestamp,
         orders.avg_order_value,
@@ -48,9 +54,12 @@ with customer_emails as (
     left join orders
         on customer_emails.email = orders.email
         and customer_emails.source_relation = orders.source_relation
+
+    {% if var('shopify_using_abandoned_checkout', True) %}
     left join abandoned
         on customer_emails.email = abandoned.email
         and customer_emails.source_relation = abandoned.source_relation
+    {% endif %}
 )
 
 select *

models/shopify__customers.sql

@@ -9,6 +9,7 @@ with customers as (
     select *
     from {{ ref('shopify__customers__order_aggregates' )}}
 
+{% if var('shopify_using_abandoned_checkout', True) %}
 ), abandoned as (
 
     select 
@@ -18,6 +19,7 @@ with customers as (
     from {{ var('shopify_abandoned_checkout' )}}
     where customer_id is not null
     group by 1,2
+{% endif %}
 
 ), customer_tags_aggregated as (
 
@@ -33,7 +35,11 @@ with customers as (
 
     select 
         customers.*,
+
+        {% if var('shopify_using_abandoned_checkout', True) %}
         coalesce(abandoned.lifetime_abandoned_checkouts, 0) as lifetime_abandoned_checkouts,
+        {% endif %}
+
         orders.first_order_timestamp,
         orders.most_recent_order_timestamp,
         customer_tags_aggregated.customer_tags,
@@ -58,12 +64,16 @@ with customers as (
     left join orders
         on customers.customer_id = orders.customer_id
         and customers.source_relation = orders.source_relation
-    left join abandoned
-        on customers.customer_id = abandoned.customer_id
-        and customers.source_relation = abandoned.source_relation
     left join customer_tags_aggregated
         on customers.customer_id = customer_tags_aggregated.customer_id
         and customers.source_relation = customer_tags_aggregated.source_relation
+
+    {% if var('shopify_using_abandoned_checkout', True) %}
+    left join abandoned
+        on customers.customer_id = abandoned.customer_id
+        and customers.source_relation = abandoned.source_relation
+    {% endif %}
+
 )
 
 select *

models/shopify__daily_shop.sql

@@ -17,11 +17,13 @@ daily_orders as (
     from {{ ref('int_shopify__daily_orders') }}
 ),
 
+{% if var('shopify_using_abandoned_checkout', True) %}
 daily_abandoned_checkouts as (
 
     select *
     from {{ ref('int_shopify__daily_abandoned_checkouts') }}
 ),
+{% endif %}
 
 {% if var('shopify_using_fulfillment_event', false) %}
 daily_fulfillment as (
@@ -81,7 +83,6 @@ final as (
         coalesce(daily_orders.count_orders_with_refunds, 0) as count_orders_with_refunds,
         daily_orders.first_order_timestamp,
         daily_orders.last_order_timestamp,
-
         coalesce(daily_orders.quantity_sold, 0) as quantity_sold,
         coalesce(daily_orders.quantity_refunded, 0) as quantity_refunded,
         coalesce(daily_orders.quantity_net, 0) as quantity_net,
@@ -90,11 +91,13 @@ final as (
         coalesce(daily_orders.count_variants_sold, 0) as count_variants_sold,
         coalesce(daily_orders.count_products_sold, 0) as count_products_sold,
         coalesce(daily_orders.quantity_gift_cards_sold, 0) as quantity_gift_cards_sold,
-        coalesce(daily_orders.quantity_requiring_shipping, 0) as quantity_requiring_shipping,
+        coalesce(daily_orders.quantity_requiring_shipping, 0) as quantity_requiring_shipping
 
-        coalesce(daily_abandoned_checkouts.count_abandoned_checkouts, 0) as count_abandoned_checkouts,
+        {% if var('shopify_using_abandoned_checkout', True) %}
+        , coalesce(daily_abandoned_checkouts.count_abandoned_checkouts, 0) as count_abandoned_checkouts,
         coalesce(daily_abandoned_checkouts.count_customers_abandoned_checkout, 0) as count_customers_abandoned_checkout,
         coalesce(daily_abandoned_checkouts.count_customer_emails_abandoned_checkout, 0) as count_customer_emails_abandoned_checkout
+        {% endif %}
 
         {% if var('shopify_using_fulfillment_event', false) %}
             {% for status in ['attempted_delivery', 'delayed', 'delivered', 'failure', 'in_transit', 'out_for_delivery', 'ready_for_pickup', 'picked_up', 'label_printed', 'label_purchased', 'confirmed']%}
@@ -106,9 +109,13 @@ final as (
     left join daily_orders 
         on shop_calendar.source_relation = daily_orders.source_relation
         and shop_calendar.date_day = daily_orders.date_day
+
+    {% if var('shopify_using_abandoned_checkout', True) %}
     left join daily_abandoned_checkouts 
         on shop_calendar.source_relation = daily_abandoned_checkouts.source_relation
         and shop_calendar.date_day = daily_abandoned_checkouts.date_day
+    {% endif %}
+
     {% if var('shopify_using_fulfillment_event', false) %}
     left join daily_fulfillment 
         on shop_calendar.source_relation = daily_fulfillment.source_relation

models/shopify__discounts.sql

@@ -34,11 +34,13 @@ orders_aggregated as (
     from {{ ref('int_shopify__discounts__order_aggregates')}}
 ),
 
+{% if var('shopify_using_abandoned_checkout', True) %}
 abandoned_checkouts_aggregated as (
 
     select *
     from {{ ref('int_shopify__discounts__abandoned_checkouts')}}
 ),
+{% endif %}
 
 discount_price_rule_joined as (
 
@@ -77,18 +79,21 @@ aggregates_joined as (
     select 
         discount_price_rule_joined.*,
         coalesce(orders_aggregated.count_orders, 0) as count_orders,
-        coalesce(abandoned_checkouts_aggregated.count_abandoned_checkouts, 0) as count_abandoned_checkouts,
         orders_aggregated.avg_order_discount_amount,
         coalesce(orders_aggregated.total_order_discount_amount, 0) as total_order_discount_amount,
-        coalesce(abandoned_checkouts_aggregated.total_abandoned_checkout_discount_amount, 0) as total_abandoned_checkout_discount_amount,
         coalesce(orders_aggregated.total_order_line_items_price, 0) as total_order_line_items_price,
         coalesce(orders_aggregated.total_order_shipping_cost, 0) as total_order_shipping_cost,
-        coalesce(abandoned_checkouts_aggregated.total_abandoned_checkout_shipping_price, 0) as total_abandoned_checkout_shipping_price,
         coalesce(orders_aggregated.total_order_refund_amount, 0) as total_order_refund_amount,
         coalesce(orders_aggregated.count_customers, 0) as count_customers,
-        coalesce(orders_aggregated.count_customer_emails, 0) as count_customer_emails,
+        coalesce(orders_aggregated.count_customer_emails, 0) as count_customer_emails
+
+        {% if var('shopify_using_abandoned_checkout', True) %}
+        , coalesce(abandoned_checkouts_aggregated.total_abandoned_checkout_discount_amount, 0) as total_abandoned_checkout_discount_amount,
+        coalesce(abandoned_checkouts_aggregated.total_abandoned_checkout_shipping_price, 0) as total_abandoned_checkout_shipping_price,
+        coalesce(abandoned_checkouts_aggregated.count_abandoned_checkouts, 0) as count_abandoned_checkouts,
         coalesce(abandoned_checkouts_aggregated.count_abandoned_checkout_customers, 0) as count_abandoned_checkout_customers,
         coalesce(abandoned_checkouts_aggregated.count_abandoned_checkout_customer_emails, 0) as count_abandoned_checkout_customer_emails
+        {% endif %} 
 
     from discount_price_rule_joined
     left join orders_aggregated
@@ -98,14 +103,16 @@ aggregates_joined as (
         and (case 
                 when discount_price_rule_joined.target_type = 'shipping_line' then 'shipping' -- when target_type = 'shipping', value_type = 'percentage'
                 else discount_price_rule_joined.value_type end) = orders_aggregated.type
-
+
+    {% if var('shopify_using_abandoned_checkout', True) %}
     left join abandoned_checkouts_aggregated
         on discount_price_rule_joined.code = abandoned_checkouts_aggregated.code
         and discount_price_rule_joined.source_relation = abandoned_checkouts_aggregated.source_relation
         -- in case one CODE can apply to both shipping and line items, percentages and fixed_amounts
         and (case 
                 when discount_price_rule_joined.target_type = 'shipping_line' then 'shipping' -- when target_type = 'shipping', value_type = 'percentage'
-                else discount_price_rule_joined.value_type end) = abandoned_checkouts_aggregated.type 
+                else discount_price_rule_joined.value_type end) = abandoned_checkouts_aggregated.type
+    {% endif %}
 )
 
 select * 

packages.yml

@@ -1,3 +1,7 @@
 packages:
-- package: fivetran/shopify_source
-  version: [">=0.12.0", "<0.13.0"]
+# - package: fivetran/shopify_source
+#   version: [">=0.13.0", "<0.14.0"]
+
+  - git: https://github.com/fivetran/dbt_shopify_source.git
+    revision: bugfix/redshift_constant_expressions
+    warn-unpinned: false