From 63ecab280a941c3e3a7b0a3944fc692b451df1a1 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:40:49 -0800 Subject: [PATCH 01/16] running successfully --- CHANGELOG.md | 4 + README.md | 2 +- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- models/docs.md | 4 + models/stg_hubspot__company.sql | 19 +++- models/stg_hubspot__company.yml | 10 +- .../stg_hubspot__company_property_history.sql | 11 +- models/stg_hubspot__contact.sql | 19 +++- models/stg_hubspot__contact.yml | 24 +++- models/stg_hubspot__contact_list.sql | 11 +- models/stg_hubspot__contact_list_member.sql | 11 +- models/stg_hubspot__contact_merge_audit.sql | 11 +- .../stg_hubspot__contact_property_history.sql | 8 ++ models/stg_hubspot__deal.sql | 19 +++- models/stg_hubspot__deal.yml | 33 +++++- models/stg_hubspot__deal_company.sql | 11 +- models/stg_hubspot__deal_contact.sql | 11 +- models/stg_hubspot__deal_pipeline.sql | 12 +- models/stg_hubspot__deal_pipeline_stage.sql | 12 +- models/stg_hubspot__deal_property_history.sql | 12 +- models/stg_hubspot__deal_stage.sql | 11 +- models/stg_hubspot__email.yml | 107 +++++++++++++++--- models/stg_hubspot__email_campaign.sql | 12 +- models/stg_hubspot__email_event.sql | 12 +- models/stg_hubspot__email_event_bounce.sql | 12 +- models/stg_hubspot__email_event_click.sql | 12 +- models/stg_hubspot__email_event_deferred.sql | 12 +- models/stg_hubspot__email_event_delivered.sql | 12 +- models/stg_hubspot__email_event_dropped.sql | 12 +- models/stg_hubspot__email_event_forward.sql | 12 +- models/stg_hubspot__email_event_open.sql | 12 +- models/stg_hubspot__email_event_print.sql | 12 +- models/stg_hubspot__email_event_sent.sql | 12 +- .../stg_hubspot__email_event_spam_report.sql | 11 +- ...stg_hubspot__email_event_status_change.sql | 12 +- models/stg_hubspot__engagement.sql | 12 +- models/stg_hubspot__engagement.yml | 56 +++++++-- models/stg_hubspot__engagement_call.sql | 8 ++ models/stg_hubspot__engagement_company.sql | 8 ++ models/stg_hubspot__engagement_contact.sql | 8 ++ models/stg_hubspot__engagement_deal.sql | 8 ++ models/stg_hubspot__engagement_email.sql | 8 ++ models/stg_hubspot__engagement_meeting.sql | 8 ++ models/stg_hubspot__engagement_note.sql | 8 ++ models/stg_hubspot__engagement_task.sql | 8 ++ models/stg_hubspot__owner.sql | 12 +- models/stg_hubspot__owner.yml | 4 +- models/stg_hubspot__property.sql | 12 +- models/stg_hubspot__property.yml | 13 ++- models/stg_hubspot__property_option.sql | 12 +- models/stg_hubspot__ticket.sql | 19 +++- models/stg_hubspot__ticket.yml | 24 +++- models/stg_hubspot__ticket_company.sql | 8 ++ models/stg_hubspot__ticket_contact.sql | 8 ++ models/stg_hubspot__ticket_deal.sql | 8 ++ models/stg_hubspot__ticket_engagement.sql | 8 ++ models/stg_hubspot__ticket_pipeline.sql | 10 +- models/stg_hubspot__ticket_pipeline_stage.sql | 11 +- .../stg_hubspot__ticket_property_history.sql | 8 ++ ..._hubspot__company_property_history_tmp.sql | 14 ++- models/tmp/stg_hubspot__company_tmp.sql | 14 ++- .../stg_hubspot__contact_list_member_tmp.sql | 14 ++- models/tmp/stg_hubspot__contact_list_tmp.sql | 14 ++- .../stg_hubspot__contact_merge_audit_tmp.sql | 14 ++- ..._hubspot__contact_property_history_tmp.sql | 14 ++- models/tmp/stg_hubspot__contact_tmp.sql | 14 ++- models/tmp/stg_hubspot__deal_company_tmp.sql | 14 ++- models/tmp/stg_hubspot__deal_contact_tmp.sql | 14 ++- .../stg_hubspot__deal_pipeline_stage_tmp.sql | 14 ++- models/tmp/stg_hubspot__deal_pipeline_tmp.sql | 14 ++- ...stg_hubspot__deal_property_history_tmp.sql | 14 ++- models/tmp/stg_hubspot__deal_stage_tmp.sql | 14 ++- models/tmp/stg_hubspot__deal_tmp.sql | 14 ++- .../tmp/stg_hubspot__email_campaign_tmp.sql | 14 ++- .../stg_hubspot__email_event_bounce_tmp.sql | 14 ++- .../stg_hubspot__email_event_click_tmp.sql | 14 ++- .../stg_hubspot__email_event_deferred_tmp.sql | 14 ++- ...stg_hubspot__email_event_delivered_tmp.sql | 14 ++- .../stg_hubspot__email_event_dropped_tmp.sql | 14 ++- .../stg_hubspot__email_event_forward_tmp.sql | 14 ++- .../tmp/stg_hubspot__email_event_open_tmp.sql | 14 ++- .../stg_hubspot__email_event_print_tmp.sql | 14 ++- .../tmp/stg_hubspot__email_event_sent_tmp.sql | 14 ++- ...g_hubspot__email_event_spam_report_tmp.sql | 14 ++- ...hubspot__email_event_status_change_tmp.sql | 14 ++- models/tmp/stg_hubspot__email_event_tmp.sql | 14 ++- .../tmp/stg_hubspot__engagement_call_tmp.sql | 14 ++- .../stg_hubspot__engagement_company_tmp.sql | 14 ++- .../stg_hubspot__engagement_contact_tmp.sql | 14 ++- .../tmp/stg_hubspot__engagement_deal_tmp.sql | 14 ++- .../tmp/stg_hubspot__engagement_email_tmp.sql | 14 ++- .../stg_hubspot__engagement_meeting_tmp.sql | 14 ++- .../tmp/stg_hubspot__engagement_note_tmp.sql | 14 ++- .../tmp/stg_hubspot__engagement_task_tmp.sql | 14 ++- models/tmp/stg_hubspot__engagement_tmp.sql | 14 ++- models/tmp/stg_hubspot__owner_tmp.sql | 14 ++- .../tmp/stg_hubspot__property_option_tmp.sql | 14 ++- models/tmp/stg_hubspot__property_tmp.sql | 14 ++- .../tmp/stg_hubspot__ticket_company_tmp.sql | 14 ++- .../tmp/stg_hubspot__ticket_contact_tmp.sql | 14 ++- models/tmp/stg_hubspot__ticket_deal_tmp.sql | 14 ++- .../stg_hubspot__ticket_engagement_tmp.sql | 14 ++- ...stg_hubspot__ticket_pipeline_stage_tmp.sql | 14 ++- .../tmp/stg_hubspot__ticket_pipeline_tmp.sql | 14 ++- ...g_hubspot__ticket_property_history_tmp.sql | 14 ++- models/tmp/stg_hubspot__ticket_tmp.sql | 14 ++- packages.yml | 9 +- 108 files changed, 1308 insertions(+), 169 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 183c7ee1..0e5d1f61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# dbt_hubspot_source v0.15.0 + +Unioning coming soon... + # dbt_hubspot_source v0.14.0 [PR #122](https://github.com/fivetran/dbt_hubspot_source/pull/122) includes the following updates: diff --git a/README.md b/README.md index 157d5478..6fd70bbb 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Include the following hubspot_source package version in your `packages.yml` file ```yaml packages: - package: fivetran/hubspot_source - version: [">=0.14.0", "<0.15.0"] + version: [">=0.15.0", "<0.16.0"] ``` ## Step 3: Define database and schema variables By default, this package runs using your destination and the `hubspot` schema. If this is not where your HubSpot data is (for example, if your HubSpot schema is named `hubspot_fivetran`), add the following configuration to your root `dbt_project.yml` file: diff --git a/dbt_project.yml b/dbt_project.yml index 8d3f0771..be57dab5 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'hubspot_source' -version: '0.14.0' +version: '0.15.0' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f5f7a96e..e3833b20 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'hubspot_source_integration_tests' -version: '0.14.0' +version: '0.15.0' profile: 'integration_tests' config-version: 2 models: diff --git a/models/docs.md b/models/docs.md index 29ea86e4..720b668a 100644 --- a/models/docs.md +++ b/models/docs.md @@ -2,6 +2,10 @@ Timestamp of when Fivetran synced a record. {% enddocs %} +{% docs source_relation %} +The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. If you are running the package on a single connector, this will be its schema name. +{% enddocs %} + {% docs _fivetran_deleted %} Boolean indicating whether a record has been deleted in Hubspot and/or inferred deleted in Hubspot by Fivetran; _fivetran_deleted and is_deleted fields are equivalent. {% enddocs %} diff --git a/models/stg_hubspot__company.sql b/models/stg_hubspot__company.sql index 3304a8ea..bb541fd8 100644 --- a/models/stg_hubspot__company.sql +++ b/models/stg_hubspot__company.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_company_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -27,6 +35,14 @@ with base as ( staging_columns=get_company_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if all_passthrough_column_check('stg_hubspot__company_tmp',get_company_columns()) > 0 %} -- just pass everything through if extra columns are present, but ensure required columns are present. ,{{ @@ -52,7 +68,8 @@ with base as ( city, state, country, - company_annual_revenue + company_annual_revenue, + source_relation --The below macro adds the fields defined within your hubspot__ticket_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('hubspot__company_pass_through_columns') }} diff --git a/models/stg_hubspot__company.yml b/models/stg_hubspot__company.yml index faf6fa0c..8b72961d 100644 --- a/models/stg_hubspot__company.yml +++ b/models/stg_hubspot__company.yml @@ -19,9 +19,16 @@ models: description: '{{ doc("history_name") }}' - name: new_value description: '{{ doc("history_value") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__company description: Each record represents a company in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - company_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -30,7 +37,6 @@ models: - name: company_id description: The ID of the company. tests: - - unique - not_null - name: company_name description: The name of the company. @@ -52,3 +58,5 @@ models: description: The country where the company is located. - name: company_annual_revenue description: The actual or estimated annual revenue of the company. + - name: source_relation + description: '{{ doc("source_relation") }}' diff --git a/models/stg_hubspot__company_property_history.sql b/models/stg_hubspot__company_property_history.sql index 3b673245..45fdad9e 100644 --- a/models/stg_hubspot__company_property_history.sql +++ b/models/stg_hubspot__company_property_history.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_company_property_history_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -25,7 +33,8 @@ with base as ( source as change_source, source_id as change_source_id, cast(change_timestamp as {{ dbt.type_timestamp() }}) as change_timestamp, -- source field name = timestamp ; alias declared in macros/get_company_property_history_columns.sql - value as new_value + value as new_value, + source_relation from macro ) diff --git a/models/stg_hubspot__contact.sql b/models/stg_hubspot__contact.sql index 83c838ce..e882764d 100644 --- a/models/stg_hubspot__contact.sql +++ b/models/stg_hubspot__contact.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_contact_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -27,6 +35,14 @@ with base as ( staging_columns=get_contact_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if all_passthrough_column_check('stg_hubspot__contact_tmp',get_contact_columns()) > 0 %} -- just pass everything through if extra columns are present, but ensure required columns are present. ,{{ @@ -50,7 +66,8 @@ with base as ( cast(created_date as {{ dbt.type_timestamp() }}) as created_date, job_title, company_annual_revenue, - cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced + cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, + source_relation --The below macro adds the fields defined within your hubspot__contact_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('hubspot__contact_pass_through_columns') }} diff --git a/models/stg_hubspot__contact.yml b/models/stg_hubspot__contact.yml index 7e4f50d7..d294605f 100644 --- a/models/stg_hubspot__contact.yml +++ b/models/stg_hubspot__contact.yml @@ -14,9 +14,16 @@ models: description: The ID of the related contact. - name: contact_list_id description: The ID of the related contact list. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__contact_list description: Each record represents a contact list in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - contact_list_id + - source_relation columns: - name: is_contact_list_deleted description: '{{ doc("is_deleted") }}' @@ -25,7 +32,6 @@ models: - name: contact_list_id description: The ID of the contact list. tests: - - unique - not_null - name: contact_list_name description: The name of the contact list. @@ -51,6 +57,8 @@ models: description: '{{ doc("portal_id") }}' - name: updated_timestamp description: A timestamp of the time that the list was last modified. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__contact_property_history description: Each record represents a change to contact record in Hubspot. @@ -69,6 +77,8 @@ models: description: '{{ doc("history_name") }}' - name: new_value description: '{{ doc("history_value") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__contact_merge_audit description: Each record contains a contact merge event and the contacts effected by the merge. @@ -93,9 +103,16 @@ models: description: The ID of the user. - name: vid_to_merge description: The ID of the contact which was merged. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__contact description: Each record represents a contact in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - contact_id + - source_relation columns: - name: is_contact_deleted description: '{{ doc("is_deleted") }}' @@ -104,7 +121,6 @@ models: - name: contact_id description: The ID of the contact. tests: - - unique - not_null - name: email description: The email address of the contact. @@ -126,4 +142,6 @@ models: description: > List of mappings representing contact IDs that have been merged into the contact at hand. Format: :;: - This field has replaced the `CONTACT_MERGE_AUDIT` table, which was deprecated by the Hubspot v3 CRM API. \ No newline at end of file + This field has replaced the `CONTACT_MERGE_AUDIT` table, which was deprecated by the Hubspot v3 CRM API. + - name: source_relation + description: '{{ doc("source_relation") }}' \ No newline at end of file diff --git a/models/stg_hubspot__contact_list.sql b/models/stg_hubspot__contact_list.sql index 1cbcd5a6..c4f4fcfa 100644 --- a/models/stg_hubspot__contact_list.sql +++ b/models/stg_hubspot__contact_list.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_contact_list_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -32,7 +40,8 @@ with base as ( metadata_size, name as contact_list_name, portal_id, - cast(updated_at as {{ dbt.type_timestamp() }}) as updated_timestamp + cast(updated_at as {{ dbt.type_timestamp() }}) as updated_timestamp, + source_relation from macro ) diff --git a/models/stg_hubspot__contact_list_member.sql b/models/stg_hubspot__contact_list_member.sql index 6289ed36..c487a110 100644 --- a/models/stg_hubspot__contact_list_member.sql +++ b/models/stg_hubspot__contact_list_member.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_contact_list_member_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -23,7 +31,8 @@ with base as ( cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, cast(added_at as {{ dbt.type_timestamp() }}) as added_timestamp, contact_id, - contact_list_id + contact_list_id, + source_relation from macro ) diff --git a/models/stg_hubspot__contact_merge_audit.sql b/models/stg_hubspot__contact_merge_audit.sql index 62a95a55..3dcf0fad 100644 --- a/models/stg_hubspot__contact_merge_audit.sql +++ b/models/stg_hubspot__contact_merge_audit.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_contact_merge_audit_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -33,7 +41,8 @@ with base as ( as {{ dbt.type_timestamp() }}) as timestamp_at, user_id, vid_to_merge, - cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced + cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, + source_relation from macro ) diff --git a/models/stg_hubspot__contact_property_history.sql b/models/stg_hubspot__contact_property_history.sql index b8551e7a..b8ea5294 100644 --- a/models/stg_hubspot__contact_property_history.sql +++ b/models/stg_hubspot__contact_property_history.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_contact_property_history_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( diff --git a/models/stg_hubspot__deal.sql b/models/stg_hubspot__deal.sql index 3899416e..f9742ff8 100644 --- a/models/stg_hubspot__deal.sql +++ b/models/stg_hubspot__deal.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_deal_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -27,6 +35,14 @@ with base as ( staging_columns=get_deal_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if all_passthrough_column_check('stg_hubspot__deal_tmp',get_deal_columns()) > 0 %} -- just pass everything through if extra columns are present, but ensure required columns are present. ,{{ @@ -51,7 +67,8 @@ with base as ( owner_id, portal_id, description, - amount + amount, + source_relation --The below macro adds the fields defined within your hubspot__deal_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('hubspot__deal_pass_through_columns') }} diff --git a/models/stg_hubspot__deal.yml b/models/stg_hubspot__deal.yml index eec005eb..9d8203ee 100644 --- a/models/stg_hubspot__deal.yml +++ b/models/stg_hubspot__deal.yml @@ -4,6 +4,11 @@ models: - name: stg_hubspot__deal_pipeline_stage description: Each record represents a pipeline stage in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - deal_pipeline_stage_id + - source_relation columns: - name: is_deal_pipeline_stage_deleted description: '{{ doc("is_deleted") }}' @@ -14,7 +19,6 @@ models: - name: deal_pipeline_stage_id description: The ID of the pipeline stage. tests: - - unique - not_null - name: display_order description: Used to determine the order in which the stages appear when viewed in HubSpot. @@ -30,9 +34,16 @@ models: description: A timestamp representing when the record was created. - name: deal_pipeline_stage_updated_at description: A timestamp representing when the record was updated. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__deal_pipeline description: Each record represents a pipeline in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - deal_pipeline_id + - source_relation columns: - name: is_deal_pipeline_deleted description: '{{ doc("is_deleted") }}' @@ -41,7 +52,6 @@ models: - name: deal_pipeline_id description: The ID of the pipeline. tests: - - unique - not_null - name: display_order description: Used to determine the order in which the pipelines appear when viewed in HubSpot @@ -49,14 +59,20 @@ models: description: A timestamp representing when the record was created. - name: deal_pipeline_updated_at description: A timestamp representing when the record was updated. - - name: is_active description: Whether the stage is currently in use. - name: pipeline_label description: The human-readable label for the pipeline. The label is used when showing the pipeline in HubSpot. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__deal description: Each record represents a deal in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - deal_id + - source_relation columns: - name: is_deal_deleted description: '{{ doc("is_deleted") }}' @@ -66,7 +82,6 @@ models: description: The ID of the deal tests: - not_null - - unique - name: portal_id description: '{{ doc("portal_id") }}' - name: deal_pipeline_id @@ -85,6 +100,8 @@ models: description: The day the deal is expected to close, or was closed. - name: created_date description: The date the deal was created. This property is set automatically by HubSpot. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__deal_stage description: Each record represents a deal stage. @@ -107,6 +124,8 @@ models: description: Reference to the source. - name: deal_stage_name description: The value of the deal stage. Typically the name of the stage. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__deal_company description: Each record represents a 'link' between a deal and company. @@ -116,6 +135,7 @@ models: - company_id - deal_id - type_id + - source_relation columns: - name: company_id description: The ID of the company. @@ -134,6 +154,7 @@ models: - contact_id - deal_id - type_id + - source_relation columns: - name: contact_id description: The ID of the contact. @@ -143,6 +164,8 @@ models: description: The ID of the type. - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__deal_property_history description: Each record represents the details of your deal properties. @@ -161,3 +184,5 @@ models: description: '{{ doc("history_name") }}' - name: new_value description: '{{ doc("history_value") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' \ No newline at end of file diff --git a/models/stg_hubspot__deal_company.sql b/models/stg_hubspot__deal_company.sql index 09f2e28c..9374a0f4 100644 --- a/models/stg_hubspot__deal_company.sql +++ b/models/stg_hubspot__deal_company.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_deal_company_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -22,7 +30,8 @@ with base as ( company_id, deal_id, type_id, - cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced + cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, + source_relation from macro diff --git a/models/stg_hubspot__deal_contact.sql b/models/stg_hubspot__deal_contact.sql index 8cfc0692..78a1e115 100644 --- a/models/stg_hubspot__deal_contact.sql +++ b/models/stg_hubspot__deal_contact.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_deal_contact_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -22,7 +30,8 @@ with base as ( contact_id, deal_id, type_id, - cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced + cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, + source_relation from macro diff --git a/models/stg_hubspot__deal_pipeline.sql b/models/stg_hubspot__deal_pipeline.sql index c0bfd19c..374ef740 100644 --- a/models/stg_hubspot__deal_pipeline.sql +++ b/models/stg_hubspot__deal_pipeline.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_deal_pipeline_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -26,7 +34,9 @@ with base as ( label as pipeline_label, cast(pipeline_id as {{ dbt.type_string() }}) as deal_pipeline_id, created_at as deal_pipeline_created_at, - updated_at as deal_pipeline_updated_at + updated_at as deal_pipeline_updated_at, + source_relation + from macro ) diff --git a/models/stg_hubspot__deal_pipeline_stage.sql b/models/stg_hubspot__deal_pipeline_stage.sql index cbd40767..ffb9c1df 100644 --- a/models/stg_hubspot__deal_pipeline_stage.sql +++ b/models/stg_hubspot__deal_pipeline_stage.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_deal_pipeline_stage_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -29,7 +37,9 @@ with base as ( probability, cast(stage_id as {{ dbt.type_string() }}) as deal_pipeline_stage_id, created_at as deal_pipeline_stage_created_at, - updated_at as deal_pipeline_stage_updated_at + updated_at as deal_pipeline_stage_updated_at, + source_relation + from macro ) diff --git a/models/stg_hubspot__deal_property_history.sql b/models/stg_hubspot__deal_property_history.sql index ccede7b4..de86d7da 100644 --- a/models/stg_hubspot__deal_property_history.sql +++ b/models/stg_hubspot__deal_property_history.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_deal_property_history_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -25,7 +33,9 @@ with base as ( source as change_source, source_id as change_source_id, cast(change_timestamp as {{ dbt.type_timestamp() }}) as change_timestamp, -- source field name = timestamp ; alias declared in macros/get_deal_property_history_columns.sql - value as new_value + value as new_value, + source_relation + from macro ) diff --git a/models/stg_hubspot__deal_stage.sql b/models/stg_hubspot__deal_stage.sql index 28aa16d5..aa9c3682 100644 --- a/models/stg_hubspot__deal_stage.sql +++ b/models/stg_hubspot__deal_stage.sql @@ -17,6 +17,13 @@ fields as ( ) }} + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), @@ -30,7 +37,9 @@ final as ( value as deal_stage_name, _fivetran_active, cast(_fivetran_end as {{ dbt.type_timestamp() }}) as _fivetran_end, - cast(_fivetran_start as {{ dbt.type_timestamp() }}) as _fivetran_start + cast(_fivetran_start as {{ dbt.type_timestamp() }}) as _fivetran_start, + source_relation + from fields ) diff --git a/models/stg_hubspot__email.yml b/models/stg_hubspot__email.yml index de022ddb..70075109 100644 --- a/models/stg_hubspot__email.yml +++ b/models/stg_hubspot__email.yml @@ -4,6 +4,11 @@ models: - name: stg_hubspot__email_event_bounce description: Each record represents a BOUNCE email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -15,15 +20,21 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: returned_response description: The full response from the recipient's email server. - name: returned_status description: The status code returned from the recipient's email server. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_click description: Each record represents a CLICK email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -34,7 +45,6 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: geo_location description: '{{ doc("email_event_location") }}' @@ -44,9 +54,16 @@ models: description: The URL of the webpage that linked to the URL clicked. Whether this is provided, and what its value is, is determined by the recipient's email client. - name: user_agent description: '{{ doc("email_event_user_agent") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_deferred description: Each record represents a DEFERRED email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -55,28 +72,40 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: returned_response description: The full response from the recipient's email server. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_delivered description: Each record represents a DELIVERED email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' - name: event_id description: The ID of the event. tests: - - unique - not_null - name: returned_response description: The full response from the recipient's email server. - name: smtp_id description: An ID attached to the message by HubSpot. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_dropped description: Each record represents a DROPPED email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -93,15 +122,21 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: from_email description: The 'from' field of the email message. - name: reply_to_email description: The 'reply-to' field of the email message. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_forward description: Each record represents a FORWARD email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -110,7 +145,6 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: geo_location description: '{{ doc("email_event_location") }}' @@ -118,9 +152,16 @@ models: description: '{{ doc("email_event_ip_address") }}' - name: user_agent description: '{{ doc("email_event_user_agent") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_open description: Each record represents a OPEN email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -131,7 +172,6 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: geo_location description: '{{ doc("email_event_location") }}' @@ -139,9 +179,16 @@ models: description: '{{ doc("email_event_ip_address") }}' - name: user_agent description: '{{ doc("email_event_user_agent") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_print description: Each record represents a PRINT email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -150,7 +197,6 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: geo_location description: '{{ doc("email_event_location") }}' @@ -158,9 +204,16 @@ models: description: '{{ doc("email_event_ip_address") }}' - name: user_agent description: '{{ doc("email_event_user_agent") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_sent description: Each record represents a SENT email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -173,46 +226,55 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: from_email description: The 'from' field of the email message. - name: reply_to_email description: The 'reply-to' field of the email message. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_spam_report description: Each record represents a SPAM_REPORT email event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' - name: event_id description: The ID of the event. tests: - - unique - not_null - name: ip_address description: '{{ doc("email_event_ip_address") }}' - name: user_agent description: '{{ doc("email_event_user_agent") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event_status_change description: Each record represents a STATUS_CHANGE email event in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' - name: change_source description: The source of the subscription change. - - name: event_id description: The ID of the event. tests: - - unique - not_null - name: is_bounced description: | A HubSpot employee explicitly initiated the status change to block messages to the recipient. (Note this usage has been deprecated in favor of dropping messages with a 'dropReason' of BLOCKED_ADDRESS.) - - name: requested_by_email description: The email address of the person requesting the change on behalf of the recipient. If not applicable, this property is omitted. - name: subscription_status @@ -220,14 +282,20 @@ models: The recipient's portal subscription status. Note that if this is 'UNSUBSCRIBED', the property 'subscriptions' is not necessarily an empty array, nor are all subscriptions contained in it necessarily going to have their statuses set to 'UNSUBSCRIBED'.) - - name: subscriptions description: | An array of JSON objects representing the status of subscriptions for the recipient. Each JSON subscription object is comprised of the properties: 'id', 'status'. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_event description: Each record represents an email event in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - event_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -244,7 +312,6 @@ models: - name: event_id description: The ID of the event. tests: - - unique - not_null - name: event_type description: The type of event. @@ -262,9 +329,16 @@ models: description: The event ID which uniquely identifies the email message's SENT event. If not applicable, this property is omitted. - name: sent_timestamp description: The timestamp of the SENT event related to this event. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__email_campaign description: Each record represents an email campaign in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - email_campaign_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -277,7 +351,6 @@ models: - name: email_campaign_id description: The ID of the email campaign. tests: - - unique - not_null - name: email_campaign_name description: The name of the email campaign. @@ -291,3 +364,5 @@ models: description: The number of messages included as part of the email campaign. - name: num_queued description: The number of messages queued as part of the email campaign. + - name: source_relation + description: '{{ doc("source_relation") }}' diff --git a/models/stg_hubspot__email_campaign.sql b/models/stg_hubspot__email_campaign.sql index ba39bc56..ec2737a1 100644 --- a/models/stg_hubspot__email_campaign.sql +++ b/models/stg_hubspot__email_campaign.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_campaign_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -29,7 +37,9 @@ with base as ( num_queued, sub_type as email_campaign_sub_type, subject as email_campaign_subject, - type as email_campaign_type + type as email_campaign_type, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event.sql b/models/stg_hubspot__email_event.sql index 95928b0b..e125803a 100644 --- a/models/stg_hubspot__email_event.sql +++ b/models/stg_hubspot__email_event.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -33,7 +41,9 @@ with base as ( recipient as recipient_email_address, cast(sent_by_created as {{ dbt.type_timestamp() }}) as sent_timestamp, sent_by_id as sent_by_event_id, - type as event_type + type as event_type, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_bounce.sql b/models/stg_hubspot__email_event_bounce.sql index 7d5879d1..70dae13c 100644 --- a/models/stg_hubspot__email_event_bounce.sql +++ b/models/stg_hubspot__email_event_bounce.sql @@ -15,6 +15,14 @@ with base as ( staging_columns=get_email_event_bounce_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -24,7 +32,9 @@ with base as ( category as bounce_category, id as event_id, response as returned_response, - status as returned_status + status as returned_status, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_click.sql b/models/stg_hubspot__email_event_click.sql index 0c71f265..18db3465 100644 --- a/models/stg_hubspot__email_event_click.sql +++ b/models/stg_hubspot__email_event_click.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_click_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -26,7 +34,9 @@ with base as ( location as geo_location, referer as referer_url, url as click_url, - user_agent + user_agent, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_deferred.sql b/models/stg_hubspot__email_event_deferred.sql index a18570fc..af96cfcf 100644 --- a/models/stg_hubspot__email_event_deferred.sql +++ b/models/stg_hubspot__email_event_deferred.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_deferred_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -22,7 +30,9 @@ with base as ( cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, attempt as attempt_number, id as event_id, - response as returned_response + response as returned_response, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_delivered.sql b/models/stg_hubspot__email_event_delivered.sql index 065d1159..63cd6cac 100644 --- a/models/stg_hubspot__email_event_delivered.sql +++ b/models/stg_hubspot__email_event_delivered.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_delivered_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -22,7 +30,9 @@ with base as ( cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, id as event_id, response as returned_response, - smtp_id + smtp_id, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_dropped.sql b/models/stg_hubspot__email_event_dropped.sql index 6501c839..7bd073e6 100644 --- a/models/stg_hubspot__email_event_dropped.sql +++ b/models/stg_hubspot__email_event_dropped.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_dropped_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -27,7 +35,9 @@ with base as ( from_email, -- source field name = from ; alias declared in macros/get_email_event_dropped_columns.sql id as event_id, reply_to as reply_to_email, - subject as email_subject + subject as email_subject, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_forward.sql b/models/stg_hubspot__email_event_forward.sql index b1377d38..84887258 100644 --- a/models/stg_hubspot__email_event_forward.sql +++ b/models/stg_hubspot__email_event_forward.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_forward_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -24,7 +32,9 @@ with base as ( id as event_id, ip_address, location as geo_location, - user_agent + user_agent, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_open.sql b/models/stg_hubspot__email_event_open.sql index 6f35fcf6..ebfb5501 100644 --- a/models/stg_hubspot__email_event_open.sql +++ b/models/stg_hubspot__email_event_open.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_open_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -25,7 +33,9 @@ with base as ( id as event_id, ip_address, location as geo_location, - user_agent + user_agent, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_print.sql b/models/stg_hubspot__email_event_print.sql index b1eb3146..8c08ed53 100644 --- a/models/stg_hubspot__email_event_print.sql +++ b/models/stg_hubspot__email_event_print.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_print_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -24,7 +32,9 @@ with base as ( id as event_id, ip_address, location as geo_location, - user_agent + user_agent, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_sent.sql b/models/stg_hubspot__email_event_sent.sql index b0ca9936..3548be4e 100644 --- a/models/stg_hubspot__email_event_sent.sql +++ b/models/stg_hubspot__email_event_sent.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_sent_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -25,7 +33,9 @@ with base as ( from_email, -- source field name = from ; alias declared in macros/get_email_event_sent_columns.sql id as event_id, reply_to as reply_to_email, - subject as email_subject + subject as email_subject, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_spam_report.sql b/models/stg_hubspot__email_event_spam_report.sql index 84e9a739..e1c08792 100644 --- a/models/stg_hubspot__email_event_spam_report.sql +++ b/models/stg_hubspot__email_event_spam_report.sql @@ -14,6 +14,13 @@ with base as ( staging_columns=get_email_event_spam_report_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} from base ), fields as ( @@ -22,7 +29,9 @@ with base as ( cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced, id as event_id, ip_address, - user_agent + user_agent, + source_relation + from macro ) diff --git a/models/stg_hubspot__email_event_status_change.sql b/models/stg_hubspot__email_event_status_change.sql index 131952c9..3cddab6d 100644 --- a/models/stg_hubspot__email_event_status_change.sql +++ b/models/stg_hubspot__email_event_status_change.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_email_event_status_change_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -25,7 +33,9 @@ with base as ( portal_subscription_status as subscription_status, requested_by as requested_by_email, source as change_source, - subscriptions + subscriptions, + source_relation + from macro ) diff --git a/models/stg_hubspot__engagement.sql b/models/stg_hubspot__engagement.sql index 86eb601e..3a5fae86 100644 --- a/models/stg_hubspot__engagement.sql +++ b/models/stg_hubspot__engagement.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_engagement_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -26,7 +34,9 @@ with base as ( occurred_timestamp, portal_id, engagement_type, - is_active + is_active, + source_relation + from macro ) diff --git a/models/stg_hubspot__engagement.yml b/models/stg_hubspot__engagement.yml index 643bdd3d..26482cd6 100644 --- a/models/stg_hubspot__engagement.yml +++ b/models/stg_hubspot__engagement.yml @@ -4,6 +4,11 @@ models: - name: stg_hubspot__engagement_call description: Each record represents a CALL engagement event. All source `property_hs_*` fields + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - engagement_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -11,7 +16,6 @@ models: description: The ID of the engagement. tests: - not_null - - unique - name: _fivetran_deleted description: Boolean to mark rows that were deleted in the source database. - name: engagement_type @@ -36,6 +40,8 @@ models: The ID of the team associated with the call. This field determines the team listed as the call creator on the record timeline. PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__engagement_company description: Each record represents a 'link' between a company and an engagement. @@ -46,6 +52,8 @@ models: description: The ID of the related company. - name: engagement_id description: The ID of the related engagement. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__engagement_contact description: Each record represents a 'link' between a contact and an engagement. @@ -56,6 +64,8 @@ models: description: The ID of the related contact. - name: engagement_id description: The ID of the related engagement. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__engagement_deal description: Each record represents a 'link' between a deal and an engagement. @@ -66,9 +76,16 @@ models: description: The ID of the related contact. - name: engagement_id description: The ID of the related engagement. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__engagement_email description: Each record represents an EMAIL engagement event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - engagement_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -78,7 +95,6 @@ models: description: The ID of the engagement. tests: - not_null - - unique - name: engagement_type description: The type of the engagement. - name: created_timestamp @@ -101,9 +117,16 @@ models: The ID of the team associated with the email. This field determines the team listed as the email creator on the record timeline. PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__engagement_meeting description: Each record represents a MEETING engagement event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - engagement_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -113,7 +136,6 @@ models: description: The ID of the engagement. tests: - not_null - - unique - name: engagement_type description: The type of the engagement. - name: created_timestamp @@ -136,9 +158,16 @@ models: The ID of the team associated with the meeting. This field determines the team listed as the meeting creator on the record timeline. PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__engagement_note description: Each record represents a NOTE engagement event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - engagement_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -148,7 +177,6 @@ models: description: The ID of the engagement. tests: - not_null - - unique - name: engagement_type description: The type of the engagement. - name: note @@ -173,9 +201,16 @@ models: The ID of the team associated with the note. This field determines the team listed as the note creator on the record timeline. PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__engagement_task description: Each record represents a TASK engagement event. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - engagement_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -185,7 +220,6 @@ models: description: The ID of the engagement. tests: - not_null - - unique - name: engagement_type description: The type of the engagement. - name: created_timestamp @@ -208,9 +242,16 @@ models: The ID of the team associated with the task. This field determines the team listed as the task creator on the record timeline. PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__engagement description: Each record represents an engagement + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - engagement_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -218,7 +259,6 @@ models: description: The ID of the engagement. tests: - not_null - - unique - name: engagement_type description: One of NOTE, EMAIL, TASK, MEETING, or CALL, the type of the engagement. - name: is_active @@ -242,4 +282,6 @@ models: PLEASE NOTE - This field will only be populated for pre HubSpot v3 API versions. This field is only included to allow for backwards compatibility between HubSpot API versions. This field will be deprecated in the near future. - name: portal_id - description: '{{ doc("portal_id") }}' \ No newline at end of file + description: '{{ doc("portal_id") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' \ No newline at end of file diff --git a/models/stg_hubspot__engagement_call.sql b/models/stg_hubspot__engagement_call.sql index debc6523..df29d22b 100644 --- a/models/stg_hubspot__engagement_call.sql +++ b/models/stg_hubspot__engagement_call.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_engagement_call_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__engagement_company.sql b/models/stg_hubspot__engagement_company.sql index a6a606c5..6cea6bf7 100644 --- a/models/stg_hubspot__engagement_company.sql +++ b/models/stg_hubspot__engagement_company.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_engagement_company_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__engagement_contact.sql b/models/stg_hubspot__engagement_contact.sql index c18bd7c7..f8fd072a 100644 --- a/models/stg_hubspot__engagement_contact.sql +++ b/models/stg_hubspot__engagement_contact.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_engagement_contact_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__engagement_deal.sql b/models/stg_hubspot__engagement_deal.sql index b2afe929..0a302ac8 100644 --- a/models/stg_hubspot__engagement_deal.sql +++ b/models/stg_hubspot__engagement_deal.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_engagement_deal_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__engagement_email.sql b/models/stg_hubspot__engagement_email.sql index 9285c486..dfd89827 100644 --- a/models/stg_hubspot__engagement_email.sql +++ b/models/stg_hubspot__engagement_email.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_engagement_email_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__engagement_meeting.sql b/models/stg_hubspot__engagement_meeting.sql index 9ef290ee..9321aeb4 100644 --- a/models/stg_hubspot__engagement_meeting.sql +++ b/models/stg_hubspot__engagement_meeting.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_engagement_meeting_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__engagement_note.sql b/models/stg_hubspot__engagement_note.sql index f5593a89..c1ccea32 100644 --- a/models/stg_hubspot__engagement_note.sql +++ b/models/stg_hubspot__engagement_note.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_engagement_note_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__engagement_task.sql b/models/stg_hubspot__engagement_task.sql index bffec784..9908e3a0 100644 --- a/models/stg_hubspot__engagement_task.sql +++ b/models/stg_hubspot__engagement_task.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_engagement_task_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__owner.sql b/models/stg_hubspot__owner.sql index 2d72d680..85c369aa 100644 --- a/models/stg_hubspot__owner.sql +++ b/models/stg_hubspot__owner.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_owner_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -28,7 +36,9 @@ with base as ( portal_id, type as owner_type, cast(updated_at as {{ dbt.type_timestamp() }}) as updated_timestamp, - trim( {{ dbt.concat(['first_name', "' '", 'last_name']) }} ) as full_name + trim( {{ dbt.concat(['first_name', "' '", 'last_name']) }} ) as full_name, + source_relation + from macro ) diff --git a/models/stg_hubspot__owner.yml b/models/stg_hubspot__owner.yml index 1de01ba9..9f59a555 100644 --- a/models/stg_hubspot__owner.yml +++ b/models/stg_hubspot__owner.yml @@ -24,4 +24,6 @@ models: - name: owner_type description: The type of owner. - name: updated_timestamp - description: A timestamp for when the owner was last updated. \ No newline at end of file + description: A timestamp for when the owner was last updated. + - name: source_relation + description: '{{ doc("source_relation") }}' \ No newline at end of file diff --git a/models/stg_hubspot__property.sql b/models/stg_hubspot__property.sql index cdcaf671..1aae7c41 100644 --- a/models/stg_hubspot__property.sql +++ b/models/stg_hubspot__property.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_property_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -31,7 +39,9 @@ with base as ( label as property_label, name as property_name, type as property_type, - updated_at + updated_at, + source_relation + from macro ) diff --git a/models/stg_hubspot__property.yml b/models/stg_hubspot__property.yml index 2f71a7a3..0209fe7b 100644 --- a/models/stg_hubspot__property.yml +++ b/models/stg_hubspot__property.yml @@ -4,11 +4,15 @@ models: - name: stg_hubspot__property description: Each record represents a property. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - _fivetran_id + - source_relation columns: - name: _fivetran_id description: Fivetran generated id. Joins to `property_id` in the `property_option` table. tests: - - unique - not_null - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -34,6 +38,8 @@ models: description: One of string, number, date, datetime, or enumeration. - name: updated_at description: Timestamp representing when the property was last updated. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__property_option description: Each record represents an option for the related property. @@ -42,6 +48,7 @@ models: combination_of_columns: - property_id - property_option_label + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' @@ -54,4 +61,6 @@ models: - name: hidden description: Boolean if the option will be displayed in HubSpot - name: property_option_value - description: The internal value of the option. \ No newline at end of file + description: The internal value of the option. + - name: source_relation + description: '{{ doc("source_relation") }}' \ No newline at end of file diff --git a/models/stg_hubspot__property_option.sql b/models/stg_hubspot__property_option.sql index c1f62d0b..9bf29832 100644 --- a/models/stg_hubspot__property_option.sql +++ b/models/stg_hubspot__property_option.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_property_option_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -24,7 +32,9 @@ with base as ( _fivetran_synced, display_order, hidden, - value as property_option_value + value as property_option_value, + source_relation + from macro ) diff --git a/models/stg_hubspot__ticket.sql b/models/stg_hubspot__ticket.sql index 4de0e0e1..87f2efeb 100644 --- a/models/stg_hubspot__ticket.sql +++ b/models/stg_hubspot__ticket.sql @@ -14,6 +14,14 @@ with base as ( staging_columns=get_ticket_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), fields as ( @@ -27,6 +35,14 @@ with base as ( staging_columns=get_ticket_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if all_passthrough_column_check('stg_hubspot__ticket_tmp',get_ticket_columns()) > 0 %} -- just pass everything through if extra columns are present, but ensure required columns are present. {{ @@ -52,7 +68,8 @@ with base as ( ticket_priority, owner_id, ticket_subject, - ticket_content + ticket_content, + source_relation --The below macro adds the fields defined within your hubspot__ticket_pass_through_columns variable into the staging model {{ fivetran_utils.fill_pass_through_columns('hubspot__ticket_pass_through_columns') }} diff --git a/models/stg_hubspot__ticket.yml b/models/stg_hubspot__ticket.yml index 8c9872f9..be6fadf4 100644 --- a/models/stg_hubspot__ticket.yml +++ b/models/stg_hubspot__ticket.yml @@ -11,7 +11,9 @@ models: description: The ID of the related ticket. - name: company_id description: The ID of the related company. - + - name: source_relation + description: '{{ doc("source_relation") }}' + - name: stg_hubspot__ticket_contact description: Each record represents a 'link' between a ticket and a contact. columns: @@ -21,6 +23,8 @@ models: description: The ID of the related ticket. - name: contact_id description: The ID of the related contact. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__ticket_deal description: Each record represents a 'link' between a ticket and a deal. @@ -31,6 +35,8 @@ models: description: The ID of the related ticket. - name: deal_id description: The ID of the related deal. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__ticket_engagement description: Each record represents a 'link' between a ticket and an engagement. @@ -41,6 +47,8 @@ models: description: The ID of the related ticket. - name: engagement_id description: The ID of the related deal. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__ticket_property_history description: Each record represents a change to ticket record in Hubspot. @@ -61,16 +69,22 @@ models: description: '{{ doc("history_name") }}' - name: new_value description: '{{ doc("history_value") }}' + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__ticket description: Each record represents a ticket in Hubspot. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ticket_id + - source_relation columns: - name: _fivetran_synced description: '{{ doc("_fivetran_synced") }}' - name: ticket_id description: ID of the ticket. tests: - - unique - not_null - name: is_ticket_deleted description: '{{ doc("is_deleted") }}' @@ -94,6 +108,8 @@ models: description: Short summary of ticket. - name: ticket_content description: Text in body of the ticket. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__ticket_pipeline_stage description: Each record represents a ticket pipeline stage. @@ -120,6 +136,8 @@ models: description: A timestamp representing when the record was created. - name: ticket_pipeline_stage_updated_at description: A timestamp representing when the record was updated. + - name: source_relation + description: '{{ doc("source_relation") }}' - name: stg_hubspot__ticket_pipeline description: Each record represents a ticket pipeline. @@ -142,3 +160,5 @@ models: description: A timestamp representing when the record was created. - name: ticket_pipeline_updated_at description: A timestamp representing when the record was updated. + - name: source_relation + description: '{{ doc("source_relation") }}' \ No newline at end of file diff --git a/models/stg_hubspot__ticket_company.sql b/models/stg_hubspot__ticket_company.sql index 3221a4f9..cd8d491e 100644 --- a/models/stg_hubspot__ticket_company.sql +++ b/models/stg_hubspot__ticket_company.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_ticket_company_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__ticket_contact.sql b/models/stg_hubspot__ticket_contact.sql index 23978b9c..12f2ab7b 100644 --- a/models/stg_hubspot__ticket_contact.sql +++ b/models/stg_hubspot__ticket_contact.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_ticket_contact_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__ticket_deal.sql b/models/stg_hubspot__ticket_deal.sql index 008d86db..10c35d2f 100644 --- a/models/stg_hubspot__ticket_deal.sql +++ b/models/stg_hubspot__ticket_deal.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_ticket_deal_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__ticket_engagement.sql b/models/stg_hubspot__ticket_engagement.sql index 30ae7f3b..f49faead 100644 --- a/models/stg_hubspot__ticket_engagement.sql +++ b/models/stg_hubspot__ticket_engagement.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_ticket_engagement_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/stg_hubspot__ticket_pipeline.sql b/models/stg_hubspot__ticket_pipeline.sql index fb1e959f..c47ed56d 100644 --- a/models/stg_hubspot__ticket_pipeline.sql +++ b/models/stg_hubspot__ticket_pipeline.sql @@ -17,6 +17,13 @@ fields as ( ) }} + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), @@ -31,7 +38,8 @@ final as ( label as pipeline_label, object_type_id, created_at as ticket_pipeline_created_at, - updated_at as ticket_pipeline_updated_at + updated_at as ticket_pipeline_updated_at, + source_relation from fields ) diff --git a/models/stg_hubspot__ticket_pipeline_stage.sql b/models/stg_hubspot__ticket_pipeline_stage.sql index b7e21599..83ec34b5 100644 --- a/models/stg_hubspot__ticket_pipeline_stage.sql +++ b/models/stg_hubspot__ticket_pipeline_stage.sql @@ -17,6 +17,13 @@ fields as ( ) }} + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + from base ), @@ -33,7 +40,9 @@ final as ( cast(stage_id as {{ dbt.type_int() }} ) as ticket_pipeline_stage_id, ticket_state, created_at as ticket_pipeline_stage_created_at, - updated_at as ticket_pipeline_stage_updated_at + updated_at as ticket_pipeline_stage_updated_at, + source_relation + from fields ) diff --git a/models/stg_hubspot__ticket_property_history.sql b/models/stg_hubspot__ticket_property_history.sql index a559daf1..45288842 100644 --- a/models/stg_hubspot__ticket_property_history.sql +++ b/models/stg_hubspot__ticket_property_history.sql @@ -16,6 +16,14 @@ with base as ( staging_columns=get_ticket_property_history_columns() ) }} + + {{ + fivetran_utils.source_relation( + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) + }} + {% if new_cols | length > 0 %} {{ new_cols }} {% endif %} diff --git a/models/tmp/stg_hubspot__company_property_history_tmp.sql b/models/tmp/stg_hubspot__company_property_history_tmp.sql index 491f77e2..e2b9b53c 100644 --- a/models/tmp/stg_hubspot__company_property_history_tmp.sql +++ b/models/tmp/stg_hubspot__company_property_history_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_company_enabled','hubspot_company_property_history_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','company_property_history')) }} -from {{ var('company_property_history') }} +{{ + fivetran_utils.union_data( + table_identifier='company_property_history', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='company_property_history', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__company_tmp.sql b/models/tmp/stg_hubspot__company_tmp.sql index 2e6fcf60..e1cb17f5 100644 --- a/models/tmp/stg_hubspot__company_tmp.sql +++ b/models/tmp/stg_hubspot__company_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_company_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','company')) }} -from {{ var('company') }} +{{ + fivetran_utils.union_data( + table_identifier='company', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='company', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__contact_list_member_tmp.sql b/models/tmp/stg_hubspot__contact_list_member_tmp.sql index 0a057707..c9859b9c 100644 --- a/models/tmp/stg_hubspot__contact_list_member_tmp.sql +++ b/models/tmp/stg_hubspot__contact_list_member_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled', 'hubspot_contact_list_member_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','contact_list_member')) }} -from {{ var('contact_list_member') }} +{{ + fivetran_utils.union_data( + table_identifier='contact_list_member', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='contact_list_member', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__contact_list_tmp.sql b/models/tmp/stg_hubspot__contact_list_tmp.sql index 24848865..a0d1f331 100644 --- a/models/tmp/stg_hubspot__contact_list_tmp.sql +++ b/models/tmp/stg_hubspot__contact_list_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled', 'hubspot_contact_list_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','contact_list')) }} -from {{ var('contact_list') }} +{{ + fivetran_utils.union_data( + table_identifier='contact_list', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='contact_list', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__contact_merge_audit_tmp.sql b/models/tmp/stg_hubspot__contact_merge_audit_tmp.sql index 91a0ab11..03dea083 100644 --- a/models/tmp/stg_hubspot__contact_merge_audit_tmp.sql +++ b/models/tmp/stg_hubspot__contact_merge_audit_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=(var('hubspot_marketing_enabled', true) and var('hubspot_contact_merge_audit_enabled', false))) }} -select {{ dbt_utils.star(source('hubspot','contact_merge_audit')) }} -from {{ var('contact_merge_audit') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='contact_merge_audit', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='contact_merge_audit', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__contact_property_history_tmp.sql b/models/tmp/stg_hubspot__contact_property_history_tmp.sql index 23d36076..02624d41 100644 --- a/models/tmp/stg_hubspot__contact_property_history_tmp.sql +++ b/models/tmp/stg_hubspot__contact_property_history_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled', 'hubspot_contact_property_enabled', 'hubspot_contact_property_history_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','contact_property_history')) }} -from {{ var('contact_property_history') }} +{{ + fivetran_utils.union_data( + table_identifier='contact_property_history', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='contact_property_history', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__contact_tmp.sql b/models/tmp/stg_hubspot__contact_tmp.sql index f87850c9..8f85fd4e 100644 --- a/models/tmp/stg_hubspot__contact_tmp.sql +++ b/models/tmp/stg_hubspot__contact_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled', 'hubspot_contact_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','contact')) }} -from {{ var('contact') }} +{{ + fivetran_utils.union_data( + table_identifier='contact', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='contact', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__deal_company_tmp.sql b/models/tmp/stg_hubspot__deal_company_tmp.sql index 19debf14..ed9e1fe6 100644 --- a/models/tmp/stg_hubspot__deal_company_tmp.sql +++ b/models/tmp/stg_hubspot__deal_company_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled','hubspot_deal_company_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','deal_company')) }} -from {{ var('deal_company') }} +{{ + fivetran_utils.union_data( + table_identifier='deal_company', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='deal_company', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__deal_contact_tmp.sql b/models/tmp/stg_hubspot__deal_contact_tmp.sql index f4738d95..f5ba922a 100644 --- a/models/tmp/stg_hubspot__deal_contact_tmp.sql +++ b/models/tmp/stg_hubspot__deal_contact_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled','hubspot_deal_contact_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','deal_contact')) }} -from {{ var('deal_contact') }} +{{ + fivetran_utils.union_data( + table_identifier='deal_contact', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='deal_contact', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__deal_pipeline_stage_tmp.sql b/models/tmp/stg_hubspot__deal_pipeline_stage_tmp.sql index ef755b37..a13985dd 100644 --- a/models/tmp/stg_hubspot__deal_pipeline_stage_tmp.sql +++ b/models/tmp/stg_hubspot__deal_pipeline_stage_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','deal_pipeline_stage')) }} -from {{ var('deal_pipeline_stage') }} +{{ + fivetran_utils.union_data( + table_identifier='deal_pipeline_stage', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='deal_pipeline_stage', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__deal_pipeline_tmp.sql b/models/tmp/stg_hubspot__deal_pipeline_tmp.sql index 2f2ebf08..a309f63e 100644 --- a/models/tmp/stg_hubspot__deal_pipeline_tmp.sql +++ b/models/tmp/stg_hubspot__deal_pipeline_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','deal_pipeline')) }} -from {{ var('deal_pipeline') }} +{{ + fivetran_utils.union_data( + table_identifier='deal_pipeline', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='deal_pipeline', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__deal_property_history_tmp.sql b/models/tmp/stg_hubspot__deal_property_history_tmp.sql index 061dc582..0001b08b 100644 --- a/models/tmp/stg_hubspot__deal_property_history_tmp.sql +++ b/models/tmp/stg_hubspot__deal_property_history_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled','hubspot_deal_property_history_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','deal_property_history')) }} -from {{ var('deal_property_history') }} +{{ + fivetran_utils.union_data( + table_identifier='deal_property_history', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='deal_property_history', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__deal_stage_tmp.sql b/models/tmp/stg_hubspot__deal_stage_tmp.sql index 943b7b5e..ba50618c 100644 --- a/models/tmp/stg_hubspot__deal_stage_tmp.sql +++ b/models/tmp/stg_hubspot__deal_stage_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','deal_stage')) }} -from {{ var('deal_stage') }} +{{ + fivetran_utils.union_data( + table_identifier='deal_stage', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='deal_stage', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__deal_tmp.sql b/models/tmp/stg_hubspot__deal_tmp.sql index 3f2f43cb..646c171e 100644 --- a/models/tmp/stg_hubspot__deal_tmp.sql +++ b/models/tmp/stg_hubspot__deal_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_deal_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','deal')) }} -from {{ var('deal') }} +{{ + fivetran_utils.union_data( + table_identifier='deal', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='deal', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_campaign_tmp.sql b/models/tmp/stg_hubspot__email_campaign_tmp.sql index f8c05e97..19b0ea83 100644 --- a/models/tmp/stg_hubspot__email_campaign_tmp.sql +++ b/models/tmp/stg_hubspot__email_campaign_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_campaign')) }} -from {{ var('email_campaign') }} +{{ + fivetran_utils.union_data( + table_identifier='email_campaign', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_campaign', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_bounce_tmp.sql b/models/tmp/stg_hubspot__email_event_bounce_tmp.sql index 176a10ac..8d32f374 100644 --- a/models/tmp/stg_hubspot__email_event_bounce_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_bounce_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_bounce_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_bounce')) }} -from {{ var('email_event_bounce') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_bounce', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_bounce', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_click_tmp.sql b/models/tmp/stg_hubspot__email_event_click_tmp.sql index cf83ce77..cf60005a 100644 --- a/models/tmp/stg_hubspot__email_event_click_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_click_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_click_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_click')) }} -from {{ var('email_event_click') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_click', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_click', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_deferred_tmp.sql b/models/tmp/stg_hubspot__email_event_deferred_tmp.sql index 1aecd211..ea529e15 100644 --- a/models/tmp/stg_hubspot__email_event_deferred_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_deferred_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_deferred_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_deferred')) }} -from {{ var('email_event_deferred') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_deferred', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_deferred', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_delivered_tmp.sql b/models/tmp/stg_hubspot__email_event_delivered_tmp.sql index 9da08fbd..c8197d55 100644 --- a/models/tmp/stg_hubspot__email_event_delivered_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_delivered_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_delivered_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_delivered')) }} -from {{ var('email_event_delivered') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_delivered', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_delivered', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_dropped_tmp.sql b/models/tmp/stg_hubspot__email_event_dropped_tmp.sql index e5515bf8..55221f2b 100644 --- a/models/tmp/stg_hubspot__email_event_dropped_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_dropped_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_dropped_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_dropped')) }} -from {{ var('email_event_dropped') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_dropped', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_dropped', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_forward_tmp.sql b/models/tmp/stg_hubspot__email_event_forward_tmp.sql index 1e38824a..f0f08345 100644 --- a/models/tmp/stg_hubspot__email_event_forward_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_forward_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_forward_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_forward')) }} -from {{ var('email_event_forward') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_forward', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_forward', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_open_tmp.sql b/models/tmp/stg_hubspot__email_event_open_tmp.sql index 875d7f28..e5fde364 100644 --- a/models/tmp/stg_hubspot__email_event_open_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_open_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_open_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_open')) }} -from {{ var('email_event_open') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_open', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_open', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_print_tmp.sql b/models/tmp/stg_hubspot__email_event_print_tmp.sql index eb3daeb9..cf39bcba 100644 --- a/models/tmp/stg_hubspot__email_event_print_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_print_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_print_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_print')) }} -from {{ var('email_event_print') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_print', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_print', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_sent_tmp.sql b/models/tmp/stg_hubspot__email_event_sent_tmp.sql index 6874148b..7638adb4 100644 --- a/models/tmp/stg_hubspot__email_event_sent_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_sent_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_sent_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_sent')) }} -from {{ var('email_event_sent') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_sent', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_sent', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_spam_report_tmp.sql b/models/tmp/stg_hubspot__email_event_spam_report_tmp.sql index 6c43e218..882bba01 100644 --- a/models/tmp/stg_hubspot__email_event_spam_report_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_spam_report_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_spam_report_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_spam_report')) }} -from {{ var('email_event_spam_report') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_spam_report', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_spam_report', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_status_change_tmp.sql b/models/tmp/stg_hubspot__email_event_status_change_tmp.sql index e657f30c..e41bff80 100644 --- a/models/tmp/stg_hubspot__email_event_status_change_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_status_change_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled','hubspot_email_event_status_change_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event_status_change')) }} -from {{ var('email_event_status_change') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event_status_change', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event_status_change', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__email_event_tmp.sql b/models/tmp/stg_hubspot__email_event_tmp.sql index 7ce08128..2c9087ac 100644 --- a/models/tmp/stg_hubspot__email_event_tmp.sql +++ b/models/tmp/stg_hubspot__email_event_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_marketing_enabled','hubspot_email_event_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','email_event')) }} -from {{ var('email_event') }} +{{ + fivetran_utils.union_data( + table_identifier='email_event', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='email_event', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_call_tmp.sql b/models/tmp/stg_hubspot__engagement_call_tmp.sql index 657748c4..00f3b04f 100644 --- a/models/tmp/stg_hubspot__engagement_call_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_call_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled','hubspot_engagement_call_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement_call')) }} -from {{ var('engagement_call') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement_call', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement_call', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_company_tmp.sql b/models/tmp/stg_hubspot__engagement_company_tmp.sql index 6251a86a..47c676e0 100644 --- a/models/tmp/stg_hubspot__engagement_company_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_company_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled','hubspot_engagement_company_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement_company')) }} -from {{ var('engagement_company') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement_company', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement_company', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_contact_tmp.sql b/models/tmp/stg_hubspot__engagement_contact_tmp.sql index 4fd671ba..d561d93f 100644 --- a/models/tmp/stg_hubspot__engagement_contact_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_contact_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled','hubspot_engagement_contact_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement_contact')) }} -from {{ var('engagement_contact') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement_contact', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement_contact', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_deal_tmp.sql b/models/tmp/stg_hubspot__engagement_deal_tmp.sql index 66407d72..ef4096c9 100644 --- a/models/tmp/stg_hubspot__engagement_deal_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_deal_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled','hubspot_engagement_deal_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement_deal')) }} -from {{ var('engagement_deal') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement_deal', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement_deal', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_email_tmp.sql b/models/tmp/stg_hubspot__engagement_email_tmp.sql index e800209f..2ce06632 100644 --- a/models/tmp/stg_hubspot__engagement_email_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_email_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled','hubspot_engagement_email_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement_email')) }} -from {{ var('engagement_email') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement_email', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement_email', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_meeting_tmp.sql b/models/tmp/stg_hubspot__engagement_meeting_tmp.sql index bf582549..6e5eea44 100644 --- a/models/tmp/stg_hubspot__engagement_meeting_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_meeting_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled','hubspot_engagement_meeting_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement_meeting')) }} -from {{ var('engagement_meeting') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement_meeting', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement_meeting', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_note_tmp.sql b/models/tmp/stg_hubspot__engagement_note_tmp.sql index 4b568f95..674af7f0 100644 --- a/models/tmp/stg_hubspot__engagement_note_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_note_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled','hubspot_engagement_note_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement_note')) }} -from {{ var('engagement_note') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement_note', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement_note', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_task_tmp.sql b/models/tmp/stg_hubspot__engagement_task_tmp.sql index 42f56c69..64e9520a 100644 --- a/models/tmp/stg_hubspot__engagement_task_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_task_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled','hubspot_engagement_task_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement_task')) }} -from {{ var('engagement_task') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement_task', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement_task', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__engagement_tmp.sql b/models/tmp/stg_hubspot__engagement_tmp.sql index 71b556f6..d389f8fb 100644 --- a/models/tmp/stg_hubspot__engagement_tmp.sql +++ b/models/tmp/stg_hubspot__engagement_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_engagement_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','engagement')) }} -from {{ var('engagement') }} +{{ + fivetran_utils.union_data( + table_identifier='engagement', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='engagement', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__owner_tmp.sql b/models/tmp/stg_hubspot__owner_tmp.sql index d175dc90..86871f6e 100644 --- a/models/tmp/stg_hubspot__owner_tmp.sql +++ b/models/tmp/stg_hubspot__owner_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=fivetran_utils.enabled_vars(['hubspot_sales_enabled','hubspot_owner_enabled'])) }} -select {{ dbt_utils.star(source('hubspot','owner')) }} -from {{ var('owner') }} +{{ + fivetran_utils.union_data( + table_identifier='owner', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='owner', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__property_option_tmp.sql b/models/tmp/stg_hubspot__property_option_tmp.sql index f4c66a3b..76b54087 100644 --- a/models/tmp/stg_hubspot__property_option_tmp.sql +++ b/models/tmp/stg_hubspot__property_option_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_property_enabled', True)) }} -select {{ dbt_utils.star(source('hubspot','property_option')) }} -from {{ var('property_option') }} +{{ + fivetran_utils.union_data( + table_identifier='property_option', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='property_option', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__property_tmp.sql b/models/tmp/stg_hubspot__property_tmp.sql index b6489113..312fa817 100644 --- a/models/tmp/stg_hubspot__property_tmp.sql +++ b/models/tmp/stg_hubspot__property_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_property_enabled', True)) }} -select {{ dbt_utils.star(source('hubspot','property')) }} -from {{ var('property') }} +{{ + fivetran_utils.union_data( + table_identifier='property', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='property', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__ticket_company_tmp.sql b/models/tmp/stg_hubspot__ticket_company_tmp.sql index a1638288..cc1849ec 100644 --- a/models/tmp/stg_hubspot__ticket_company_tmp.sql +++ b/models/tmp/stg_hubspot__ticket_company_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_service_enabled', False)) }} -select {{ dbt_utils.star(source('hubspot','ticket_company')) }} -from {{ var('ticket_company') }} +{{ + fivetran_utils.union_data( + table_identifier='ticket_company', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='ticket_company', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__ticket_contact_tmp.sql b/models/tmp/stg_hubspot__ticket_contact_tmp.sql index 547fae96..cd4438a2 100644 --- a/models/tmp/stg_hubspot__ticket_contact_tmp.sql +++ b/models/tmp/stg_hubspot__ticket_contact_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_service_enabled', False)) }} -select {{ dbt_utils.star(source('hubspot','ticket_contact')) }} -from {{ var('ticket_contact') }} +{{ + fivetran_utils.union_data( + table_identifier='ticket_contact', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='ticket_contact', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__ticket_deal_tmp.sql b/models/tmp/stg_hubspot__ticket_deal_tmp.sql index 904127fc..c57201a2 100644 --- a/models/tmp/stg_hubspot__ticket_deal_tmp.sql +++ b/models/tmp/stg_hubspot__ticket_deal_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=(var('hubspot_service_enabled', false) and var('hubspot_ticket_deal_enabled', false))) }} -select {{ dbt_utils.star(source('hubspot','ticket_deal')) }} -from {{ var('ticket_deal') }} +{{ + fivetran_utils.union_data( + table_identifier='ticket_deal', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='ticket_deal', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__ticket_engagement_tmp.sql b/models/tmp/stg_hubspot__ticket_engagement_tmp.sql index 0aec21ae..ba1d765b 100644 --- a/models/tmp/stg_hubspot__ticket_engagement_tmp.sql +++ b/models/tmp/stg_hubspot__ticket_engagement_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_service_enabled', False)) }} -select {{ dbt_utils.star(source('hubspot','ticket_engagement')) }} -from {{ var('ticket_engagement') }} +{{ + fivetran_utils.union_data( + table_identifier='ticket_engagement', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='ticket_engagement', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__ticket_pipeline_stage_tmp.sql b/models/tmp/stg_hubspot__ticket_pipeline_stage_tmp.sql index 7cb7ed91..bfb8addc 100644 --- a/models/tmp/stg_hubspot__ticket_pipeline_stage_tmp.sql +++ b/models/tmp/stg_hubspot__ticket_pipeline_stage_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_service_enabled', False)) }} -select {{ dbt_utils.star(source('hubspot','ticket_pipeline_stage')) }} -from {{ var('ticket_pipeline_stage') }} +{{ + fivetran_utils.union_data( + table_identifier='ticket_pipeline_stage', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='ticket_pipeline_stage', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__ticket_pipeline_tmp.sql b/models/tmp/stg_hubspot__ticket_pipeline_tmp.sql index 044df0e9..22b4c02a 100644 --- a/models/tmp/stg_hubspot__ticket_pipeline_tmp.sql +++ b/models/tmp/stg_hubspot__ticket_pipeline_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_service_enabled', False)) }} -select {{ dbt_utils.star(source('hubspot','ticket_pipeline')) }} -from {{ var('ticket_pipeline') }} +{{ + fivetran_utils.union_data( + table_identifier='ticket_pipeline', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='ticket_pipeline', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__ticket_property_history_tmp.sql b/models/tmp/stg_hubspot__ticket_property_history_tmp.sql index 4bf047b4..6dd3e090 100644 --- a/models/tmp/stg_hubspot__ticket_property_history_tmp.sql +++ b/models/tmp/stg_hubspot__ticket_property_history_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_service_enabled', False)) }} -select {{ dbt_utils.star(source('hubspot','ticket_property_history')) }} -from {{ var('ticket_property_history') }} +{{ + fivetran_utils.union_data( + table_identifier='ticket_property_history', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='ticket_property_history', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/models/tmp/stg_hubspot__ticket_tmp.sql b/models/tmp/stg_hubspot__ticket_tmp.sql index 945886cd..db2d4379 100644 --- a/models/tmp/stg_hubspot__ticket_tmp.sql +++ b/models/tmp/stg_hubspot__ticket_tmp.sql @@ -1,4 +1,14 @@ {{ config(enabled=var('hubspot_service_enabled', False)) }} -select {{ dbt_utils.star(source('hubspot','ticket')) }} -from {{ var('ticket') }} \ No newline at end of file +{{ + fivetran_utils.union_data( + table_identifier='ticket', + database_variable='hubspot_database', + schema_variable='hubspot_schema', + default_database=target.database, + default_schema='hubspot', + default_variable='ticket', + union_schema_variable='hubspot_union_schemas', + union_database_variable='hubspot_union_databases' + ) +}} \ No newline at end of file diff --git a/packages.yml b/packages.yml index d445c69d..78b676ee 100644 --- a/packages.yml +++ b/packages.yml @@ -1,6 +1,9 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.4.0", "<0.5.0"] - +# - package: fivetran/fivetran_utils +# version: [">=0.4.0", "<0.5.0"] +- git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: feature/enhance-union-data + warn-unpinned: false + - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] \ No newline at end of file From b668a584935515edbacfe82ae1636ae4850246c4 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 8 Jan 2024 10:44:54 -0300 Subject: [PATCH 02/16] update readme --- README.md | 1296 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1296 insertions(+) diff --git a/README.md b/README.md index 6fd70bbb..55b0d425 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,9 @@ packages: - package: fivetran/hubspot_source version: [">=0.15.0", "<0.16.0"] ``` + ## Step 3: Define database and schema variables +### Option 1: Single connector By default, this package runs using your destination and the `hubspot` schema. If this is not where your HubSpot data is (for example, if your HubSpot schema is named `hubspot_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -54,6 +56,1298 @@ vars: hubspot_database: your_destination_name hubspot_schema: your_schema_name ``` +> **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. + +### Option 2: Union multiple connectors +If you have multiple Hubspot connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `hubspot_union_schemas` OR `hubspot_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: + +```yml +# dbt_project.yml + +vars: + hubspot_union_schemas: ['hubspot_usa','hubspot_canada'] # use this if the data is in different schemas/datasets of the same database/project + hubspot_union_databases: ['hubspot_usa','hubspot_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` + +#### Recommended: Incorporate unioned sources into DAG +By default, this package defines one single-connector source, called `hubspot`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Hubspot sources, though the package will run successfully. + +To properly incorporate all of your Hubspot connectors into your project's DAG: +1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. + +
Expand for source configuration template

+ +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_hubspot_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition + +```yml +sources: + - name: + schema: + database: + loader: Fivetran + loaded_at_field: _fivetran_synced + tables: &hubspot_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ + - name: calendar_event + description: Each record represents a calendar event. + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: avatar_url + description: URL of image associated with social media event. + - name: campaign_guid + description: Value of campaign GUID associated with Task. + - name: category + description: | + Type of event. If the event type is PUBLISHING_TASK, it is one of BLOG_POST, EMAIL, LANDING_PAGE, CUSTOM. + If event type is SOCIAL, it is one of twitter, facebook, linkedin, googlepluspages. + If event type is CONTENT, it is one of email, recurring-email, blog-post, landing-page, legacy-page, site-page. + - name: category_id + description: | + For event type of PUBLISHING_TASK, a numeric value corresponding to the type of task; one of 3 (BLOG_POST), 2 (EMAIL), 1 (LANDING_PAGE), 0 (CUSTOM). + For event type of SOCIAL, this is 0. + If event type is CONTENT, it is one of 2 (email, recurring-email), 3 (blog-post), 1 (landing-page), 5 (legacy-page), 4 (site-page). + - name: content_group_id + description: | + The ID of the content group (aka blog) that the associated Blog Post belongs to, if any. + Otherwise null. Only populated for single task GETs and for Blog Post Tasks. + - name: content_id + description: ID value of the COS content object associated with the event, null for social or if nothing associated. + - name: created_by + description: HubSpot ID of the user that the event was created by. + - name: description + description: Description of Event. + - name: event_date + description: If task, When the task is set to be due, otherwise when the event is/ was scheduled for; in milliseconds since the epoch. + - name: event_type + description: Type of calendar event; for tasks this is PUBLISHING_TASK, for COS Items, this is CONTENT, for social media events, this is SOCIAL + - name: id + description: The unique ID of the task. + - name: is_recurring + description: Whether the event is recurring. + - name: name + description: Name of Event. + - name: owner_id + description: TASK - HubSpot ID of the user that the task is assigned to. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: preview_key + description: Preview key of content object; used for showing previews of unpublished items. + - name: social_display_name + description: Social media full name associate with event. + - name: social_username + description: Social media user name associated with event. + - name: state + description: For type publishing task, value of TODO or DONE, for others, a value of SCHEDULED, PUBLISHED, PUBLISHED_OR_SCHEDULED. + - name: topic_ids + description: The list of IDs of topics associated with the associated Blog Post, if any. Otherwise null. Only populated for single task GETs and for Blog Post Tasks. + - name: url + description: Public URL of content item. + + - name: company + description: Each record represents a company in Hubspot. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_company_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: id + description: The ID of the company. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: is_deleted + description: '{{ doc("is_deleted") }}' + - name: property_name + description: The name of the company. + - name: property_description + description: A short statement about the company's mission and goals. + - name: property_createdate + description: The date the company was added to your account. + - name: property_industry + description: The type of business the company performs. + - name: property_address + description: The street address of the company. + - name: property_address_2 + description: Additional address information for the company. + - name: property_city + description: The city where the company is located. + - name: property_state + description: The state where the company is located. + - name: property_country + description: The country where the company is located. + - name: property_annualrevenue + description: The actual or estimated annual revenue of the company. + + - name: company_property_history + description: Each record represents a change to company record in Hubspot. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_company_enabled', true) and var('hubspot_company_property_history_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: company_id + description: The ID of the related company record. + - name: name + description: '{{ doc("history_name") }}' + - name: source + description: '{{ doc("history_source") }}' + - name: source_id + description: '{{ doc("history_source_id") }}' + - name: timestamp + description: '{{ doc("history_timestamp") }}' + - name: value + description: '{{ doc("history_value") }}' + + - name: contact_merge_audit + description: > + DEPRECATED FOR NON-BIGQUERY USERS (will be deprecated on BigQuery as well). Each record contains a contact merge event and the contacts effected by the merge. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_merge_audit_enabled', false) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: canonical_vid + description: The contact ID of the contact which the vid_to_merge contact was merged into. + - name: contact_id + description: The ID of the contact. + - name: entity_id + description: The ID of the related entity. + - name: first_name + description: The contact's first name. + - name: last_name + description: The contact's last name. + - name: num_properties_moved + description: The number of properties which were removed from the merged contact. + - name: timestamp + description: Timestamp of when the contacts were merged. + - name: user_id + description: The ID of the user. + - name: vid_to_merge + description: The ID of the contact which was merged. + + - name: contact + freshness: + warn_after: {count: 84, period: hour} + error_after: {count: 168, period: hour} + description: Each record represents a contact in Hubspot. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_enabled', true) }}" + columns: + - name: _fivetran_deleted + description: '{{ doc("_fivetran_deleted") }}' + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: id + description: The ID of the contact. + - name: property_email_1 + description: The email address of the contact. + - name: property_company + description: The name of the contact's company. + - name: property_firstname + description: The contact's first name. + - name: property_lastname + description: The contact's last name. + - name: property_email + description: The contact's email. + - name: property_createdate + description: The date that the contact was created in your HubSpot account. + - name: property_jobtitle + description: The contact's job title. + - name: property_annualrevenue + description: The contact's annual company revenue. + - name: property_hs_calculated_merged_vids + description: > + List of mappings representing contact IDs that have been merged into the contact at hand. + Format: :;: + This field has replaced the `CONTACT_MERGE_AUDIT` table, which was deprecated by the Hubspot v3 CRM API. + + - name: contact_form_submission + description: Table containing contact form submission information + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: contact_id + description: The ID of the related contact. + - name: conversion_id + description: A Unique ID for the specific form conversion. + - name: form_id + description: The GUID of the form that the submission belongs to. + - name: page_url + description: The URL that the form was submitted on, if applicable. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: timestamp + description: A Unix timestamp in milliseconds of the time the submission occurred. + - name: title + description: The title of the page that the form was submitted on. This will default to the name of the form if no title is provided. + + - name: contact_list + description: Each record represents a contact list in Hubspot. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_list_enabled', true) }}" + columns: + - name: _fivetran_deleted + description: '{{ doc("_fivetran_deleted") }}' + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: created_at + description: A timestamp of the time the list was created. + - name: deleteable + description: If this is false, this is a system list and cannot be deleted. + - name: dynamic + description: Whether the contact list is dynamic. + - name: id + description: The ID of the contact list. + - name: metadata_error + description: Any errors that happened the last time the list was processed. + - name: metadata_last_processing_state_change_at + description: A timestamp of the last time that the processing state changed. + - name: metadata_last_size_change_at + description: A timestamp of the last time that the size of the list changed. + - name: metadata_processing + description: | + One of DONE, REFRESHING, INITIALIZING, or PROCESSING. + DONE indicates the list has finished processing, any other value indicates that list membership is being evaluated. + - name: metadata_size + description: The approximate number of contacts in the list. + - name: name + description: The name of the contact list. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: updated_at + description: A timestamp of the time that the list was last modified. + + - name: contact_list_member + description: Each record represents a 'link' between a contact and a contact list. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_list_member_enabled', true) }}" + columns: + - name: _fivetran_deleted + description: '{{ doc("_fivetran_deleted") }}' + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: added_at + description: The timestamp a contact was added to a list. + - name: contact_id + description: The ID of the related contact. + - name: contact_list_id + description: The ID of the related contact list. + + - name: contact_property_history + freshness: + warn_after: {count: 84, period: hour} + error_after: {count: 168, period: hour} + description: Each record represents a change to contact record in Hubspot. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_property_enabled', true) and var('hubspot_contact_property_history_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: contact_id + description: The ID of the related contact record. + - name: name + description: '{{ doc("history_name") }}' + - name: source + description: '{{ doc("history_source") }}' + - name: source_id + description: '{{ doc("history_source_id") }}' + - name: timestamp + description: '{{ doc("history_timestamp") }}' + - name: value + description: '{{ doc("history_value") }}' + + - name: deal + description: Each record represents a deal in Hubspot. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) }}" + columns: + - name: deal_id + description: The ID of the deal. + - name: is_deleted + description: Whether the record was deleted. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: deal_pipeline_id + description: The ID of the deal's pipeline. + - name: deal_pipeline_stage_id + description: The ID of the deal's pipeline stage. + - name: owner_id + description: The ID of the deal's owner. + - name: property_dealname + description: The name you have given this deal. + - name: property_description + description: A brief description of the deal. + - name: property_amount + description: The total value of the deal in the deal's currency. + - name: property_closedate + description: The day the deal is expected to close, or was closed. + - name: property_createdate + description: The date the deal was created. This property is set automatically by HubSpot. + + - name: deal_stage + description: Each record represents a deal stage. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) }}" + columns: + - name: _fivetran_active + description: Boolean indicating whether the deal stage is active. + - name: _fivetran_end + description: The Fivetran calculated exist time of the deal stage. + - name: _fivetran_start + description: The date the deal stage was entered. + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: date_entered + description: The timestamp the deal stage was entered. + - name: deal_id + description: Reference to the deal. + - name: source + description: The relevant source of the deal stage. + - name: source_id + description: Reference to the source. + - name: value + description: The value of the deal stage. Typically the name of the stage. + + - name: deal_company + description: Each record represents a 'link' between a deal and company. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) and var('hubspot_deal_company_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: company_id + description: The ID of the company. + - name: deal_id + description: The ID of the deal. + - name: type_id + description: The ID of the type. + + - name: deal_contact + description: Each record represents a 'link' between a deal and a contact. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) and var('hubspot_deal_contact_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: contact_id + description: The ID of the contact. + - name: deal_id + description: The ID of the deal. + - name: type_id + description: The ID of the type. + + - name: deal_pipeline + description: Each record represents a pipeline in Hubspot. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) }}" + columns: + - name: _fivetran_deleted + description: '{{ doc("_fivetran_deleted") }}' + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: active + description: Whether the stage is currently in use. + - name: display_order + description: Used to determine the order in which the pipelines appear when viewed in HubSpot. + - name: label + description: The human-readable label for the pipeline. The label is used when showing the pipeline in HubSpot. + - name: pipeline_id + description: The ID of the pipeline. + - name: created_at + description: A timestamp representing when the record was created. + - name: updated_at + description: A timestamp representing when the record was updated. + + - name: deal_pipeline_stage + description: Each record represents a pipeline stage in Hubspot. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) }}" + columns: + - name: _fivetran_deleted + description: '{{ doc("_fivetran_deleted") }}' + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: active + description: Whether the pipeline stage is currently in use. + - name: closed_won + description: Whether the stage represents a Closed Won deal. + - name: display_order + description: Used to determine the order in which the stages appear when viewed in HubSpot. + - name: label + description: The human-readable label for the stage. The label is used when showing the stage in HubSpot. + - name: pipeline_id + description: The ID of the related pipeline. + - name: probability + description: The probability that the deal will close. Used for the deal forecast. + - name: stage_id + description: The ID of the pipeline stage. + - name: created_at + description: A timestamp representing when the record was created. + - name: updated_at + description: A timestamp representing when the record was updated. + + - name: deal_property_history + description: Each record represents a change to deal record in Hubspot. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) and var('hubspot_deal_property_history_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: deal_id + description: The ID of the related deal record. + - name: name + description: '{{ doc("history_name") }}' + - name: source + description: '{{ doc("history_source") }}' + - name: source_id + description: '{{ doc("history_source_id") }}' + - name: timestamp + description: '{{ doc("history_timestamp") }}' + - name: value + description: '{{ doc("history_value") }}' + + - name: email_campaign + description: Each record represents an email campaign in Hubspot. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: app_id + description: The app ID. + - name: app_name + description: The app name. + - name: content_id + description: The ID of the content. + - name: id + description: The ID of the email campaign. + - name: name + description: The name of the email campaign. + - name: num_included + description: The number of messages included as part of the email campaign. + - name: num_queued + description: The number of messages queued as part of the email campaign. + - name: sub_type + description: The email campaign sub-type. + - name: subject + description: The subject of the email campaign. + - name: type + description: The email campaign type. + + - name: email_event + freshness: + warn_after: {count: 84, period: hour} + error_after: {count: 168, period: hour} + description: Each record represents an email event in Hubspot. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: app_id + description: The ID of the app that sent the email. + - name: caused_by_created + description: The timestamp of the event that caused this event. + - name: caused_by_id + description: The event ID which uniquely identifies the event which directly caused this event. If not applicable, this property is omitted. + - name: created + description: The created timestamp of the event. + - name: email_campaign_id + description: The ID of the related email campaign. + - name: filtered_event + description: A boolean representing whether the event has been filtered out of reporting based on customer reports settings or not. + - name: id + description: The ID of the event. + - name: obsoleted_by_created + description: The timestamp of the event that made the current event obsolete. + - name: obsoleted_by_id + description: The event ID which uniquely identifies the follow-on event which makes this current event obsolete. If not applicable, this property is omitted. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: recipient + description: The email address of the contact related to the event. + - name: sent_by_created + description: The timestamp of the SENT event related to this event. + - name: sent_by_id + description: The event ID which uniquely identifies the email message's SENT event. If not applicable, this property is omitted. + - name: type + description: The type of event. + + - name: email_event_bounce + description: Each record represents a BOUNCE email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_bounce_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: category + description: | + The best-guess of the type of bounce encountered. + If an appropriate category couldn't be determined, this property is omitted. See below for the possible values. + Note that this is a derived value, and may be modified at any time to improve the accuracy of classification. + - name: id + description: The ID of the event. + - name: response + description: The full response from the recipient's email server. + - name: status + description: The status code returned from the recipient's email server. + + - name: email_event_click + description: Each record represents a CLICK email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_click_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: browser + description: '{{ doc("email_event_browser") }}' + - name: id + description: The ID of the event. + - name: ip_address + description: '{{ doc("email_event_ip_address") }}' + - name: location + description: '{{ doc("email_event_location") }}' + - name: referer + description: The URL of the webpage that linked to the URL clicked. Whether this is provided, and what its value is, is determined by the recipient's email client. + - name: url + description: The URL within the message that the recipient clicked. + - name: user_agent + description: '{{ doc("email_event_user_agent") }}' + + - name: email_event_deferred + description: Each record represents a DEFERRED email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_deferred_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: attempt + description: The delivery attempt number. + - name: id + description: The ID of the event. + - name: response + description: The full response from the recipient's email server. + + - name: email_event_delivered + description: Each record represents a DELIVERED email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_delivered_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: id + description: The ID of the event. + - name: response + description: The full response from the recipient's email server. + - name: smtp_id + description: An ID attached to the message by HubSpot. + + - name: email_event_dropped + description: Each record represents a DROPPED email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_dropped_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: bcc + description: The 'bcc' field of the email message. + - name: cc + description: The 'cc' field of the email message. + - name: drop_message + description: The raw message describing why the email message was dropped. This will usually provide additional details beyond 'dropReason'. + - name: drop_reason + description: The reason why the email message was dropped. See below for the possible values. + - name: from + description: The 'from' field of the email message. + - name: id + description: The ID of the event. + - name: reply_to + description: The 'reply-to' field of the email message. + - name: subject + description: The subject line of the email message. + + - name: email_event_forward + description: Each record represents a FORWARD email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_forward_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: browser + description: '{{ doc("email_event_browser") }}' + - name: id + description: The ID of the event. + - name: ip_address + description: '{{ doc("email_event_ip_address") }}' + - name: location + description: '{{ doc("email_event_location") }}' + - name: user_agent + description: '{{ doc("email_event_user_agent") }}' + + - name: email_event_open + description: Each record represents a OPEN email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_open_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: browser + description: '{{ doc("email_event_browser") }}' + - name: duration + description: If provided and nonzero, the approximate number of milliseconds the user had opened the email. + - name: id + description: The ID of the event. + - name: ip_address + description: '{{ doc("email_event_ip_address") }}' + - name: location + description: '{{ doc("email_event_location") }}' + - name: user_agent + description: '{{ doc("email_event_user_agent") }}' + + - name: email_event_print + description: Each record represents a PRINT email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_print_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: browser + description: '{{ doc("email_event_browser") }}' + - name: id + description: The ID of the event. + - name: ip_address + description: '{{ doc("email_event_ip_address") }}' + - name: location + description: '{{ doc("email_event_location") }}' + - name: user_agent + description: '{{ doc("email_event_user_agent") }}' + + - name: email_event_sent + description: Each record represents a SENT email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_sent_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: bcc + description: The 'cc' field of the email message. + - name: cc + description: The 'bcc' field of the email message. + - name: from + description: The 'from' field of the email message. + - name: id + description: The ID of the event. + - name: reply_to + description: The 'reply-to' field of the email message. + - name: subject + description: The subject line of the email message. + + - name: email_event_spam_report + description: Each record represents a SPAM_REPORT email event. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_spam_report_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: id + description: The ID of the event. + - name: ip_address + description: '{{ doc("email_event_ip_address") }}' + - name: user_agent + description: '{{ doc("email_event_user_agent") }}' + + - name: email_event_status_change + description: Each record represents a STATUS_CHANGE email event in Hubspot. + config: + enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_status_change_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: bounced + description: | + A HubSpot employee explicitly initiated the status change to block messages to the recipient. + (Note this usage has been deprecated in favor of dropping messages with a 'dropReason' of BLOCKED_ADDRESS.) + - name: id + description: The ID of the event. + - name: portal_subscription_status + description: | + The recipient's portal subscription status. + Note that if this is 'UNSUBSCRIBED', the property 'subscriptions' is not necessarily an empty array, nor are all + subscriptions contained in it necessarily going to have their statuses set to 'UNSUBSCRIBED'.) + - name: requested_by + description: The email address of the person requesting the change on behalf of the recipient. If not applicable, this property is omitted. + - name: source + description: The source of the subscription change. + - name: subscriptions + description: | + An array of JSON objects representing the status of subscriptions for the recipient. + Each JSON subscription object is comprised of the properties: 'id', 'status'. + + - name: email_subscription + description: Each record represents an email subscription in Hubspot. + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: active + description: Whether the subscription is active. + - name: description + description: The description of the subscription. + - name: id + description: The ID of the email subscription. + - name: name + description: The name of the email subscription. + - name: portal_id + description: '{{ doc("portal_id") }}' + + - name: email_subscription_change + description: Each record represents a change to an email subscription in Hubspot. + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: caused_by_event_id + description: The ID of the event that caused the subscription change. + - name: change + description: The change which occurred. This enumeration is specific to the 'changeType'; see below for the possible values. + - name: change_type + description: The type of change which occurred. + - name: email_subscription_id + description: The ID of the related email subscription. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: recipient + description: The email address of the related contact. + - name: source + description: The source of the subscription change. + - name: timestamp + description: The timestamp when this change occurred. If 'causedByEvent' is present, this will be absent. + + - name: engagement + description: Each record represents an engagement + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: active + description: > + Whether the engagement is currently being shown in the UI. + + PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release. + - name: created_at + description: > + A timestamp representing when the engagement was created. + + PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release. + - name: id + description: The ID of the engagement. + - name: owner_id + description: > + The ID of the engagement's owner. + + PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: timestamp + description: > + A timestamp in representing the time that the engagement should appear in the timeline. + + PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release. + - name: type + description: One of NOTE, EMAIL, TASK, MEETING, or CALL, the type of the engagement. + + - name: engagement_call + description: Each record represents a CALL engagement event. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_call_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: engagement_id + description: The ID of the engagement. + - name: _fivetran_deleted + description: Boolean to mark rows that were deleted in the source database. + - name: property_hs_createdate + description: > + This field marks the call's time of creation and determines where the call sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: timestamp + description: > + This field marks the call's time of occurrence and determines where the call sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_owner_id + description: > + The ID of the owner associated with the call. This field determines the user listed as the call creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_team_id + description: > + The ID of the team associated with the call. This field determines the team listed as the call creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + + - name: engagement_company + description: Each record represents a 'link' between a company and an engagement. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_company_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: company_id + description: The ID of the related company. + - name: engagement_id + description: The ID of the related engagement. + + - name: engagement_contact + description: Each record represents a 'link' between a contact and an engagement. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_contact_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: contact_id + description: The ID of the related contact. + - name: engagement_id + description: The ID of the related engagement. + + - name: engagement_deal + description: Each record represents a 'link' between a deal and an engagement. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_deal_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: deal_id + description: The ID of the related contact. + - name: engagement_id + description: The ID of the related engagement. + + - name: engagement_email + description: Each record represents an EMAIL engagement event. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_email_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: engagement_id + description: The ID of the engagement. + - name: property_hs_createdate + description: > + This field marks the email's time of creation and determines where the email sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: timestamp + description: > + This field marks the email's time of occurrence and determines where the email sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_owner_id + description: > + The ID of the owner associated with the email. This field determines the user listed as the email creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_team_id + description: > + The ID of the team associated with the email. This field determines the team listed as the email creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + + - name: engagement_email_cc + description: Each record represents a CC'd email address as part of an EMAIL engagement. + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: email + description: The email address of the recipient. + - name: engagement_id + description: The ID of the related engagement. + - name: first_name + description: The first name of the recipient. + - name: last_name + description: The last name of the recipient. + + - name: engagement_email_to + description: Each record represents a TO email address as part of an EMAIL engagement. + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: email + description: The email address of the recipient. + - name: engagement_id + description: The ID of the related engagement. + - name: first_name + description: The first name of the recipient. + - name: last_name + description: The last name of the recipient. + + - name: engagement_meeting + description: Each record represents a MEETING engagement event. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_meeting_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: engagement_id + description: The ID of the engagement. + - name: property_hs_createdate + description: > + This field marks the meeting's time of creation and determines where the meeting sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: timestamp + description: > + This field marks the meeting's time of occurrence and determines where the meeting sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_owner_id + description: > + The ID of the owner associated with the meeting. This field determines the user listed as the meeting creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_team_id + description: > + The ID of the team associated with the meeting. This field determines the team listed as the meeting creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + + - name: engagement_note + description: Each record represents a NOTE engagement event. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_note_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: body + description: The body of the note. The body has a limit of 65536 characters. + - name: engagement_id + description: The ID of the engagement. + - name: property_hs_createdate + description: > + This field marks the note's time of creation and determines where the note sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: timestamp + description: > + This field marks the note's time of occurrence and determines where the note sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_owner_id + description: > + The ID of the owner associated with the note. This field determines the user listed as the note creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_team_id + description: > + The ID of the team associated with the note. This field determines the team listed as the note creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + + - name: engagement_task + description: Each record represents a TASK engagement event. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_task_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: engagement_id + description: The ID of the engagement. + - name: property_hs_createdate + description: > + This field marks the task's time of creation and determines where the task sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: timestamp + description: > + This field marks the task's time of occurrence and determines where the task sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_owner_id + description: > + The ID of the owner associated with the task. This field determines the user listed as the task creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. + - name: property_hubspot_team_id + description: > + The ID of the team associated with the task. This field determines the team listed as the task creator on the record timeline. + + PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. + + - name: form + description: Each record represents a Hubspot form. + columns: + - name: _fivetran_deleted + description: '{{ doc("_fivetran_deleted") }}' + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: created_at + description: A timestamp for when the form was created. + - name: css_class + description: The CSS classes assigned to the form. + - name: follow_up_id + description: This field is no longer used. + - name: guid + description: The internal ID of the form. + - name: lead_nurturing_campaign_id + description: TBD + - name: method + description: This field is no longer used. + - name: name + description: The name of the form. + + - name: notify_recipients + description: | + A comma-separated list of user IDs that should receive submission notifications. + Email addresses will be returned for individuals who aren't users. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: redirect + description: The URL that the visitor will be redirected to after filling out the form. + - name: submit_text + description: The text used for the submit button. + - name: updated_at + description: A timestamp for when the form was last updated. + + - name: owner + description: Each record represents an owner/user in Hubspot. + config: + enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_owner_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: created_at + description: A timestamp for when the owner was created. + - name: email + description: The email address of the owner. + - name: first_name + description: The first name of the owner. + - name: last_name + description: The last name of the owner. + - name: owner_id + description: The ID of the owner. + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: type + description: The type of owner. + - name: updated_at + + - name: property + description: Each record represents a property. + config: + enabled: "{{ var('hubspot_property_enabled', true) }}" + columns: + - name: _fivetran_id + description: Fivetran generated id. Joins to `property_id` in the `property_option` table. + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: calculated + description: Indicates if the property is calculated by a HubSpot process + - name: created_at + description: Timestamp representing when the property was created. + - name: description + description: A description of the property. + - name: field_type + description: One of textarea, text, date, file, number, select, radio, checkbox, or booleancheckbox. + - name: group_name + description: The name of the property group that the property belongs to. + - name: hubspot_defined + description: This will be true for default properties that are built into HubSpot. + - name: hubspot_object + description: If this property is related to other objects, the object will be listed here. + - name: label + description: A human readable label for the property. + - name: name + description: The internal name of the property. + - name: show_currency_symbol + description: If true, the property will display the currency symbol set in the portal settings. + - name: type + description: One of string, number, date, datetime, or enumeration. + - name: updated_at + description: Timestamp representing when the property was last updated. + + - name: property_option + description: Each record represents an option for the related property. + config: + enabled: "{{ var('hubspot_property_enabled', true) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: label + description: The label of the option displayed inside the HubSpot app. + - name: property_id + description: The ID of the related property. + - name: display_order + description: Order of options displayed in Hubspot, starting with the lowest positive integer value. Values of -1 will cause the option to be displayed after any positive values. + - name: hidden + description: Boolean if the option will be displayed in HubSpot + - name: value + description: The internal value of the option. + + - name: ticket_company + description: Each record represents a 'link' between a ticket and company. + config: + enabled: "{{ var('hubspot_service_enabled', false) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: ticket_id + description: The ID of the related ticket. + - name: company_id + description: The ID of the related company. + + - name: ticket_contact + description: Each record represents a 'link' between a ticket and a contact. + config: + enabled: "{{ var('hubspot_service_enabled', false) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: ticket_id + description: The ID of the related ticket. + - name: contact_id + description: The ID of the related contact. + + - name: ticket_deal + description: Each record represents a 'link' between a ticket and a deal. + config: + enabled: "{{ var('hubspot_service_enabled', false) and var('hubspot_ticket_deal_enabled', false) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: ticket_id + description: The ID of the related ticket. + - name: deal_id + description: The ID of the related deal. + + - name: ticket_engagement + description: Each record represents a 'link' between a ticket and an engagement. + config: + enabled: "{{ var('hubspot_service_enabled', false) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: ticket_id + description: The ID of the related ticket. + - name: engagement_id + description: The ID of the related deal. + + - name: ticket_pipeline_stage + description: Each record represents a ticket pipeline stage. + config: + enabled: "{{ var('hubspot_service_enabled', false) }}" + columns: + - name: _fivetran_deleted + description: '{{ doc("_fivetran_deleted") }}' + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: active + description: Boolean indicating whether the pipeline stage is active. + - name: display_order + description: Used to determine the order in which the stages appear when viewed in HubSpot. + - name: is_closed + description: Boolean indicating if the pipeline stage is closed. + - name: label + description: The human-readable label for the stage. The label is used when showing the stage in HubSpot. + - name: pipeline_id + description: The ID of the pipeline. + - name: stage_id + description: The ID of the pipeline stage. + - name: ticket_state + description: State of the ticket. + - name: created_at + description: A timestamp representing when the record was created. + - name: updated_at + description: A timestamp representing when the record was updated. + + - name: ticket_pipeline + description: Each record represents a ticket pipeline. + config: + enabled: "{{ var('hubspot_service_enabled', false) }}" + columns: + - name: _fivetran_deleted + description: '{{ doc("_fivetran_deleted") }}' + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: active + description: Boolean indicating whether the pipeline is active. + - name: display_order + description: Used to determine the order in which the stages appear when viewed in HubSpot. + - name: label + description: The human-readable label for the stage. The label is used when showing the stage in HubSpot. + - name: object_type_id + description: Reference to the object type. + - name: pipeline_id + description: Reference to the pipeline. + - name: created_at + description: A timestamp representing when the record was created. + - name: updated_at + description: A timestamp representing when the record was updated. + + - name: ticket_property_history + description: Each record represents a change to ticket record in Hubspot. + config: + enabled: "{{ var('hubspot_service_enabled', false) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: source + description: '{{ doc("history_source") }}' + - name: source_id + description: '{{ doc("history_source_id") }}' + - name: timestamp_instant + description: '{{ doc("history_timestamp") }}' + - name: ticket_id + description: The ID of the related ticket record. + - name: name + description: '{{ doc("history_name") }}' + - name: value + description: '{{ doc("history_value") }}' + + - name: ticket + description: Each record represents a ticket in Hubspot. + config: + enabled: "{{ var('hubspot_service_enabled', false) }}" + columns: + - name: _fivetran_synced + description: '{{ doc("_fivetran_synced") }}' + - name: id + description: ID of the ticket. + - name: is_deleted + description: Whether the record was deleted (v2 endpoint). + - name: _fivetran_deleted + description: Whether the record was deleted (v3 endpoint). + - name: portal_id + description: '{{ doc("portal_id") }}' + - name: property_closed_date + description: The date the ticket was closed. + - name: property_createdate + description: The date the ticket was created. + - name: property_first_agent_reply_date + description: the date for the first agent reply on the ticket. + - name: property_hs_pipeline + description: The ID of the ticket's pipeline. + - name: property_hs_pipeline_stage + description: The ID of the ticket's pipeline stage. + - name: property_hs_ticket_priority + description: The priority of the ticket. + - name: property_hs_ticket_category + description: The category of the ticket. + - name: property_hubspot_owner_id + description: The ID of the deal's owner. + - name: property_subject + description: Short summary of ticket. + - name: property_content + description: Text in body of the ticket. +``` +

+ +2. Set the `has_defined_sources` variable (scoped to the `hubspot_source` package) to true, like such: +```yml +# dbt_project.yml +vars: + hubspot_source: + has_defined_sources: true +``` + ## Step 4: Disable models for non-existent sources When setting up your Hubspot connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Hubspot or have actively decided to not sync some tables. Therefore we have added enable/disable configs in the `src.yml` to allow you to disable certain sources not present. Downstream models are automatically disabled as well. In order to disable the relevant functionality in the package, you will need to add the relevant variables in your root `dbt_project.yml`. By default, all variables are assumed to be `true` (with exception of `hubspot_service_enabled`, `hubspot_ticket_deal_enabled`, and `hubspot_contact_merge_audit_enabled`). You only need to add variables for the tables different from default: @@ -215,6 +1509,8 @@ vars: hubspot__identifier: your_table_name ``` +This solution is most relevant to users running the package on a single connector. + ## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™ Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Learn how to set up your project for orchestration through Fivetran in our [Transformations for dbt Core setup guides](https://fivetran.com/docs/transformations/dbt#setupguide). From e35e9eba2c32e44a124cccc81a84605720ea8a7b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 8 Jan 2024 13:24:07 -0300 Subject: [PATCH 03/16] docs and buildkite --- README.md | 3 +++ integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 55b0d425..81bec480 100644 --- a/README.md +++ b/README.md @@ -1349,6 +1349,9 @@ vars: ``` ## Step 4: Disable models for non-existent sources + +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Hubspot schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior_ + When setting up your Hubspot connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Hubspot or have actively decided to not sync some tables. Therefore we have added enable/disable configs in the `src.yml` to allow you to disable certain sources not present. Downstream models are automatically disabled as well. In order to disable the relevant functionality in the package, you will need to add the relevant variables in your root `dbt_project.yml`. By default, all variables are assumed to be `true` (with exception of `hubspot_service_enabled`, `hubspot_ticket_deal_enabled`, and `hubspot_contact_merge_audit_enabled`). You only need to add variables for the tables different from default: ```yml diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index b6dc00b3..1f906a05 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: hubspot_source_integration_tests_999 + schema: hubspot_source_integration_tests_001 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: hubspot_source_integration_tests_999 + schema: hubspot_source_integration_tests_001 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: hubspot_source_integration_tests_999 + schema: hubspot_source_integration_tests_001 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: hubspot_source_integration_tests_999 + schema: hubspot_source_integration_tests_001 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: hubspot_source_integration_tests_999 + schema: hubspot_source_integration_tests_001 threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index e3833b20..43bcdb9a 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,7 +6,7 @@ models: hubspot_source: +schema: vars: - hubspot_schema: hubspot_source_integration_tests_999 + hubspot_schema: hubspot_source_integration_tests_001 hubspot_source: hubspot_service_enabled: true # hubspot_sales_enabled: true # enable when generating docs From 9caf6adf2afc9575f30739b63a2d637f491860dd Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:06:14 -0300 Subject: [PATCH 04/16] doc tweak --- models/docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/docs.md b/models/docs.md index 720b668a..d84fa2c7 100644 --- a/models/docs.md +++ b/models/docs.md @@ -3,7 +3,7 @@ Timestamp of when Fivetran synced a record. {% enddocs %} {% docs source_relation %} -The schema or database this record came from if you are unioning multiple Zendesk connectors together in this package. If you are running the package on a single connector, this will be its schema name. +The schema or database this record came from if you are unioning multiple connectors together in this package. If you are running the package on a single connector, this will be its schema name. {% enddocs %} {% docs _fivetran_deleted %} From 93ca4f33f100e77e0646c7b1d95a0cc4a28aa086 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 8 Jan 2024 17:15:55 -0300 Subject: [PATCH 05/16] add missing source_relation --- models/stg_hubspot__contact_property_history.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/stg_hubspot__contact_property_history.sql b/models/stg_hubspot__contact_property_history.sql index b8ea5294..22a6fdb5 100644 --- a/models/stg_hubspot__contact_property_history.sql +++ b/models/stg_hubspot__contact_property_history.sql @@ -33,7 +33,9 @@ with base as ( source as change_source, source_id as change_source_id, cast(change_timestamp as {{ dbt.type_timestamp() }}) as change_timestamp, -- source field name = timestamp ; alias declared in macros/get_contact_property_history_columns.sql - value as new_value + value as new_value, + source_relation + from macro ) From ea142f458d7ac232ad4674d102b5f8b56f8e0b69 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 9 Jan 2024 12:19:20 -0300 Subject: [PATCH 06/16] doc tweaks --- CHANGELOG.md | 4 +++- README.md | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e5d1f61..b66b6ede 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # dbt_hubspot_source v0.15.0 +[PR #123](https://github.com/fivetran/dbt_hubspot_source/pull/123) includes the following updates: -Unioning coming soon... +## 🎉 Feature Update 🎉 +- This release supports running the package on multiple Hubspot sources at once! See the [README](https://github.com/fivetran/dbt_hubspot_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature. # dbt_hubspot_source v0.14.0 [PR #122](https://github.com/fivetran/dbt_hubspot_source/pull/122) includes the following updates: diff --git a/README.md b/README.md index 81bec480..2bfd2d47 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ To properly incorporate all of your Hubspot connectors into your project's DAG:
Expand for source configuration template

-> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_hubspot_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_hubspot_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them here, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition. ```yml sources: @@ -1350,7 +1350,7 @@ vars: ## Step 4: Disable models for non-existent sources -> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Hubspot schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior_ +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Hubspot schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior._ When setting up your Hubspot connection in Fivetran, it is possible that not every table this package expects will be synced. This can occur because you either don't use that functionality in Hubspot or have actively decided to not sync some tables. Therefore we have added enable/disable configs in the `src.yml` to allow you to disable certain sources not present. Downstream models are automatically disabled as well. In order to disable the relevant functionality in the package, you will need to add the relevant variables in your root `dbt_project.yml`. By default, all variables are assumed to be `true` (with exception of `hubspot_service_enabled`, `hubspot_ticket_deal_enabled`, and `hubspot_contact_merge_audit_enabled`). You only need to add variables for the tables different from default: From 4e390b5a71facab3cb1f6dafafb539dcaac2752d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 9 Jan 2024 14:24:48 -0300 Subject: [PATCH 07/16] add source relation to add_property_labels macro joins --- macros/add_property_labels.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/macros/add_property_labels.sql b/macros/add_property_labels.sql index 08741ee1..dd0c8d16 100644 --- a/macros/add_property_labels.sql +++ b/macros/add_property_labels.sql @@ -22,16 +22,19 @@ select {{ cte_name }}.* left join -- create subset of property and property_options for property in question (select property_option.property_option_value, - property_option.property_option_label + property_option.property_option_label, + property_option.source_relation from {{ ref('stg_hubspot__property_option') }} as property_option join {{ ref('stg_hubspot__property') }} as property on property_option.property_id = property._fivetran_id + and property_option.source_relation = property.source_relation where property.property_name = '{{ col.name.replace('property_', '') }}' and property.hubspot_object = '{{ source_name }}' ) as {{ col.name }}_option on cast({{ cte_name }}.{{ col_alias }} as {{ dbt.type_string() }}) = cast({{ col.name }}_option.property_option_value as {{ dbt.type_string() }}) + and {{ cte_name }}.source_relation = {{ col.name }}_option.source_relation {% endif -%} {%- endfor %} From 0644b8244b6cac8097f20f7c575f37e06727e079 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:33:19 -0300 Subject: [PATCH 08/16] make sure _dbt_source_relation doesn't slip in when all fields are passed through --- models/stg_hubspot__company.sql | 2 +- models/stg_hubspot__contact.sql | 2 +- models/stg_hubspot__deal.sql | 2 +- models/stg_hubspot__ticket.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/models/stg_hubspot__company.sql b/models/stg_hubspot__company.sql index bb541fd8..f311cf03 100644 --- a/models/stg_hubspot__company.sql +++ b/models/stg_hubspot__company.sql @@ -48,7 +48,7 @@ with base as ( ,{{ fivetran_utils.remove_prefix_from_columns( columns=adapter.get_columns_in_relation(ref('stg_hubspot__company_tmp')), - prefix='property_', exclude=get_macro_columns(get_company_columns())) + prefix='property_', exclude=(get_macro_columns(get_company_columns()) + ['_dbt_source_relation'])) }} {% endif %} from base diff --git a/models/stg_hubspot__contact.sql b/models/stg_hubspot__contact.sql index e882764d..e341d204 100644 --- a/models/stg_hubspot__contact.sql +++ b/models/stg_hubspot__contact.sql @@ -48,7 +48,7 @@ with base as ( ,{{ fivetran_utils.remove_prefix_from_columns( columns=adapter.get_columns_in_relation(ref('stg_hubspot__contact_tmp')), - prefix='property_', exclude=get_macro_columns(get_contact_columns())) + prefix='property_', exclude=(get_macro_columns(get_contact_columns()) + ['_dbt_source_relation'])) }} {% endif %} from base diff --git a/models/stg_hubspot__deal.sql b/models/stg_hubspot__deal.sql index f9742ff8..4940f3fe 100644 --- a/models/stg_hubspot__deal.sql +++ b/models/stg_hubspot__deal.sql @@ -48,7 +48,7 @@ with base as ( ,{{ fivetran_utils.remove_prefix_from_columns( columns=adapter.get_columns_in_relation(ref('stg_hubspot__deal_tmp')), - prefix='property_',exclude=get_macro_columns(get_deal_columns())) + prefix='property_',exclude=(get_macro_columns(get_deal_columns()) + ['_dbt_source_relation'])) }} {% endif %} from base diff --git a/models/stg_hubspot__ticket.sql b/models/stg_hubspot__ticket.sql index 87f2efeb..28da8af9 100644 --- a/models/stg_hubspot__ticket.sql +++ b/models/stg_hubspot__ticket.sql @@ -48,7 +48,7 @@ with base as ( {{ remove_duplicate_and_prefix_from_columns( columns=adapter.get_columns_in_relation(ref('stg_hubspot__ticket_tmp')), - prefix='property_', exclude=get_macro_columns(get_ticket_columns())) + prefix='property_', exclude=(get_macro_columns(get_ticket_columns()) + ['_dbt_source_relation'])) }} {% endif %} from base From b6900a59fc5c62e3e0736136718de5c425a60417 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:51:49 -0300 Subject: [PATCH 09/16] exclude _dbt_source_relation from all_passthrough_column_check macro --- macros/all_passthrough_column_check.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/all_passthrough_column_check.sql b/macros/all_passthrough_column_check.sql index 08eda12e..071f6817 100644 --- a/macros/all_passthrough_column_check.sql +++ b/macros/all_passthrough_column_check.sql @@ -2,7 +2,7 @@ {% set available_passthrough_columns = fivetran_utils.remove_prefix_from_columns( columns=adapter.get_columns_in_relation(ref(relation)), - prefix='property_', exclude=get_macro_columns(get_columns)) + prefix='property_', exclude=(get_macro_columns(get_columns) + ['_dbt_source_relation'])) %} {{ return(available_passthrough_columns|length) }} From 1f7cda89f1ab3c94e0f7c7ac6f9534401f5898ed Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 10 Jan 2024 13:07:21 -0300 Subject: [PATCH 10/16] disable source dynamically --- models/src_hubspot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/models/src_hubspot.yml b/models/src_hubspot.yml index b3206e30..c6029ab6 100644 --- a/models/src_hubspot.yml +++ b/models/src_hubspot.yml @@ -6,6 +6,8 @@ sources: database: "{% if target.type != 'spark'%}{{ var('hubspot_database', target.database) }}{% endif %}" loader: Fivetran loaded_at_field: _fivetran_synced + config: + enabled: "{{ var('hubspot_union_schemas', []) == [] and var('hubspot_union_databases', []) == [] }}" tables: - name: calendar_event identifier: "{{ var('hubspot_calendar_event_identifier', 'calendar_event')}}" From c8c51dad4d4ff96024db0f9a2f601109e0dce380 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 10 Jan 2024 16:02:35 -0300 Subject: [PATCH 11/16] add github stuff --- .../maintainer_pull_request_template.md | 43 +++++-------------- .github/workflows/auto-release.yml | 13 ++++++ .github/workflows/check_docs.yml | 35 +++++++++++++++ CHANGELOG.md | 5 +++ 4 files changed, 64 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/auto-release.yml create mode 100644 .github/workflows/check_docs.yml diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index 768ac3f4..1e22b099 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -4,48 +4,27 @@ **This PR will result in the following new package version:** -**Please detail what change(s) this PR introduces and any additional information that should be known during the review of this PR:** +**Please provide the finalized CHANGELOG entry which details the relevant changes included in this PR:** + ## PR Checklist ### Basic Validation Please acknowledge that you have successfully performed the following commands locally: -- [ ] dbt compile -- [ ] dbt run –full-refresh -- [ ] dbt run -- [ ] dbt test -- [ ] dbt run –vars (if applicable) +- [ ] dbt run –full-refresh && dbt test +- [ ] dbt run (if incremental models are present) && dbt test Before marking this PR as "ready for review" the following have been applied: -- [ ] The appropriate issue has been linked and tagged -- [ ] You are assigned to the corresponding issue and this PR +- [ ] The appropriate issue has been linked, tagged, and properly assigned +- [ ] All necessary documentation and version upgrades have been applied + +- [ ] docs were regenerated (unless this PR does not include any code or yml updates) - [ ] BuildKite integration tests are passing +- [ ] Detailed validation steps have been provided below ### Detailed Validation -Please acknowledge that the following validation checks have been performed prior to marking this PR as "ready for review": -- [ ] You have validated these changes and assure this PR will address the respective Issue/Feature. -- [ ] You are reasonably confident these changes will not impact any other components of this package or any dependent packages. -- [ ] You have provided details below around the validation steps performed to gain confidence in these changes. +Please share any and all of your validation steps: -### Standard Updates -Please acknowledge that your PR contains the following standard updates: -- Package versioning has been appropriately indexed in the following locations: - - [ ] indexed within dbt_project.yml - - [ ] indexed within integration_tests/dbt_project.yml -- [ ] CHANGELOG has individual entries for each respective change in this PR - -- [ ] README updates have been applied (if applicable) - -- [ ] DECISIONLOG updates have been updated (if applicable) -- [ ] Appropriate yml documentation has been added (if applicable) - -### dbt Docs -Please acknowledge that after the above were all completed the below were applied to your branch: -- [ ] docs were regenerated (unless this PR does not include any code or yml updates) - ### If you had to summarize this PR in an emoji, which would it be? -:dancer: +:dancer: \ No newline at end of file diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 00000000..8ed58532 --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,13 @@ +name: 'auto release' +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + call-workflow-passing-data: + if: github.event.pull_request.merged + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/check_docs.yml b/.github/workflows/check_docs.yml new file mode 100644 index 00000000..c1e2a89a --- /dev/null +++ b/.github/workflows/check_docs.yml @@ -0,0 +1,35 @@ +name: 'check docs' +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + changed-files: + runs-on: ubuntu-latest + name: test changed-files + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v41.0.0 + with: + files: docs/** + + - name: Check to see if docs folder hasn't changed + if: steps.changed-files.outputs.any_changed == 'false' + run: | + echo "Docs have not been regenerated." + exit 1 + + - name: Check if docs folder has changed + if: steps.changed-files.outputs.any_changed == 'true' + run: | + echo "Docs have been regenerated!" + exit 0 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index b66b6ede..819745f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ ## 🎉 Feature Update 🎉 - This release supports running the package on multiple Hubspot sources at once! See the [README](https://github.com/fivetran/dbt_hubspot_source?tab=readme-ov-file#step-3-define-database-and-schema-variables) for details on how to leverage this feature. +## 🛠️ Under the Hood 🛠️ +- Included auto-releaser GitHub Actions workflow to automate future releases. +- Included Github Actions workflow to check for docs updates. +- Updated the maintainer PR template to resemble the most up to date format. + # dbt_hubspot_source v0.14.0 [PR #122](https://github.com/fivetran/dbt_hubspot_source/pull/122) includes the following updates: From 7b2a5ca8391a4410e91fc67885f35885681aef5b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:45:26 -0400 Subject: [PATCH 12/16] polish readme --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2bfd2d47..92c141fd 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ packages: ``` ## Step 3: Define database and schema variables -### Option 1: Single connector +### Option 1: Single connector 💃 By default, this package runs using your destination and the `hubspot` schema. If this is not where your HubSpot data is (for example, if your HubSpot schema is named `hubspot_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -58,7 +58,7 @@ vars: ``` > **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. -### Option 2: Union multiple connectors +### Option 2: Union multiple connectors 👯 If you have multiple Hubspot connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `hubspot_union_schemas` OR `hubspot_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: ```yml @@ -1412,6 +1412,7 @@ vars: If you are not using a source table that involves freshness tests, please be aware that the feature to disable freshness was only introduced in dbt-core 1.1.0. Therefore ensure the dbt version you're using is v1.1.0 or greater for this config to work. ## (Optional) Step 5: Additional configurations +

Expand/collapse configurations ### Adding passthrough columns This package includes all source columns defined in the macros folder. Models by default only bring in a few fields for the `company`, `contact`, `deal`, and `ticket` tables. You can add more columns using our pass-through column variables. These variables allow for the pass-through fields to be aliased (`alias`) and casted (`transform_sql`) if desired, but not required. Datatype casting is configured via a sql snippet within the `transform_sql` key. You may add the desired sql while omitting the `as field_name` at the end and your custom pass-though fields will be casted accordingly. Use the below format for declaring the respective pass-through variables within your root `dbt_project.yml`. @@ -1503,7 +1504,7 @@ models: +schema: my_new_schema_name # leave blank for just the target_schema ``` -### Change the source table references +### Change the source table references (only if using a single connector) If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: > IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_hubspot_source/blob/main/dbt_project.yml) variable declarations to see the expected names. @@ -1512,7 +1513,7 @@ vars: hubspot__identifier: your_table_name ``` -This solution is most relevant to users running the package on a single connector. +
## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™ From 7a37bb8376f7e41cb15a45848b0e3a3df0d1ae8a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:25:14 -0500 Subject: [PATCH 13/16] joe feedback --- .github/workflows/check_docs.yml | 35 -------------------------------- CHANGELOG.md | 1 - 2 files changed, 36 deletions(-) delete mode 100644 .github/workflows/check_docs.yml diff --git a/.github/workflows/check_docs.yml b/.github/workflows/check_docs.yml deleted file mode 100644 index c1e2a89a..00000000 --- a/.github/workflows/check_docs.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: 'check docs' -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - changed-files: - runs-on: ubuntu-latest - name: test changed-files - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v41.0.0 - with: - files: docs/** - - - name: Check to see if docs folder hasn't changed - if: steps.changed-files.outputs.any_changed == 'false' - run: | - echo "Docs have not been regenerated." - exit 1 - - - name: Check if docs folder has changed - if: steps.changed-files.outputs.any_changed == 'true' - run: | - echo "Docs have been regenerated!" - exit 0 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 819745f1..f2465464 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,6 @@ ## 🛠️ Under the Hood 🛠️ - Included auto-releaser GitHub Actions workflow to automate future releases. -- Included Github Actions workflow to check for docs updates. - Updated the maintainer PR template to resemble the most up to date format. # dbt_hubspot_source v0.14.0 From 363527b8b7879fb30d7860f5b6f085bbbe57fb7a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:40:58 -0500 Subject: [PATCH 14/16] joe feedback --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 92c141fd..aaef1472 100644 --- a/README.md +++ b/README.md @@ -1408,9 +1408,6 @@ vars: hubspot_ticket_deal_enabled: true ``` -### Dbt-core Version Requirement for disabling freshness tests -If you are not using a source table that involves freshness tests, please be aware that the feature to disable freshness was only introduced in dbt-core 1.1.0. Therefore ensure the dbt version you're using is v1.1.0 or greater for this config to work. - ## (Optional) Step 5: Additional configurations
Expand/collapse configurations From ad0c752c347777a59daa49c336c65984f0db5331 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 17:19:35 -0500 Subject: [PATCH 15/16] switch source defining instructions --- README.md | 1268 +---------------------------------------------------- 1 file changed, 8 insertions(+), 1260 deletions(-) diff --git a/README.md b/README.md index aaef1472..83c2cb6c 100644 --- a/README.md +++ b/README.md @@ -73,1272 +73,20 @@ vars: By default, this package defines one single-connector source, called `hubspot`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Hubspot sources, though the package will run successfully. To properly incorporate all of your Hubspot connectors into your project's DAG: -1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. - -
Expand for source configuration template

- -> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_hubspot_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them here, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definition. +1. Define each of your sources in a `.yml` file in your project. Utilize the following template for the `source`-level configurations, and, **most importantly**, copy and paste the table and column-level definitions from the package's `src_hubspot.yml` [file](https://github.com/fivetran/dbt_hubspot_source/blob/main/models/src_hubspot.yml#L9-L1313). ```yml +# a .yml file in your root project sources: - - name: - schema: - database: + - name: # ex: hubspot_usa + schema: # one of var('hubspot_union_schemas') if unioning schemas, otherwise just 'hubspot' + database: # one of var('hubspot_union_databases') if unioning databases, otherwise whatever DB your hubspot schemas all live in loader: Fivetran loaded_at_field: _fivetran_synced - tables: &hubspot_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ - - name: calendar_event - description: Each record represents a calendar event. - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: avatar_url - description: URL of image associated with social media event. - - name: campaign_guid - description: Value of campaign GUID associated with Task. - - name: category - description: | - Type of event. If the event type is PUBLISHING_TASK, it is one of BLOG_POST, EMAIL, LANDING_PAGE, CUSTOM. - If event type is SOCIAL, it is one of twitter, facebook, linkedin, googlepluspages. - If event type is CONTENT, it is one of email, recurring-email, blog-post, landing-page, legacy-page, site-page. - - name: category_id - description: | - For event type of PUBLISHING_TASK, a numeric value corresponding to the type of task; one of 3 (BLOG_POST), 2 (EMAIL), 1 (LANDING_PAGE), 0 (CUSTOM). - For event type of SOCIAL, this is 0. - If event type is CONTENT, it is one of 2 (email, recurring-email), 3 (blog-post), 1 (landing-page), 5 (legacy-page), 4 (site-page). - - name: content_group_id - description: | - The ID of the content group (aka blog) that the associated Blog Post belongs to, if any. - Otherwise null. Only populated for single task GETs and for Blog Post Tasks. - - name: content_id - description: ID value of the COS content object associated with the event, null for social or if nothing associated. - - name: created_by - description: HubSpot ID of the user that the event was created by. - - name: description - description: Description of Event. - - name: event_date - description: If task, When the task is set to be due, otherwise when the event is/ was scheduled for; in milliseconds since the epoch. - - name: event_type - description: Type of calendar event; for tasks this is PUBLISHING_TASK, for COS Items, this is CONTENT, for social media events, this is SOCIAL - - name: id - description: The unique ID of the task. - - name: is_recurring - description: Whether the event is recurring. - - name: name - description: Name of Event. - - name: owner_id - description: TASK - HubSpot ID of the user that the task is assigned to. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: preview_key - description: Preview key of content object; used for showing previews of unpublished items. - - name: social_display_name - description: Social media full name associate with event. - - name: social_username - description: Social media user name associated with event. - - name: state - description: For type publishing task, value of TODO or DONE, for others, a value of SCHEDULED, PUBLISHED, PUBLISHED_OR_SCHEDULED. - - name: topic_ids - description: The list of IDs of topics associated with the associated Blog Post, if any. Otherwise null. Only populated for single task GETs and for Blog Post Tasks. - - name: url - description: Public URL of content item. - - - name: company - description: Each record represents a company in Hubspot. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_company_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: id - description: The ID of the company. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: is_deleted - description: '{{ doc("is_deleted") }}' - - name: property_name - description: The name of the company. - - name: property_description - description: A short statement about the company's mission and goals. - - name: property_createdate - description: The date the company was added to your account. - - name: property_industry - description: The type of business the company performs. - - name: property_address - description: The street address of the company. - - name: property_address_2 - description: Additional address information for the company. - - name: property_city - description: The city where the company is located. - - name: property_state - description: The state where the company is located. - - name: property_country - description: The country where the company is located. - - name: property_annualrevenue - description: The actual or estimated annual revenue of the company. - - - name: company_property_history - description: Each record represents a change to company record in Hubspot. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_company_enabled', true) and var('hubspot_company_property_history_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: company_id - description: The ID of the related company record. - - name: name - description: '{{ doc("history_name") }}' - - name: source - description: '{{ doc("history_source") }}' - - name: source_id - description: '{{ doc("history_source_id") }}' - - name: timestamp - description: '{{ doc("history_timestamp") }}' - - name: value - description: '{{ doc("history_value") }}' - - - name: contact_merge_audit - description: > - DEPRECATED FOR NON-BIGQUERY USERS (will be deprecated on BigQuery as well). Each record contains a contact merge event and the contacts effected by the merge. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_merge_audit_enabled', false) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: canonical_vid - description: The contact ID of the contact which the vid_to_merge contact was merged into. - - name: contact_id - description: The ID of the contact. - - name: entity_id - description: The ID of the related entity. - - name: first_name - description: The contact's first name. - - name: last_name - description: The contact's last name. - - name: num_properties_moved - description: The number of properties which were removed from the merged contact. - - name: timestamp - description: Timestamp of when the contacts were merged. - - name: user_id - description: The ID of the user. - - name: vid_to_merge - description: The ID of the contact which was merged. - - - name: contact - freshness: - warn_after: {count: 84, period: hour} - error_after: {count: 168, period: hour} - description: Each record represents a contact in Hubspot. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_enabled', true) }}" - columns: - - name: _fivetran_deleted - description: '{{ doc("_fivetran_deleted") }}' - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: id - description: The ID of the contact. - - name: property_email_1 - description: The email address of the contact. - - name: property_company - description: The name of the contact's company. - - name: property_firstname - description: The contact's first name. - - name: property_lastname - description: The contact's last name. - - name: property_email - description: The contact's email. - - name: property_createdate - description: The date that the contact was created in your HubSpot account. - - name: property_jobtitle - description: The contact's job title. - - name: property_annualrevenue - description: The contact's annual company revenue. - - name: property_hs_calculated_merged_vids - description: > - List of mappings representing contact IDs that have been merged into the contact at hand. - Format: :;: - This field has replaced the `CONTACT_MERGE_AUDIT` table, which was deprecated by the Hubspot v3 CRM API. - - - name: contact_form_submission - description: Table containing contact form submission information - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: contact_id - description: The ID of the related contact. - - name: conversion_id - description: A Unique ID for the specific form conversion. - - name: form_id - description: The GUID of the form that the submission belongs to. - - name: page_url - description: The URL that the form was submitted on, if applicable. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: timestamp - description: A Unix timestamp in milliseconds of the time the submission occurred. - - name: title - description: The title of the page that the form was submitted on. This will default to the name of the form if no title is provided. - - - name: contact_list - description: Each record represents a contact list in Hubspot. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_list_enabled', true) }}" - columns: - - name: _fivetran_deleted - description: '{{ doc("_fivetran_deleted") }}' - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: created_at - description: A timestamp of the time the list was created. - - name: deleteable - description: If this is false, this is a system list and cannot be deleted. - - name: dynamic - description: Whether the contact list is dynamic. - - name: id - description: The ID of the contact list. - - name: metadata_error - description: Any errors that happened the last time the list was processed. - - name: metadata_last_processing_state_change_at - description: A timestamp of the last time that the processing state changed. - - name: metadata_last_size_change_at - description: A timestamp of the last time that the size of the list changed. - - name: metadata_processing - description: | - One of DONE, REFRESHING, INITIALIZING, or PROCESSING. - DONE indicates the list has finished processing, any other value indicates that list membership is being evaluated. - - name: metadata_size - description: The approximate number of contacts in the list. - - name: name - description: The name of the contact list. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: updated_at - description: A timestamp of the time that the list was last modified. - - - name: contact_list_member - description: Each record represents a 'link' between a contact and a contact list. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_list_member_enabled', true) }}" - columns: - - name: _fivetran_deleted - description: '{{ doc("_fivetran_deleted") }}' - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: added_at - description: The timestamp a contact was added to a list. - - name: contact_id - description: The ID of the related contact. - - name: contact_list_id - description: The ID of the related contact list. - - - name: contact_property_history - freshness: - warn_after: {count: 84, period: hour} - error_after: {count: 168, period: hour} - description: Each record represents a change to contact record in Hubspot. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_contact_property_enabled', true) and var('hubspot_contact_property_history_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: contact_id - description: The ID of the related contact record. - - name: name - description: '{{ doc("history_name") }}' - - name: source - description: '{{ doc("history_source") }}' - - name: source_id - description: '{{ doc("history_source_id") }}' - - name: timestamp - description: '{{ doc("history_timestamp") }}' - - name: value - description: '{{ doc("history_value") }}' - - - name: deal - description: Each record represents a deal in Hubspot. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) }}" - columns: - - name: deal_id - description: The ID of the deal. - - name: is_deleted - description: Whether the record was deleted. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: deal_pipeline_id - description: The ID of the deal's pipeline. - - name: deal_pipeline_stage_id - description: The ID of the deal's pipeline stage. - - name: owner_id - description: The ID of the deal's owner. - - name: property_dealname - description: The name you have given this deal. - - name: property_description - description: A brief description of the deal. - - name: property_amount - description: The total value of the deal in the deal's currency. - - name: property_closedate - description: The day the deal is expected to close, or was closed. - - name: property_createdate - description: The date the deal was created. This property is set automatically by HubSpot. - - - name: deal_stage - description: Each record represents a deal stage. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) }}" - columns: - - name: _fivetran_active - description: Boolean indicating whether the deal stage is active. - - name: _fivetran_end - description: The Fivetran calculated exist time of the deal stage. - - name: _fivetran_start - description: The date the deal stage was entered. - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: date_entered - description: The timestamp the deal stage was entered. - - name: deal_id - description: Reference to the deal. - - name: source - description: The relevant source of the deal stage. - - name: source_id - description: Reference to the source. - - name: value - description: The value of the deal stage. Typically the name of the stage. - - - name: deal_company - description: Each record represents a 'link' between a deal and company. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) and var('hubspot_deal_company_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: company_id - description: The ID of the company. - - name: deal_id - description: The ID of the deal. - - name: type_id - description: The ID of the type. - - - name: deal_contact - description: Each record represents a 'link' between a deal and a contact. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) and var('hubspot_deal_contact_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: contact_id - description: The ID of the contact. - - name: deal_id - description: The ID of the deal. - - name: type_id - description: The ID of the type. - - - name: deal_pipeline - description: Each record represents a pipeline in Hubspot. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) }}" - columns: - - name: _fivetran_deleted - description: '{{ doc("_fivetran_deleted") }}' - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: active - description: Whether the stage is currently in use. - - name: display_order - description: Used to determine the order in which the pipelines appear when viewed in HubSpot. - - name: label - description: The human-readable label for the pipeline. The label is used when showing the pipeline in HubSpot. - - name: pipeline_id - description: The ID of the pipeline. - - name: created_at - description: A timestamp representing when the record was created. - - name: updated_at - description: A timestamp representing when the record was updated. - - - name: deal_pipeline_stage - description: Each record represents a pipeline stage in Hubspot. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) }}" - columns: - - name: _fivetran_deleted - description: '{{ doc("_fivetran_deleted") }}' - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: active - description: Whether the pipeline stage is currently in use. - - name: closed_won - description: Whether the stage represents a Closed Won deal. - - name: display_order - description: Used to determine the order in which the stages appear when viewed in HubSpot. - - name: label - description: The human-readable label for the stage. The label is used when showing the stage in HubSpot. - - name: pipeline_id - description: The ID of the related pipeline. - - name: probability - description: The probability that the deal will close. Used for the deal forecast. - - name: stage_id - description: The ID of the pipeline stage. - - name: created_at - description: A timestamp representing when the record was created. - - name: updated_at - description: A timestamp representing when the record was updated. - - - name: deal_property_history - description: Each record represents a change to deal record in Hubspot. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_deal_enabled', true) and var('hubspot_deal_property_history_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: deal_id - description: The ID of the related deal record. - - name: name - description: '{{ doc("history_name") }}' - - name: source - description: '{{ doc("history_source") }}' - - name: source_id - description: '{{ doc("history_source_id") }}' - - name: timestamp - description: '{{ doc("history_timestamp") }}' - - name: value - description: '{{ doc("history_value") }}' - - - name: email_campaign - description: Each record represents an email campaign in Hubspot. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: app_id - description: The app ID. - - name: app_name - description: The app name. - - name: content_id - description: The ID of the content. - - name: id - description: The ID of the email campaign. - - name: name - description: The name of the email campaign. - - name: num_included - description: The number of messages included as part of the email campaign. - - name: num_queued - description: The number of messages queued as part of the email campaign. - - name: sub_type - description: The email campaign sub-type. - - name: subject - description: The subject of the email campaign. - - name: type - description: The email campaign type. - - - name: email_event - freshness: - warn_after: {count: 84, period: hour} - error_after: {count: 168, period: hour} - description: Each record represents an email event in Hubspot. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: app_id - description: The ID of the app that sent the email. - - name: caused_by_created - description: The timestamp of the event that caused this event. - - name: caused_by_id - description: The event ID which uniquely identifies the event which directly caused this event. If not applicable, this property is omitted. - - name: created - description: The created timestamp of the event. - - name: email_campaign_id - description: The ID of the related email campaign. - - name: filtered_event - description: A boolean representing whether the event has been filtered out of reporting based on customer reports settings or not. - - name: id - description: The ID of the event. - - name: obsoleted_by_created - description: The timestamp of the event that made the current event obsolete. - - name: obsoleted_by_id - description: The event ID which uniquely identifies the follow-on event which makes this current event obsolete. If not applicable, this property is omitted. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: recipient - description: The email address of the contact related to the event. - - name: sent_by_created - description: The timestamp of the SENT event related to this event. - - name: sent_by_id - description: The event ID which uniquely identifies the email message's SENT event. If not applicable, this property is omitted. - - name: type - description: The type of event. - - - name: email_event_bounce - description: Each record represents a BOUNCE email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_bounce_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: category - description: | - The best-guess of the type of bounce encountered. - If an appropriate category couldn't be determined, this property is omitted. See below for the possible values. - Note that this is a derived value, and may be modified at any time to improve the accuracy of classification. - - name: id - description: The ID of the event. - - name: response - description: The full response from the recipient's email server. - - name: status - description: The status code returned from the recipient's email server. - - - name: email_event_click - description: Each record represents a CLICK email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_click_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: browser - description: '{{ doc("email_event_browser") }}' - - name: id - description: The ID of the event. - - name: ip_address - description: '{{ doc("email_event_ip_address") }}' - - name: location - description: '{{ doc("email_event_location") }}' - - name: referer - description: The URL of the webpage that linked to the URL clicked. Whether this is provided, and what its value is, is determined by the recipient's email client. - - name: url - description: The URL within the message that the recipient clicked. - - name: user_agent - description: '{{ doc("email_event_user_agent") }}' - - - name: email_event_deferred - description: Each record represents a DEFERRED email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_deferred_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: attempt - description: The delivery attempt number. - - name: id - description: The ID of the event. - - name: response - description: The full response from the recipient's email server. - - - name: email_event_delivered - description: Each record represents a DELIVERED email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_delivered_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: id - description: The ID of the event. - - name: response - description: The full response from the recipient's email server. - - name: smtp_id - description: An ID attached to the message by HubSpot. - - - name: email_event_dropped - description: Each record represents a DROPPED email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_dropped_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: bcc - description: The 'bcc' field of the email message. - - name: cc - description: The 'cc' field of the email message. - - name: drop_message - description: The raw message describing why the email message was dropped. This will usually provide additional details beyond 'dropReason'. - - name: drop_reason - description: The reason why the email message was dropped. See below for the possible values. - - name: from - description: The 'from' field of the email message. - - name: id - description: The ID of the event. - - name: reply_to - description: The 'reply-to' field of the email message. - - name: subject - description: The subject line of the email message. - - - name: email_event_forward - description: Each record represents a FORWARD email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_forward_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: browser - description: '{{ doc("email_event_browser") }}' - - name: id - description: The ID of the event. - - name: ip_address - description: '{{ doc("email_event_ip_address") }}' - - name: location - description: '{{ doc("email_event_location") }}' - - name: user_agent - description: '{{ doc("email_event_user_agent") }}' - - - name: email_event_open - description: Each record represents a OPEN email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_open_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: browser - description: '{{ doc("email_event_browser") }}' - - name: duration - description: If provided and nonzero, the approximate number of milliseconds the user had opened the email. - - name: id - description: The ID of the event. - - name: ip_address - description: '{{ doc("email_event_ip_address") }}' - - name: location - description: '{{ doc("email_event_location") }}' - - name: user_agent - description: '{{ doc("email_event_user_agent") }}' - - - name: email_event_print - description: Each record represents a PRINT email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_print_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: browser - description: '{{ doc("email_event_browser") }}' - - name: id - description: The ID of the event. - - name: ip_address - description: '{{ doc("email_event_ip_address") }}' - - name: location - description: '{{ doc("email_event_location") }}' - - name: user_agent - description: '{{ doc("email_event_user_agent") }}' - - - name: email_event_sent - description: Each record represents a SENT email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_sent_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: bcc - description: The 'cc' field of the email message. - - name: cc - description: The 'bcc' field of the email message. - - name: from - description: The 'from' field of the email message. - - name: id - description: The ID of the event. - - name: reply_to - description: The 'reply-to' field of the email message. - - name: subject - description: The subject line of the email message. - - - name: email_event_spam_report - description: Each record represents a SPAM_REPORT email event. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_spam_report_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: id - description: The ID of the event. - - name: ip_address - description: '{{ doc("email_event_ip_address") }}' - - name: user_agent - description: '{{ doc("email_event_user_agent") }}' - - - name: email_event_status_change - description: Each record represents a STATUS_CHANGE email event in Hubspot. - config: - enabled: "{{ var('hubspot_marketing_enabled', true) and var('hubspot_email_event_enabled', true) and var('hubspot_email_event_status_change_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: bounced - description: | - A HubSpot employee explicitly initiated the status change to block messages to the recipient. - (Note this usage has been deprecated in favor of dropping messages with a 'dropReason' of BLOCKED_ADDRESS.) - - name: id - description: The ID of the event. - - name: portal_subscription_status - description: | - The recipient's portal subscription status. - Note that if this is 'UNSUBSCRIBED', the property 'subscriptions' is not necessarily an empty array, nor are all - subscriptions contained in it necessarily going to have their statuses set to 'UNSUBSCRIBED'.) - - name: requested_by - description: The email address of the person requesting the change on behalf of the recipient. If not applicable, this property is omitted. - - name: source - description: The source of the subscription change. - - name: subscriptions - description: | - An array of JSON objects representing the status of subscriptions for the recipient. - Each JSON subscription object is comprised of the properties: 'id', 'status'. - - - name: email_subscription - description: Each record represents an email subscription in Hubspot. - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: active - description: Whether the subscription is active. - - name: description - description: The description of the subscription. - - name: id - description: The ID of the email subscription. - - name: name - description: The name of the email subscription. - - name: portal_id - description: '{{ doc("portal_id") }}' - - - name: email_subscription_change - description: Each record represents a change to an email subscription in Hubspot. - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: caused_by_event_id - description: The ID of the event that caused the subscription change. - - name: change - description: The change which occurred. This enumeration is specific to the 'changeType'; see below for the possible values. - - name: change_type - description: The type of change which occurred. - - name: email_subscription_id - description: The ID of the related email subscription. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: recipient - description: The email address of the related contact. - - name: source - description: The source of the subscription change. - - name: timestamp - description: The timestamp when this change occurred. If 'causedByEvent' is present, this will be absent. - - - name: engagement - description: Each record represents an engagement - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: active - description: > - Whether the engagement is currently being shown in the UI. - - PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release. - - name: created_at - description: > - A timestamp representing when the engagement was created. - - PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release. - - name: id - description: The ID of the engagement. - - name: owner_id - description: > - The ID of the engagement's owner. - - PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: timestamp - description: > - A timestamp in representing the time that the engagement should appear in the timeline. - - PLEASE NOTE: This field will not be populated for connectors utilizing the HubSpot v3 API version. This field will be deprecated in a future release. - - name: type - description: One of NOTE, EMAIL, TASK, MEETING, or CALL, the type of the engagement. - - - name: engagement_call - description: Each record represents a CALL engagement event. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_call_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: engagement_id - description: The ID of the engagement. - - name: _fivetran_deleted - description: Boolean to mark rows that were deleted in the source database. - - name: property_hs_createdate - description: > - This field marks the call's time of creation and determines where the call sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: timestamp - description: > - This field marks the call's time of occurrence and determines where the call sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_owner_id - description: > - The ID of the owner associated with the call. This field determines the user listed as the call creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_team_id - description: > - The ID of the team associated with the call. This field determines the team listed as the call creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. - - - name: engagement_company - description: Each record represents a 'link' between a company and an engagement. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_company_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: company_id - description: The ID of the related company. - - name: engagement_id - description: The ID of the related engagement. - - - name: engagement_contact - description: Each record represents a 'link' between a contact and an engagement. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_contact_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: contact_id - description: The ID of the related contact. - - name: engagement_id - description: The ID of the related engagement. - - - name: engagement_deal - description: Each record represents a 'link' between a deal and an engagement. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_deal_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: deal_id - description: The ID of the related contact. - - name: engagement_id - description: The ID of the related engagement. - - - name: engagement_email - description: Each record represents an EMAIL engagement event. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_email_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: engagement_id - description: The ID of the engagement. - - name: property_hs_createdate - description: > - This field marks the email's time of creation and determines where the email sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: timestamp - description: > - This field marks the email's time of occurrence and determines where the email sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_owner_id - description: > - The ID of the owner associated with the email. This field determines the user listed as the email creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_team_id - description: > - The ID of the team associated with the email. This field determines the team listed as the email creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. - - - name: engagement_email_cc - description: Each record represents a CC'd email address as part of an EMAIL engagement. - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: email - description: The email address of the recipient. - - name: engagement_id - description: The ID of the related engagement. - - name: first_name - description: The first name of the recipient. - - name: last_name - description: The last name of the recipient. - - - name: engagement_email_to - description: Each record represents a TO email address as part of an EMAIL engagement. - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: email - description: The email address of the recipient. - - name: engagement_id - description: The ID of the related engagement. - - name: first_name - description: The first name of the recipient. - - name: last_name - description: The last name of the recipient. - - - name: engagement_meeting - description: Each record represents a MEETING engagement event. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_meeting_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: engagement_id - description: The ID of the engagement. - - name: property_hs_createdate - description: > - This field marks the meeting's time of creation and determines where the meeting sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: timestamp - description: > - This field marks the meeting's time of occurrence and determines where the meeting sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_owner_id - description: > - The ID of the owner associated with the meeting. This field determines the user listed as the meeting creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_team_id - description: > - The ID of the team associated with the meeting. This field determines the team listed as the meeting creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. - - - name: engagement_note - description: Each record represents a NOTE engagement event. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_note_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: body - description: The body of the note. The body has a limit of 65536 characters. - - name: engagement_id - description: The ID of the engagement. - - name: property_hs_createdate - description: > - This field marks the note's time of creation and determines where the note sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: timestamp - description: > - This field marks the note's time of occurrence and determines where the note sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_owner_id - description: > - The ID of the owner associated with the note. This field determines the user listed as the note creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_team_id - description: > - The ID of the team associated with the note. This field determines the team listed as the note creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. - - - name: engagement_task - description: Each record represents a TASK engagement event. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_engagement_enabled', true) and var('hubspot_engagement_task_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: engagement_id - description: The ID of the engagement. - - name: property_hs_createdate - description: > - This field marks the task's time of creation and determines where the task sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: timestamp - description: > - This field marks the task's time of occurrence and determines where the task sits on the record timeline. You can use either a Unix timestamp in milliseconds or UTC format. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_owner_id - description: > - The ID of the owner associated with the task. This field determines the user listed as the task creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. For the pre HubSpot v3 versions, this value may be found within the parent `engagement` table. - - name: property_hubspot_team_id - description: > - The ID of the team associated with the task. This field determines the team listed as the task creator on the record timeline. - - PLEASE NOTE: This field will only be populated for connectors utilizing the HubSpot v3 API version. - - - name: form - description: Each record represents a Hubspot form. - columns: - - name: _fivetran_deleted - description: '{{ doc("_fivetran_deleted") }}' - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: created_at - description: A timestamp for when the form was created. - - name: css_class - description: The CSS classes assigned to the form. - - name: follow_up_id - description: This field is no longer used. - - name: guid - description: The internal ID of the form. - - name: lead_nurturing_campaign_id - description: TBD - - name: method - description: This field is no longer used. - - name: name - description: The name of the form. - - - name: notify_recipients - description: | - A comma-separated list of user IDs that should receive submission notifications. - Email addresses will be returned for individuals who aren't users. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: redirect - description: The URL that the visitor will be redirected to after filling out the form. - - name: submit_text - description: The text used for the submit button. - - name: updated_at - description: A timestamp for when the form was last updated. - - - name: owner - description: Each record represents an owner/user in Hubspot. - config: - enabled: "{{ var('hubspot_sales_enabled', true) and var('hubspot_owner_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: created_at - description: A timestamp for when the owner was created. - - name: email - description: The email address of the owner. - - name: first_name - description: The first name of the owner. - - name: last_name - description: The last name of the owner. - - name: owner_id - description: The ID of the owner. - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: type - description: The type of owner. - - name: updated_at - - - name: property - description: Each record represents a property. - config: - enabled: "{{ var('hubspot_property_enabled', true) }}" - columns: - - name: _fivetran_id - description: Fivetran generated id. Joins to `property_id` in the `property_option` table. - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: calculated - description: Indicates if the property is calculated by a HubSpot process - - name: created_at - description: Timestamp representing when the property was created. - - name: description - description: A description of the property. - - name: field_type - description: One of textarea, text, date, file, number, select, radio, checkbox, or booleancheckbox. - - name: group_name - description: The name of the property group that the property belongs to. - - name: hubspot_defined - description: This will be true for default properties that are built into HubSpot. - - name: hubspot_object - description: If this property is related to other objects, the object will be listed here. - - name: label - description: A human readable label for the property. - - name: name - description: The internal name of the property. - - name: show_currency_symbol - description: If true, the property will display the currency symbol set in the portal settings. - - name: type - description: One of string, number, date, datetime, or enumeration. - - name: updated_at - description: Timestamp representing when the property was last updated. - - - name: property_option - description: Each record represents an option for the related property. - config: - enabled: "{{ var('hubspot_property_enabled', true) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: label - description: The label of the option displayed inside the HubSpot app. - - name: property_id - description: The ID of the related property. - - name: display_order - description: Order of options displayed in Hubspot, starting with the lowest positive integer value. Values of -1 will cause the option to be displayed after any positive values. - - name: hidden - description: Boolean if the option will be displayed in HubSpot - - name: value - description: The internal value of the option. - - - name: ticket_company - description: Each record represents a 'link' between a ticket and company. - config: - enabled: "{{ var('hubspot_service_enabled', false) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: ticket_id - description: The ID of the related ticket. - - name: company_id - description: The ID of the related company. - - - name: ticket_contact - description: Each record represents a 'link' between a ticket and a contact. - config: - enabled: "{{ var('hubspot_service_enabled', false) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: ticket_id - description: The ID of the related ticket. - - name: contact_id - description: The ID of the related contact. - - - name: ticket_deal - description: Each record represents a 'link' between a ticket and a deal. - config: - enabled: "{{ var('hubspot_service_enabled', false) and var('hubspot_ticket_deal_enabled', false) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: ticket_id - description: The ID of the related ticket. - - name: deal_id - description: The ID of the related deal. - - - name: ticket_engagement - description: Each record represents a 'link' between a ticket and an engagement. - config: - enabled: "{{ var('hubspot_service_enabled', false) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: ticket_id - description: The ID of the related ticket. - - name: engagement_id - description: The ID of the related deal. - - - name: ticket_pipeline_stage - description: Each record represents a ticket pipeline stage. - config: - enabled: "{{ var('hubspot_service_enabled', false) }}" - columns: - - name: _fivetran_deleted - description: '{{ doc("_fivetran_deleted") }}' - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: active - description: Boolean indicating whether the pipeline stage is active. - - name: display_order - description: Used to determine the order in which the stages appear when viewed in HubSpot. - - name: is_closed - description: Boolean indicating if the pipeline stage is closed. - - name: label - description: The human-readable label for the stage. The label is used when showing the stage in HubSpot. - - name: pipeline_id - description: The ID of the pipeline. - - name: stage_id - description: The ID of the pipeline stage. - - name: ticket_state - description: State of the ticket. - - name: created_at - description: A timestamp representing when the record was created. - - name: updated_at - description: A timestamp representing when the record was updated. - - - name: ticket_pipeline - description: Each record represents a ticket pipeline. - config: - enabled: "{{ var('hubspot_service_enabled', false) }}" - columns: - - name: _fivetran_deleted - description: '{{ doc("_fivetran_deleted") }}' - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: active - description: Boolean indicating whether the pipeline is active. - - name: display_order - description: Used to determine the order in which the stages appear when viewed in HubSpot. - - name: label - description: The human-readable label for the stage. The label is used when showing the stage in HubSpot. - - name: object_type_id - description: Reference to the object type. - - name: pipeline_id - description: Reference to the pipeline. - - name: created_at - description: A timestamp representing when the record was created. - - name: updated_at - description: A timestamp representing when the record was updated. - - - name: ticket_property_history - description: Each record represents a change to ticket record in Hubspot. - config: - enabled: "{{ var('hubspot_service_enabled', false) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: source - description: '{{ doc("history_source") }}' - - name: source_id - description: '{{ doc("history_source_id") }}' - - name: timestamp_instant - description: '{{ doc("history_timestamp") }}' - - name: ticket_id - description: The ID of the related ticket record. - - name: name - description: '{{ doc("history_name") }}' - - name: value - description: '{{ doc("history_value") }}' - - - name: ticket - description: Each record represents a ticket in Hubspot. - config: - enabled: "{{ var('hubspot_service_enabled', false) }}" - columns: - - name: _fivetran_synced - description: '{{ doc("_fivetran_synced") }}' - - name: id - description: ID of the ticket. - - name: is_deleted - description: Whether the record was deleted (v2 endpoint). - - name: _fivetran_deleted - description: Whether the record was deleted (v3 endpoint). - - name: portal_id - description: '{{ doc("portal_id") }}' - - name: property_closed_date - description: The date the ticket was closed. - - name: property_createdate - description: The date the ticket was created. - - name: property_first_agent_reply_date - description: the date for the first agent reply on the ticket. - - name: property_hs_pipeline - description: The ID of the ticket's pipeline. - - name: property_hs_pipeline_stage - description: The ID of the ticket's pipeline stage. - - name: property_hs_ticket_priority - description: The priority of the ticket. - - name: property_hs_ticket_category - description: The category of the ticket. - - name: property_hubspot_owner_id - description: The ID of the deal's owner. - - name: property_subject - description: Short summary of ticket. - - name: property_content - description: Text in body of the ticket. + tables: # copy and paste from models/src_hubspot.yml ``` -

+ +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_hubspot_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them here, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. 2. Set the `has_defined_sources` variable (scoped to the `hubspot_source` package) to true, like such: ```yml From 7ef069f9eb946f6a116b760890a02c2bc8bd3de8 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 17:20:36 -0500 Subject: [PATCH 16/16] tiny tweak to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 83c2cb6c..b8531a11 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ sources: > **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_hubspot_source?tab=readme-ov-file#step-4-disable-models-for-non-existent-sources)), you may still include them here, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. -2. Set the `has_defined_sources` variable (scoped to the `hubspot_source` package) to true, like such: +2. Set the `has_defined_sources` variable (scoped to the `hubspot_source` package) to `True`, like such: ```yml # dbt_project.yml vars: