diff --git a/website/dbt-versions.js b/website/dbt-versions.js index 825af8ac6ee..f84184a486c 100644 --- a/website/dbt-versions.js +++ b/website/dbt-versions.js @@ -20,7 +20,6 @@ exports.versions = [ }, { version: "1.9", - isPrerelease: true, }, { version: "1.8", diff --git a/website/docs/docs/build/conversion-metrics.md b/website/docs/docs/build/conversion-metrics.md index 2ef2c3910b9..97809a13a76 100644 --- a/website/docs/docs/build/conversion-metrics.md +++ b/website/docs/docs/build/conversion-metrics.md @@ -20,28 +20,28 @@ The specification for conversion metrics is as follows: Note that we use the double colon (::) to indicate whether a parameter is nested within another parameter. So for example, `query_params::metrics` means the `metrics` parameter is nested under `query_params`. ::: -| Parameter | Description | Type | -| --- | --- | --- | -| `name` | The name of the metric. | Required | -| `description` | The description of the metric. | Optional | -| `type` | The type of metric (such as derived, ratio, and so on.). In this case, set as 'conversion' | Required | -| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | -| `type_params` | Specific configurations for each metric type. | Required | -| `conversion_type_params` | Additional configuration specific to conversion metrics. | Required | -| `entity` | The entity for each conversion event. | Required | -| `calculation` | Method of calculation. Either `conversion_rate` or `conversions`. Defaults to `conversion_rate`. | Optional | -| `base_measure` | A list of base measure inputs | Required | -| `base_measure:name` | The base conversion event measure. | Required | -| `base_measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | -| `base_measure:join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | -| `conversion_measure` | A list of conversion measure inputs. | Required | -| `conversion_measure:name` | The base conversion event measure.| Required | -| `conversion_measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | -| `conversion_measure:join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | -| `window` | The time window for the conversion event, such as 7 days, 1 week, 3 months. Defaults to infinity. | Optional | -| `constant_properties` | List of constant properties. | Optional | -| `base_property` | The property from the base semantic model that you want to hold constant. | Optional | -| `conversion_property` | The property from the conversion semantic model that you want to hold constant. | Optional | +| Parameter | Description | Required | Type | +| --- | --- | --- | --- | +| `name` | The name of the metric. | Required | String | +| `description` | The description of the metric. | Optional | String | +| `type` | The type of metric (such as derived, ratio, and so on.). In this case, set as 'conversion'. | Required | String | +| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | String | +| `type_params` | Specific configurations for each metric type. | Required | Dict | +| `conversion_type_params` | Additional configuration specific to conversion metrics. | Required | Dict | +| `entity` | The entity for each conversion event. | Required | String | +| `calculation` | Method of calculation. Either `conversion_rate` or `conversions`. Defaults to `conversion_rate`. | Optional | String | +| `base_measure` | A list of base measure inputs. | Required | Dict | +| `base_measure:name` | The base conversion event measure. | Required | String | +| `base_measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | String | +| `base_measure:join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | Boolean | +| `conversion_measure` | A list of conversion measure inputs. | Required | Dict | +| `conversion_measure:name` | The base conversion event measure.| Required | String | +| `conversion_measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | String | +| `conversion_measure:join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | Boolean | +| `window` | The time window for the conversion event, such as 7 days, 1 week, 3 months. Defaults to infinity. | Optional | String | +| `constant_properties` | List of constant properties. | Optional | List | +| `base_property` | The property from the base semantic model that you want to hold constant. | Optional | String | +| `conversion_property` | The property from the conversion semantic model that you want to hold constant. | Optional | String | Refer to [additional settings](#additional-settings) to learn how to customize conversion metrics with settings for null values, calculation type, and constant properties. @@ -111,7 +111,7 @@ Next, define a conversion metric as follows: base_measure: name: visits fill_nulls_with: 0 - conversion_measure: sellers + conversion_measure: name: sellers entity: user window: 7 days diff --git a/website/docs/docs/build/cumulative-metrics.md b/website/docs/docs/build/cumulative-metrics.md index b44918d2fbd..24596be8b3d 100644 --- a/website/docs/docs/build/cumulative-metrics.md +++ b/website/docs/docs/build/cumulative-metrics.md @@ -18,21 +18,21 @@ Note that we use the double colon (::) to indicate whether a parameter is nested -| Parameter |
Description
| Type | -|-------------|---------------------------------------------------|-----------| -| `name` | The name of the metric. | Required | -| `description` | The description of the metric. | Optional | -| `type` | The type of the metric (cumulative, derived, ratio, or simple). | Required | -| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | -| `type_params` | The type parameters of the metric. Supports nested parameters indicated by the double colon, such as `type_params::measure`. | Required | -| `type_params::measure` | The measure associated with the metric. Supports both shorthand (string) and object syntax. The shorthand is used if only the name is needed, while the object syntax allows specifying additional attributes. | Required | -| `measure::name` | The name of the measure being referenced. Required if using object syntax for `type_params::measure`. | Optional | -| `measure::fill_nulls_with` | Sets a value (for example, 0) to replace nulls in the metric definition. | Optional | -| `measure::join_to_timespine` | Boolean indicating if the aggregated measure should be joined to the time spine table to fill in missing dates. Default is `false`. | Optional | -| `type_params::cumulative_type_params` | Configures the attributes like `window`, `period_agg`, and `grain_to_date` for cumulative metrics. | Optional | -| `cumulative_type_params::window` | Specifies the accumulation window, such as `1 month`, `7 days`, or `1 year`. Cannot be used with `grain_to_date`. | Optional | -| `cumulative_type_params::grain_to_date` | Sets the accumulation grain, such as `month`, restarting accumulation at the beginning of each specified grain period. Cannot be used with `window`. | Optional | -| `cumulative_type_params::period_agg` | Defines how to aggregate the cumulative metric when summarizing data to a different granularity: `first`, `last`, or `average`. Defaults to `first` if `window` is not specified. | Optional | +| Parameter |
Description
| Required | Type | +|-------------|---------------------------------------------------|----------|-----------| +| `name` | The name of the metric. | Required | String | +| `description` | The description of the metric. | Optional | String | +| `type` | The type of the metric (cumulative, derived, ratio, or simple). | Required | String | +| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | String | +| `type_params` | The type parameters of the metric. Supports nested parameters indicated by the double colon, such as `type_params::measure`. | Required | Dict | +| `type_params::measure` | The measure associated with the metric. Supports both shorthand (string) and object syntax. The shorthand is used if only the name is needed, while the object syntax allows specifying additional attributes. | Required | Dict | +| `measure::name` | The name of the measure being referenced. Required if using object syntax for `type_params::measure`. | Optional | String | +| `measure::fill_nulls_with` | Sets a value (for example, 0) to replace nulls in the metric definition. | Optional | Integer or string | +| `measure::join_to_timespine` | Boolean indicating if the aggregated measure should be joined to the time spine table to fill in missing dates. Default is `false`. | Optional | Boolean | +| `type_params::cumulative_type_params` | Configures the attributes like `window`, `period_agg`, and `grain_to_date` for cumulative metrics. | Optional | Dict | +| `cumulative_type_params::window` | Specifies the accumulation window, such as `1 month`, `7 days`, or `1 year`. Cannot be used with `grain_to_date`. | Optional | String | +| `cumulative_type_params::grain_to_date` | Sets the accumulation grain, such as `month`, restarting accumulation at the beginning of each specified grain period. Cannot be used with `window`. | Optional | String | +| `cumulative_type_params::period_agg` | Defines how to aggregate the cumulative metric when summarizing data to a different granularity: `first`, `last`, or `average`. Defaults to `first` if `window` is not specified. | Optional | String | diff --git a/website/docs/docs/build/derived-metrics.md b/website/docs/docs/build/derived-metrics.md index d5f2221907e..b6184aaeebf 100644 --- a/website/docs/docs/build/derived-metrics.md +++ b/website/docs/docs/build/derived-metrics.md @@ -10,18 +10,18 @@ In MetricFlow, derived metrics are metrics created by defining an expression usi The parameters, description, and type for derived metrics are: -| Parameter | Description | Type | -| --------- | ----------- | ---- | -| `name` | The name of the metric. | Required | -| `description` | The description of the metric. | Optional | -| `type` | The type of the metric (cumulative, derived, ratio, or simple). | Required | -| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | -| `type_params` | The type parameters of the metric. | Required | -| `expr` | The derived expression. You see validation warnings when the derived metric is missing an `expr` or the `expr` does not use all the input metrics. | Required | -| `metrics` | The list of metrics used in the derived metrics. | Required | -| `alias` | Optional alias for the metric that you can use in the expr. | Optional | -| `filter` | Optional filter to apply to the metric. | Optional | -| `offset_window` | Set the period for the offset window, such as 1 month. This will return the value of the metric one month from the metric time. | Optional | +| Parameter | Description | Required | Type | +| --------- | ----------- | ---- | ---- | +| `name` | The name of the metric. | Required | String | +| `description` | The description of the metric. | Optional | String | +| `type` | The type of the metric (cumulative, derived, ratio, or simple). | Required | String | +| `label` | Defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | String | +| `type_params` | The type parameters of the metric. | Required | Dict | +| `expr` | The derived expression. You'll see validation warnings when the derived metric is missing an `expr` or the `expr` does not use all the input metrics. | Required | String | +| `metrics` | The list of metrics used in the derived metrics. Each entry can include optional fields like `alias`, `filter`, or `offset_window`. | Required | List | +| `alias` | Optional alias for the metric that you can use in the `expr`. | Optional | String | +| `filter` | Optional filter to apply to the metric. | Optional | String | +| `offset_window` | Set the period for the offset window, such as 1 month. This will return the value of the metric one month from the metric time. | Optional | String | The following displays the complete specification for derived metrics, along with an example. diff --git a/website/docs/docs/build/dimensions.md b/website/docs/docs/build/dimensions.md index 5026f4c45cd..975ae4d3160 100644 --- a/website/docs/docs/build/dimensions.md +++ b/website/docs/docs/build/dimensions.md @@ -14,14 +14,14 @@ Groups are defined within semantic models, alongside entities and measures, and All dimensions require a `name`, `type`, and can optionally include an `expr` parameter. The `name` for your Dimension must be unique within the same semantic model. -| Parameter | Description | Type | -| --------- | ----------- | ---- | -| `name` | Refers to the name of the group that will be visible to the user in downstream tools. It can also serve as an alias if the column name or SQL query reference is different and provided in the `expr` parameter.

Dimension names should be unique within a semantic model, but they can be non-unique across different models as MetricFlow uses [joins](/docs/build/join-logic) to identify the right dimension. | Required | -| `type` | Specifies the type of group created in the semantic model. There are two types:

- **Categorical**: Describe attributes or features like geography or sales region.
- **Time**: Time-based dimensions like timestamps or dates. | Required | -| `type_params` | Specific type params such as if the time is primary or used as a partition | Required | -| `description` | A clear description of the dimension | Optional | -| `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | -| `label` | A recommended string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | +| Parameter | Description | Required | Type | +| --------- | ----------- | ---- | ---- | +| `name` | Refers to the name of the group that will be visible to the user in downstream tools. It can also serve as an alias if the column name or SQL query reference is different and provided in the `expr` parameter.

Dimension names should be unique within a semantic model, but they can be non-unique across different models as MetricFlow uses [joins](/docs/build/join-logic) to identify the right dimension. | Required | String | +| `type` | Specifies the type of group created in the semantic model. There are two types:

- **Categorical**: Describe attributes or features like geography or sales region.
- **Time**: Time-based dimensions like timestamps or dates. | Required | String | +| `type_params` | Specific type params such as if the time is primary or used as a partition. | Required | Dict | +| `description` | A clear description of the dimension. | Optional | String | +| `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | String | +| `label` | Defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | String | Refer to the following for the complete specification for dimensions: diff --git a/website/docs/docs/build/incremental-microbatch.md b/website/docs/docs/build/incremental-microbatch.md index 46dfa795f1d..e6c8284cc4b 100644 --- a/website/docs/docs/build/incremental-microbatch.md +++ b/website/docs/docs/build/incremental-microbatch.md @@ -8,7 +8,9 @@ id: "incremental-microbatch" :::info Microbatch -The `microbatch` strategy is available in beta for [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless) and dbt Core v1.9. We have been developing it behind a flag to prevent unintended interactions with existing custom incremental strategies. To enable this feature, [set the environment variable](/docs/build/environment-variables#setting-and-overriding-environment-variables) `DBT_EXPERIMENTAL_MICROBATCH` to `True` in your dbt Cloud environments or wherever you're running dbt Core. +The new `microbatch` strategy is available in beta for [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless) and dbt Core v1.9. + +If you use a custom microbatch macro, set a [distinct behavior flag](/reference/global-configs/behavior-changes#custom-microbatch-strategy) in your `dbt_project.yml` to enable batched execution. If you don't have a custom microbatch macro, you don't need to set this flag as dbt will handle microbatching automatically for any model using the [microbatch strategy](#how-microbatch-compares-to-other-incremental-strategies). Read and participate in the discussion: [dbt-core#10672](https://github.com/dbt-labs/dbt-core/discussions/10672) @@ -20,17 +22,34 @@ Refer to [Supported incremental strategies by adapter](/docs/build/incremental-s Incremental models in dbt are a [materialization](/docs/build/materializations) designed to efficiently update your data warehouse tables by only transforming and loading _new or changed data_ since the last run. Instead of reprocessing an entire dataset every time, incremental models process a smaller number of rows, and then append, update, or replace those rows in the existing table. This can significantly reduce the time and resources required for your data transformations. -Microbatch incremental models make it possible to process transformations on very large time-series datasets with efficiency and resiliency. When dbt runs a microbatch model — whether for the first time, during incremental runs, or in specified backfills — it will split the processing into multiple queries (or "batches"), based on the [`event_time`](/reference/resource-configs/event-time) and `batch_size` you configure. +Microbatch is an incremental strategy designed for large time-series datasets: +- It relies solely on a time column ([`event_time`](/reference/resource-configs/event-time)) to define time-based ranges for filtering. Set the `event_time` column for your microbatch model and its direct parents (upstream models). Note, this is different to `partition_by`, which groups rows into partitions. +- It complements, rather than replaces, existing incremental strategies by focusing on efficiency and simplicity in batch processing. +- Unlike traditional incremental strategies, microbatch doesn't require implementing complex conditional logic for [backfilling](#backfills). +- Note, microbatch might not be the best strategy for all use cases. Consider other strategies for use cases such as not having a reliable `event_time` column or if you want more control over the incremental logic. Read more in [How `microbatch` compares to other incremental strategies](#how-microbatch-compares-to-other-incremental-strategies). + +### How microbatch works + +When dbt runs a microbatch model — whether for the first time, during incremental runs, or in specified backfills — it will split the processing into multiple queries (or "batches"), based on the `event_time` and `batch_size` you configure. + +Each "batch" corresponds to a single bounded time period (by default, a single day of data). Where other incremental strategies operate only on "old" and "new" data, microbatch models treat every batch as an atomic unit that can be built or replaced on its own. Each batch is independent and . -Each "batch" corresponds to a single bounded time period (by default, a single day of data). Where other incremental strategies operate only on "old" and "new" data, microbatch models treat every batch as an atomic unit that can be built or replaced on its own. Each batch is independent and . This is a powerful abstraction that makes it possible for dbt to run batches separately — in the future, concurrently — and to retry them independently. +This is a powerful abstraction that makes it possible for dbt to run batches [separately](#backfills), concurrently, and [retry](#retry) them independently. ### Example -A `sessions` model aggregates and enriches data that comes from two other models. -- `page_views` is a large, time-series table. It contains many rows, new records almost always arrive after existing ones, and existing records rarely update. -- `customers` is a relatively small dimensional table. Customer attributes update often, and not in a time-based manner — that is, older customers are just as likely to change column values as newer customers. +A `sessions` model aggregates and enriches data that comes from two other models: +- `page_views` is a large, time-series table. It contains many rows, new records almost always arrive after existing ones, and existing records rarely update. It uses the `page_view_start` column as its `event_time`. +- `customers` is a relatively small dimensional table. Customer attributes update often, and not in a time-based manner — that is, older customers are just as likely to change column values as newer customers. The customers model doesn't configure an `event_time` column. -The `page_view_start` column in `page_views` is configured as that model's `event_time`. The `customers` model does not configure an `event_time`. Therefore, each batch of `sessions` will filter `page_views` to the equivalent time-bounded batch, and it will not filter `customers` (a full scan for every batch). +As a result: + +- Each batch of `sessions` will filter `page_views` to the equivalent time-bounded batch. +- The `customers` table isn't filtered, resulting in a full scan for every batch. + +:::tip +In addition to configuring `event_time` for the target table, you should also specify it for any upstream models that you want to filter, even if they have different time columns. +::: diff --git a/website/docs/docs/build/incremental-models.md b/website/docs/docs/build/incremental-models.md index a56246addf3..d7b6ecd8f54 100644 --- a/website/docs/docs/build/incremental-models.md +++ b/website/docs/docs/build/incremental-models.md @@ -114,7 +114,7 @@ When you define a `unique_key`, you'll see this behavior for each row of "new" d Please note that if there's a unique_key with more than one row in either the existing target table or the new incremental rows, the incremental model may fail depending on your database and [incremental strategy](/docs/build/incremental-strategy). If you're having issues running an incremental model, it's a good idea to double check that the unique key is truly unique in both your existing database table and your new incremental rows. You can [learn more about surrogate keys here](https://www.getdbt.com/blog/guide-to-surrogate-key). :::info -While common incremental strategies, such as`delete+insert` + `merge`, might use `unique_key`, others don't. For example, the `insert_overwrite` strategy does not use `unique_key`, because it operates on partitions of data rather than individual rows. For more information, see [About incremental_strategy](/docs/build/incremental-strategy). +While common incremental strategies, such as `delete+insert` + `merge`, might use `unique_key`, others don't. For example, the `insert_overwrite` strategy does not use `unique_key`, because it operates on partitions of data rather than individual rows. For more information, see [About incremental_strategy](/docs/build/incremental-strategy). ::: #### `unique_key` example diff --git a/website/docs/docs/build/incremental-strategy.md b/website/docs/docs/build/incremental-strategy.md index 86b6a89edc6..959671bd16e 100644 --- a/website/docs/docs/build/incremental-strategy.md +++ b/website/docs/docs/build/incremental-strategy.md @@ -295,6 +295,8 @@ For example, a user-defined strategy named `insert_only` can be defined and used +If you use a custom microbatch macro, set a [`require_batched_execution_for_custom_microbatch_strategy` behavior flag](/reference/global-configs/behavior-changes#custom-microbatch-strategy) in your `dbt_project.yml` to enable batched execution of your custom strategy. + ### Custom strategies from a package To use the `merge_null_safe` custom incremental strategy from the `example` package: diff --git a/website/docs/docs/build/metricflow-time-spine.md b/website/docs/docs/build/metricflow-time-spine.md index 5f16af38023..48e46caeec2 100644 --- a/website/docs/docs/build/metricflow-time-spine.md +++ b/website/docs/docs/build/metricflow-time-spine.md @@ -179,8 +179,8 @@ final as ( select * from final -- filter the time spine to a specific range -where date_day > dateadd(year, -4, current_timestamp()) -and date_day < dateadd(day, 30, current_timestamp()) +where date_day > date_add(DATE(current_timestamp()), INTERVAL -4 YEAR) +and date_day < date_add(DATE(current_timestamp()), INTERVAL 30 DAY) ``` diff --git a/website/docs/docs/build/metrics-overview.md b/website/docs/docs/build/metrics-overview.md index 7021a6d7330..f1afa1f37b3 100644 --- a/website/docs/docs/build/metrics-overview.md +++ b/website/docs/docs/build/metrics-overview.md @@ -15,15 +15,15 @@ This article explains the different supported metric types you can add to your d -| Parameter | Description | Type | -| --------- | ----------- | ---- | -| `name` | Provide the reference name for the metric. This name must be a unique metric name and can consist of lowercase letters, numbers, and underscores. | Required | -| `description` | Describe your metric. | Optional | -| `type` | Define the type of metric, which can be `conversion`, `cumulative`, `derived`, `ratio`, or `simple`. | Required | -| `type_params` | Additional parameters used to configure metrics. `type_params` are different for each metric type. | Required | -| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | -| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports [`meta`](/reference/resource-configs/meta), [`group`](/reference/resource-configs/group), and [`enabled`](/reference/resource-configs/enabled) configurations. | Optional | -| `filter` | You can optionally add a [filter](#filters) string to any metric type, applying filters to dimensions, entities, time dimensions, or other metrics during metric computation. Consider it as your WHERE clause. | Optional | +| Parameter | Description | Required | Type | +| --------- | ----------- | ---- | ---- | +| `name` | Provide the reference name for the metric. This name must be a unique metric name and can consist of lowercase letters, numbers, and underscores. | Required | String | +| `description` | Describe your metric. | Optional | String | +| `type` | Define the type of metric, which can be `conversion`, `cumulative`, `derived`, `ratio`, or `simple`. | Required | String | +| `type_params` | Additional parameters used to configure metrics. `type_params` are different for each metric type. | Required | Dict | +| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | String | +| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports [`meta`](/reference/resource-configs/meta), [`group`](/reference/resource-configs/group), and [`enabled`](/reference/resource-configs/enabled) configurations. | Optional | Dict | +| `filter` | You can optionally add a [filter](#filters) string to any metric type, applying filters to dimensions, entities, time dimensions, or other metrics during metric computation. Consider it as your WHERE clause. | Optional | String | Here's a complete example of the metrics spec configuration: @@ -52,16 +52,16 @@ metrics: -| Parameter | Description | Type | -| --------- | ----------- | ---- | -| `name` | Provide the reference name for the metric. This name must be unique amongst all metrics. | Required | -| `description` | Describe your metric. | Optional | -| `type` | Define the type of metric, which can be `simple`, `ratio`, `cumulative`, or `derived`. | Required | -| `type_params` | Additional parameters used to configure metrics. `type_params` are different for each metric type. | Required | -| `config` | Provide the specific configurations for your metric. | Optional | -| `meta` | Use the [`meta` config](/reference/resource-configs/meta) to set metadata for a resource. | Optional | -| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | -| `filter` | You can optionally add a filter string to any metric type, applying filters to dimensions, entities, or time dimensions during metric computation. Consider it as your WHERE clause. | Optional | +| Parameter | Description | Required | Type | +| --------- | ----------- | ---- | ---- | +| `name` | Provide the reference name for the metric. This name must be unique amongst all metrics. | Required | String | +| `description` | Describe your metric. | Optional | String | +| `type` | Define the type of metric, which can be `simple`, `ratio`, `cumulative`, or `derived`. | Required | String | +| `type_params` | Additional parameters used to configure metrics. `type_params` are different for each metric type. | Required | Dict | +| `config` | Provide the specific configurations for your metric. | Optional | Dict | +| `meta` | Use the [`meta` config](/reference/resource-configs/meta) to set metadata for a resource. | Optional | String | +| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | String | +| `filter` | You can optionally add a filter string to any metric type, applying filters to dimensions, entities, or time dimensions during metric computation. Consider it as your WHERE clause. | Optional | String | Here's a complete example of the metrics spec configuration: diff --git a/website/docs/docs/build/python-models.md b/website/docs/docs/build/python-models.md index 28136f91e9c..c3222fb76b8 100644 --- a/website/docs/docs/build/python-models.md +++ b/website/docs/docs/build/python-models.md @@ -598,6 +598,34 @@ Python models have capabilities that SQL models do not. They also have some draw - **These capabilities are very new.** As data warehouses develop new features, we expect them to offer cheaper, faster, and more intuitive mechanisms for deploying Python transformations. **We reserve the right to change the underlying implementation for executing Python models in future releases.** Our commitment to you is around the code in your model `.py` files, following the documented capabilities and guidance we're providing here. - **Lack of `print()` support.** The data platform runs and compiles your Python model without dbt's oversight. This means it doesn't display the output of commands such as Python's built-in [`print()`](https://docs.python.org/3/library/functions.html#print) function in dbt's logs. +- + + The following explains other methods you can use for debugging, such as writing messages to a dataframe column: + + - Using platform logs: Use your data platform's logs to debug your Python models. + - Return logs as a dataframe: Create a dataframe containing your logs and build it into the warehouse. + - Develop locally with DuckDB: Test and debug your models locally using DuckDB before deploying them. + + Here's an example of debugging in a Python model: + + ```python + def model(dbt, session): + dbt.config( + materialized = "table" + ) + + df = dbt.ref("my_source_table").df() + + # One option for debugging: write messages to temporary table column + # Pros: visibility + # Cons: won't work if table isn't building for some reason + msg = "something" + df["debugging"] = f"My debug message here: {msg}" + + return df + ``` + + As a general rule, if there's a transformation you could write equally well in SQL or Python, we believe that well-written SQL is preferable: it's more accessible to a greater number of colleagues, and it's easier to write code that's performant at scale. If there's a transformation you _can't_ write in SQL, or where ten lines of elegant and well-annotated Python could save you 1000 lines of hard-to-read Jinja-SQL, Python is the way to go. ## Specific data platforms {#specific-data-platforms} @@ -645,18 +673,7 @@ def model(dbt, session: snowpark.Session): -**About "sprocs":** dbt submits Python models to run as _stored procedures_, which some people call _sprocs_ for short. By default, dbt will create a named sproc containing your model's compiled Python code, and then _call_ it to execute. Snowpark has an Open Preview feature for _temporary_ or _anonymous_ stored procedures ([docs](https://docs.snowflake.com/en/sql-reference/sql/call-with.html)), which are faster and leave a cleaner query history. You can switch this feature on for your models by configuring `use_anonymous_sproc: True`. We plan to switch this on for all dbt + Snowpark Python models starting with the release of dbt Core version 1.4. - - - -```yml -# I asked Snowflake Support to enable this Private Preview feature, -# and now my dbt-py models run even faster! -models: - use_anonymous_sproc: True -``` - - +**About "sprocs":** dbt submits Python models to run as _stored procedures_, which some people call _sprocs_ for short. By default, dbt will use Snowpark's _temporary_ or _anonymous_ stored procedures ([docs](https://docs.snowflake.com/en/sql-reference/sql/call-with.html)), which are faster and keep query history cleaner than named sprocs containing your model's compiled Python code. To disable this feature, set `use_anonymous_sproc: False` in your model configuration. **Docs:** ["Developer Guide: Snowpark Python"](https://docs.snowflake.com/en/developer-guide/snowpark/python/index.html) diff --git a/website/docs/docs/build/ratio-metrics.md b/website/docs/docs/build/ratio-metrics.md index fdaeb878450..a34dec29d71 100644 --- a/website/docs/docs/build/ratio-metrics.md +++ b/website/docs/docs/build/ratio-metrics.md @@ -10,17 +10,17 @@ Ratio allows you to create a ratio between two metrics. You simply specify a num The parameters, description, and type for ratio metrics are: -| Parameter | Description | Type | -| --------- | ----------- | ---- | -| `name` | The name of the metric. | Required | -| `description` | The description of the metric. | Optional | -| `type` | The type of the metric (cumulative, derived, ratio, or simple). | Required | -| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | -| `type_params` | The type parameters of the metric. | Required | -| `numerator` | The name of the metric used for the numerator, or structure of properties. | Required | -| `denominator` | The name of the metric used for the denominator, or structure of properties. | Required | -| `filter` | Optional filter for the numerator or denominator. | Optional | -| `alias` | Optional alias for the numerator or denominator. | Optional | +| Parameter | Description | Required | Type | +| --------- | ----------- | ---- | ---- | +| `name` | The name of the metric. | Required | String | +| `description` | The description of the metric. | Optional | String | +| `type` | The type of the metric (cumulative, derived, ratio, or simple). | Required | String | +| `label` | Defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | String | +| `type_params` | The type parameters of the metric. | Required | Dict | +| `numerator` | The name of the metric used for the numerator, or structure of properties. | Required | String or dict | +| `denominator` | The name of the metric used for the denominator, or structure of properties. | Required | String or dict | +| `filter` | Optional filter for the numerator or denominator. | Optional | String | +| `alias` | Optional alias for the numerator or denominator. | Optional | String | The following displays the complete specification for ratio metrics, along with an example. diff --git a/website/docs/docs/build/semantic-models.md b/website/docs/docs/build/semantic-models.md index 609d7f1ff8d..5ff363dd44c 100644 --- a/website/docs/docs/build/semantic-models.md +++ b/website/docs/docs/build/semantic-models.md @@ -26,18 +26,18 @@ import SLCourses from '/snippets/\_sl-course.md'; Here we describe the Semantic model components with examples: -| Component | Description | Type | -| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | -| [Name](#name) | Choose a unique name for the semantic model. Avoid using double underscores (\_\_) in the name as they're not supported. | Required | -| [Description](#description) | Includes important details in the description | Optional | -| [Model](#model) | Specifies the dbt model for the semantic model using the `ref` function | Required | -| [Defaults](#defaults) | The defaults for the model, currently only `agg_time_dimension` is supported. | Required | -| [Entities](#entities) | Uses the columns from entities as join keys and indicate their type as primary, foreign, or unique keys with the `type` parameter | Required | -| [Primary Entity](#primary-entity) | If a primary entity exists, this component is Optional. If the semantic model has no primary entity, then this property is required. | Optional | -| [Dimensions](#dimensions) | Different ways to group or slice data for a metric, they can be `time` or `categorical` | Required | -| [Measures](#measures) | Aggregations applied to columns in your data model. They can be the final metric or used as building blocks for more complex metrics | Optional | -| Label | The display name for your semantic model `node`, `dimension`, `entity`, and/or `measures` | Optional | -| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports [`meta`](/reference/resource-configs/meta), [`group`](/reference/resource-configs/group), and [`enabled`](/reference/resource-configs/enabled) configs. | Optional | +| Component | Description | Required | Type | +| ------------ | ---------------- | -------- | -------- | +| [Name](#name) | Choose a unique name for the semantic model. Avoid using double underscores (\_\_) in the name as they're not supported. | Required | String | +| [Description](#description) | Includes important details in the description. | Optional | String | +| [Model](#model) | Specifies the dbt model for the semantic model using the `ref` function. | Required | String | +| [Defaults](#defaults) | The defaults for the model, currently only `agg_time_dimension` is supported. | Required | Dict | +| [Entities](#entities) | Uses the columns from entities as join keys and indicate their type as primary, foreign, or unique keys with the `type` parameter. | Required | List | +| [Primary Entity](#primary-entity) | If a primary entity exists, this component is Optional. If the semantic model has no primary entity, then this property is required. | Optional | String | +| [Dimensions](#dimensions) | Different ways to group or slice data for a metric, they can be `time` or `categorical`. | Required | List | +| [Measures](#measures) | Aggregations applied to columns in your data model. They can be the final metric or used as building blocks for more complex metrics. | Optional | List | +| [Label](#label) | The display name for your semantic model `node`, `dimension`, `entity`, and/or `measures`. | Optional | String | +| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports [`meta`](/reference/resource-configs/meta), [`group`](/reference/resource-configs/group), and [`enabled`](/reference/resource-configs/enabled) configs. | Optional | Dict | ## Semantic models components diff --git a/website/docs/docs/build/simple.md b/website/docs/docs/build/simple.md index f57d498d290..872a4d34fa8 100644 --- a/website/docs/docs/build/simple.md +++ b/website/docs/docs/build/simple.md @@ -15,17 +15,17 @@ Simple metrics are metrics that directly reference a single measure, without any Note that we use the double colon (::) to indicate whether a parameter is nested within another parameter. So for example, `query_params::metrics` means the `metrics` parameter is nested under `query_params`. ::: -| Parameter | Description | Type | -| --------- | ----------- | ---- | -| `name` | The name of the metric. | Required | -| `description` | The description of the metric. | Optional | -| `type` | The type of the metric (cumulative, derived, ratio, or simple). | Required | -| `label` | Required string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | -| `type_params` | The type parameters of the metric. | Required | -| `measure` | A list of measure inputs | Required | -| `measure:name` | The measure you're referencing. | Required | -| `measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | -| `measure:join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | +| Parameter | Description | Required | Type | +| --------- | ----------- | ---- | ---- | +| `name` | The name of the metric. | Required | String | +| `description` | The description of the metric. | Optional | String | +| `type` | The type of the metric (cumulative, derived, ratio, or simple). | Required | String | +| `label` | Defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Required | String | +| `type_params` | The type parameters of the metric. | Required | Dict | +| `measure` | A list of measure inputs. | Required | List | +| `measure:name` | The measure you're referencing. | Required | String | +| `measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | String | +| `measure:join_to_timespine` | Indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | Boolean | The following displays the complete specification for simple metrics, along with an example. diff --git a/website/docs/docs/cloud-integrations/configure-auto-exposures.md b/website/docs/docs/cloud-integrations/configure-auto-exposures.md index 42e36e572b3..746bef62e44 100644 --- a/website/docs/docs/cloud-integrations/configure-auto-exposures.md +++ b/website/docs/docs/cloud-integrations/configure-auto-exposures.md @@ -25,6 +25,8 @@ To access the features, you should meet the following: 3. You have set up a [production](/docs/deploy/deploy-environments#set-as-production-environment) deployment environment for each project you want to explore, with at least one successful job run. 4. You have [admin permissions](/docs/cloud/manage-access/enterprise-permissions) in dbt Cloud to edit project settings or production environment settings. 5. Use Tableau as your BI tool and enable metadata permissions or work with an admin to do so. Compatible with Tableau Cloud or Tableau Server with the Metadata API enabled. + - If you're using Tableau Server, you need to [allowlist dbt Cloud's IP addresses](/docs/cloud/about-cloud/access-regions-ip-addresses) for your dbt Cloud region. + - Currently, you can only connect to a single Tableau site on the same server. ## Set up in Tableau diff --git a/website/docs/docs/cloud/manage-access/auth0-migration.md b/website/docs/docs/cloud/manage-access/auth0-migration.md index b7bab836810..2f45ad7dcc8 100644 --- a/website/docs/docs/cloud/manage-access/auth0-migration.md +++ b/website/docs/docs/cloud/manage-access/auth0-migration.md @@ -5,22 +5,10 @@ sidebar: "SSO Auth0 Migration" description: "Required actions for migrating to Auth0 for SSO services on dbt Cloud." --- -:::note - -This migration is a feature of the dbt Cloud Enterprise plan. To learn more about an Enterprise plan, contact us at [sales@getdbt.com](mailto::sales@getdbt.com). - -For single-tenant Virtual Private Cloud, you should [email dbt Cloud Support](mailto::support@getdbt.com) to set up or update your SSO configuration. - -::: - dbt Labs is partnering with Auth0 to bring enhanced features to dbt Cloud's single sign-on (SSO) capabilities. Auth0 is an identity and access management (IAM) platform with advanced security features, and it will be leveraged by dbt Cloud. These changes will require some action from customers with SSO configured in dbt Cloud today, and this guide will outline the necessary changes for each environment. If you have not yet configured SSO in dbt Cloud, refer instead to our setup guides for [SAML](/docs/cloud/manage-access/set-up-sso-saml-2.0), [Okta](/docs/cloud/manage-access/set-up-sso-okta), [Google Workspace](/docs/cloud/manage-access/set-up-sso-google-workspace), or [Microsoft Entra ID (formerly Azure AD)](/docs/cloud/manage-access/set-up-sso-microsoft-entra-id) single sign-on services. -## Auth0 Multi-tenant URIs - - - ## Start the migration The Auth0 migration feature is being rolled out incrementally to customers who have SSO features already enabled. When the migration option has been enabled on your account, you will see **SSO Updates Available** on the right side of the menu bar, near the settings icon. diff --git a/website/docs/docs/cloud/manage-access/environment-permissions.md b/website/docs/docs/cloud/manage-access/environment-permissions.md index b99da64609c..20acfae51f7 100644 --- a/website/docs/docs/cloud/manage-access/environment-permissions.md +++ b/website/docs/docs/cloud/manage-access/environment-permissions.md @@ -17,8 +17,8 @@ Environment-level permissions give dbt Cloud admins more flexibility to protect - Environment-level permissions do not allow you to create custom roles and permissions for each resource type in dbt Cloud. - You can only select environment types, and can’t specify a particular environment within a project. -- You can't select specific resources within environments. dbt Cloud jobs, runs, and environment variables are all environment resources. - - For example, you can't specify that a user only has access to jobs but not environment variables. Access to a given environment gives the user access to everything within that environment. +- You can't select specific resources within environments. dbt Cloud jobs and runs are environment resources. + - For example, you can't specify that a user only has access to jobs but not runs. Access to a given environment gives the user access to everything within that environment. ## Environments and roles diff --git a/website/docs/docs/collaborate/auto-exposures.md b/website/docs/docs/collaborate/auto-exposures.md index 9b25a2fb305..28bf5bd37b1 100644 --- a/website/docs/docs/collaborate/auto-exposures.md +++ b/website/docs/docs/collaborate/auto-exposures.md @@ -9,11 +9,16 @@ image: /img/docs/cloud-integrations/auto-exposures/explorer-lineage.jpg # Auto-exposures -As a data team, it’s critical that you have context into the downstream use cases and users of your data products. Auto-exposures integrates natively with Tableau (Power BI coming soon) and auto-generates downstream lineage in dbt Explorer for a richer experience. +As a data team, it’s critical that you have context into the downstream use cases and users of your data products. Auto-exposures integrate natively with Tableau (Power BI coming soon) and auto-generate downstream lineage in dbt Explorer for a richer experience. -Auto-exposures helps users understand how their models are used in downstream analytics tools to inform investments and reduce incidents — ultimately building trust and confidence in data products. It imports and auto-generates exposures based on Tableau dashboards, with user-defined curation. +Auto-exposures help users understand how their models are used in downstream analytics tools to inform investments and reduce incidents — ultimately building trust and confidence in data products. It imports and auto-generates exposures based on Tableau dashboards, with user-defined curation. -Auto-exposures is available on [Versionless](/docs/dbt-versions/versionless-cloud) and on [dbt Cloud Enterprise](https://www.getdbt.com/pricing/) plans. +## Supported plans +Auto-exposures is available on [Versionless](/docs/dbt-versions/versionless-cloud) and [dbt Cloud Enterprise](https://www.getdbt.com/pricing/) plans. Currently, you can only connect to a single Tableau site on the same server. + +:::info Tableau Server +If you're using Tableau Server, you need to [allowlist dbt Cloud's IP addresses](/docs/cloud/about-cloud/access-regions-ip-addresses) for your dbt Cloud region. +::: For more information on how to set up auto-exposures, prerequisites, and more — refer to [configure auto-exposures in Tableau and dbt Cloud](/docs/cloud-integrations/configure-auto-exposures). diff --git a/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md b/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md index 7ac5a743995..8b809877870 100644 --- a/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md +++ b/website/docs/docs/dbt-versions/core-upgrade/06-upgrading-to-v1.9.md @@ -1,5 +1,5 @@ --- -title: "Upgrading to v1.9 (beta)" +title: "Upgrading to v1.9" id: upgrading-to-v1.9 description: New features and changes in dbt Core v1.9 displayed_sidebar: "docs" @@ -29,7 +29,8 @@ Features and functionality new in dbt v1.9. ### Microbatch `incremental_strategy` :::info -While microbatch is in "beta", this functionality is still gated behind an env var, which will change to a behavior flag when 1.9 is GA. To use microbatch, set `DBT_EXPERIMENTAL_MICROBATCH` to `true` wherever you're running dbt Core. + +If you use a custom microbatch macro, set the [`require_batched_execution_for_custom_microbatch_strategy`](/reference/global-configs/behavior-changes#custom-microbatch-strategy) behavior flag in your `dbt_project.yml` to enable batched execution. If you don't have a custom microbatch macro, you don't need to set this flag as dbt will handle microbatching automatically for any model using the microbatch strategy. ::: Incremental models are, and have always been, a *performance optimization* — for datasets that are too large to be dropped and recreated from scratch every time you do a `dbt run`. Learn more about [incremental models](/docs/build/incremental-models-overview). @@ -83,6 +84,7 @@ You can read more about each of these behavior changes in the following links: - (Introduced, disabled by default) [`skip_nodes_if_on_run_start_fails` project config flag](/reference/global-configs/behavior-changes#behavior-change-flags). If the flag is set and **any** `on-run-start` hook fails, mark all selected nodes as skipped. - `on-run-start/end` hooks are **always** run, regardless of whether they passed or failed last time. - (Introduced, disabled by default) [[Redshift] `restrict_direct_pg_catalog_access`](/reference/global-configs/behavior-changes#redshift-restrict_direct_pg_catalog_access). If the flag is set the adapter will use the Redshift API (through the Python client) if available, or query Redshift's `information_schema` tables instead of using `pg_` tables. +- (Introduced, disabled by default) [`require_batched_execution_for_custom_microbatch_strategy`](/reference/global-configs/behavior-changes#custom-microbatch-strategy). Set to `True` if you use a custom microbatch macro to enable batched execution. If you don't have a custom microbatch macro, you don't need to set this flag as dbt will handle microbatching automatically for any model using the microbatch strategy. ## Adapter specific features and functionalities @@ -92,7 +94,7 @@ You can read more about each of these behavior changes in the following links: ### Snowflake -- Iceberg Table Format support will be available on three out of the box materializations: table, incremental, dynamic tables. +- Iceberg Table Format support will be available on three out-of-the-box materializations: table, incremental, dynamic tables. ### Bigquery @@ -107,7 +109,7 @@ You can read more about each of these behavior changes in the following links: We also made some quality-of-life improvements in Core 1.9, enabling you to: -- Maintain data quality now that dbt returns an an error (versioned models) or warning (unversioned models) when someone [removes a contracted model by deleting, renaming, or disabling](/docs/collaborate/govern/model-contracts#how-are-breaking-changes-handled) it. +- Maintain data quality now that dbt returns an error (versioned models) or warning (unversioned models) when someone [removes a contracted model by deleting, renaming, or disabling](/docs/collaborate/govern/model-contracts#how-are-breaking-changes-handled) it. - Document [data tests](/reference/resource-properties/description). - Use `ref` and `source` in [foreign key constraints](/reference/resource-properties/constraints). - Use `dbt test` with the `--resource-type` / `--exclude-resource-type` flag, making it possible to include or exclude data tests (`test`) or unit tests (`unit_test`). diff --git a/website/docs/docs/dbt-versions/release-notes.md b/website/docs/docs/dbt-versions/release-notes.md index 3cce35d9556..55116db68ba 100644 --- a/website/docs/docs/dbt-versions/release-notes.md +++ b/website/docs/docs/dbt-versions/release-notes.md @@ -19,7 +19,8 @@ Release notes are grouped by month for both multi-tenant and virtual private clo \* The official release date for this new format of release notes is May 15th, 2024. Historical release notes for prior dates may not reflect all available features released earlier this year or their tenancy availability. ## November 2024 - +- **Fix**: Job environment variable overrides in credentials are now respected for Exports. Previously, they were ignored. +- **Behavior change**: If you use a custom microbatch macro, set a [`require_batched_execution_for_custom_microbatch_strategy` behavior flag](/reference/global-configs/behavior-changes#custom-microbatch-strategy) in your `dbt_project.yml` to enable batched execution. If you don't have a custom microbatch macro, you don't need to set this flag as dbt will handle microbatching automatically for any model using the [microbatch strategy](/docs/build/incremental-microbatch#how-microbatch-compares-to-other-incremental-strategies). - **Enhancement**: For users that have Advanced CI's [compare changes](/docs/deploy/advanced-ci#compare-changes) feature enabled, you can optimize performance when running comparisons by using custom dbt syntax to customize deferral usage, exclude specific large models (or groups of models with tags), and more. Refer to [Compare changes custom commands](/docs/deploy/job-commands#compare-changes-custom-commands) for examples of how to customize the comparison command. - **New**: SQL linting in CI jobs is now generally available in dbt Cloud. You can enable SQL linting in your CI jobs, using [SQLFluff](https://sqlfluff.com/), to automatically lint all SQL files in your project as a run step before your CI job builds. SQLFluff linting is available on [dbt Cloud Versionless](/docs/dbt-versions/versionless-cloud) and to dbt Cloud [Team or Enterprise](https://www.getdbt.com/pricing/) accounts. Refer to [SQL linting](/docs/deploy/continuous-integration#sql-linting) for more information. - **New**: Use the [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current) config to set a custom indicator for the value of `dbt_valid_to` in current snapshot records (like a future date). By default, this value is `NULL`. When configured, dbt will use the specified value instead of `NULL` for `dbt_valid_to` for current records in the snapshot table. This feature is available in dbt Cloud Versionless and dbt Core v1.9 and later. diff --git a/website/docs/docs/deploy/merge-jobs.md b/website/docs/docs/deploy/merge-jobs.md index 8b2900661fa..a187e3992f8 100644 --- a/website/docs/docs/deploy/merge-jobs.md +++ b/website/docs/docs/deploy/merge-jobs.md @@ -5,7 +5,7 @@ description: "Learn how to trigger a dbt job run when a Git pull request merges. --- -You can set up a merge job to implement a continuous development (CD) workflow in dbt Cloud. The merge job triggers a dbt job to run when someone merges Git pull requests into production. This workflow creates a seamless development experience where changes made in code will automatically update production data. Also, you can use this workflow for running `dbt compile` to update your environment's manifest so subsequent CI job runs are more performant. +You can set up a merge job to implement a continuous deployment (CD) workflow in dbt Cloud. The merge job triggers a dbt job to run when someone merges Git pull requests into production. This workflow creates a seamless development experience where changes made in code will automatically update production data. Also, you can use this workflow for running `dbt compile` to update your environment's manifest so subsequent CI job runs are more performant. By using CD in dbt Cloud, you can take advantage of deferral to build only the edited model and any downstream changes. With merge jobs, state will be updated almost instantly, always giving the most up-to-date state information in [dbt Explorer](/docs/collaborate/explore-projects). @@ -62,4 +62,4 @@ The following is an example of creating a new **Code pushed** trigger in Azure D - \ No newline at end of file + diff --git a/website/docs/faqs/Troubleshooting/failed-snowflake-oauth-connection.md b/website/docs/faqs/Troubleshooting/failed-snowflake-oauth-connection.md index 5d2a2f671c9..84ef49d212c 100644 --- a/website/docs/faqs/Troubleshooting/failed-snowflake-oauth-connection.md +++ b/website/docs/faqs/Troubleshooting/failed-snowflake-oauth-connection.md @@ -4,19 +4,28 @@ description: "Edit your OAuth Security integration when you see error" sidebar_label: 'Receiving `Failed to connect to database` error' --- -1. If you see this error: +1. If you see the following error: - ```shell - + ```text Failed to connect to DB: xxxxxxx.snowflakecomputing.com:443. The role requested in the connection, or the default role if none was requested in the connection ('xxxxx'), is not listed in the Access Token or was filtered. Please specify another role, or contact your OAuth Authorization server administrator. - ``` 2. Edit your OAuth Security integration and explicitly specify this scope mapping attribute: - ```sql -ALTER INTEGRATION SET EXTERNAL_OAUTH_SCOPE_MAPPING_ATTRIBUTE = 'scp'; -``` + ```sql + ALTER INTEGRATION SET EXTERNAL_OAUTH_SCOPE_MAPPING_ATTRIBUTE = 'scp'; + ``` You can read more about this error in [Snowflake's documentation](https://community.snowflake.com/s/article/external-custom-oauth-error-the-role-requested-in-the-connection-is-not-listed-in-the-access-token). + +---- + +1. If you see the following error: + + ```text + Failed to connect to DB: xxxxxxx.snowflakecomputing.com:443. Incorrect username or password was specified. + ``` + + * **Unique email addresses** — Each user in Snowflake must have a unique email address. You can't have multiple users (for example, a human user and a service account) using the same email, such as `alice@acme.com`, to authenticate to Snowflake. + * **Match email addresses with identity provider** — The email address of your Snowflake user must exactly match the email address you use to authenticate with your Identity Provider (IdP). For example, if your Snowflake user's email is `alice@acme.com` but you log in to Entra or Okta with `alice_adm@acme.com`, this mismatch can cause an error. diff --git a/website/docs/reference/dbt-classes.md b/website/docs/reference/dbt-classes.md index 13f9263e545..a6a8c2d4fa6 100644 --- a/website/docs/reference/dbt-classes.md +++ b/website/docs/reference/dbt-classes.md @@ -98,9 +98,14 @@ col.numeric_type('numeric', 12, 4) # numeric(12,4) ### Properties -- **name**: Returns the name of the column +- **char_size**: Returns the maximum size for character varying columns +- **column**: Returns the name of the column +- **data_type**: Returns the data type of the column (with size/precision/scale included) +- **dtype**: Returns the data type of the column (without any size/precision/scale included) +- **name**: Returns the name of the column (identical to `column`, provided as an alias). +- **numeric_precision**: Returns the maximum precision for fixed decimal columns +- **numeric_scale**: Returns the maximum scale for fixed decimal columns - **quoted**: Returns the name of the column wrapped in quotes -- **data_type**: Returns the data type of the column ### Instance methods diff --git a/website/docs/reference/global-configs/behavior-changes.md b/website/docs/reference/global-configs/behavior-changes.md index c980e71e9f0..c19fc14be97 100644 --- a/website/docs/reference/global-configs/behavior-changes.md +++ b/website/docs/reference/global-configs/behavior-changes.md @@ -59,6 +59,7 @@ flags: source_freshness_run_project_hooks: False restrict_direct_pg_catalog_access: False require_yaml_configuration_for_mf_time_spines: False + require_batched_execution_for_custom_microbatch_strategy: False ``` @@ -74,6 +75,7 @@ When we use dbt Cloud in the following table, we're referring to accounts that h | [skip_nodes_if_on_run_start_fails](#failures-in-on-run-start-hooks) | 2024.10 | TBD* | 1.9.0 | TBD* | | [state_modified_compare_more_unrendered_values](#source-definitions-for-state) | 2024.10 | TBD* | 1.9.0 | TBD* | | [require_yaml_configuration_for_mf_time_spines](#metricflow-time-spine-yaml) | 2024.10 | TBD* | 1.9.0 | TBD* | +| [require_batched_execution_for_custom_microbatch_strategy](#custom-microbatch-strategy) | 2024.11 | TBD* | 1.9.0 | TBD* | When the dbt Cloud Maturity is "TBD," it means we have not yet determined the exact date when these flags' default values will change. Affected users will see deprecation warnings in the meantime, and they will receive emails providing advance warning ahead of the maturity date. In the meantime, if you are seeing a deprecation warning, you can either: - Migrate your project to support the new behavior, and then set the flag to `True` to stop seeing the warnings. @@ -164,3 +166,12 @@ In previous versions (dbt Core 1.8 and earlier), the MetricFlow time spine confi When the flag is set to `True`, dbt will continue to support the SQL file configuration. When the flag is set to `False`, dbt will raise a deprecation warning if it detects a MetricFlow time spine configured in a SQL file. The MetricFlow YAML file should have the `time_spine:` field. Refer to [MetricFlow timespine](/docs/build/metricflow-time-spine) for more details. + +### Custom microbatch strategy +The `require_batched_execution_for_custom_microbatch_strategy` flag is set to `False` by default and is only relevant if you already have a custom microbatch macro in your project. If you don't have a custom microbatch macro, you don't need to set this flag as dbt will handle microbatching automatically for any model using the [microbatch strategy](/docs/build/incremental-microbatch#how-microbatch-compares-to-other-incremental-strategies). + +Set the flag is set to `True` if you have a custom microbatch macro set up in your project. When the flag is set to `True`, dbt will execute the custom microbatch strategy in batches. + +If you have a custom microbatch macro and the flag is left as `False`, dbt will issue a deprecation warning. + +Previously, users needed to set the `DBT_EXPERIMENTAL_MICROBATCH` environment variable to `True` to prevent unintended interactions with existing custom incremental strategies. But this is no longer necessary, as setting `DBT_EXPERMINENTAL_MICROBATCH` will no longer have an effect on runtime functionality. diff --git a/website/docs/reference/global-configs/logs.md b/website/docs/reference/global-configs/logs.md index 682b9fc8393..85969a5bc02 100644 --- a/website/docs/reference/global-configs/logs.md +++ b/website/docs/reference/global-configs/logs.md @@ -66,19 +66,28 @@ See [structured logging](/reference/events-logging#structured-logging) for more The `LOG_LEVEL` config sets the minimum severity of events captured in the console and file logs. This is a more flexible alternative to the `--debug` flag. The available options for the log levels are `debug`, `info`, `warn`, `error`, or `none`. -Setting the `--log-level` will configure console and file logs. +- Setting the `--log-level` will configure console and file logs. + ```text + dbt --log-level debug run + ``` -```text -dbt --log-level debug run -``` +- Setting the `LOG_LEVEL` to `none` will disable information from being sent to either the console or file logs. + + ```text + dbt --log-level none + ``` -To set the file log level as a different value than the console, use the `--log-level-file` flag. +- To set the file log level as a different value than the console, use the `--log-level-file` flag. + ```text + dbt --log-level-file error run + ``` -```text -dbt --log-level-file error run -``` +- To only disable writing to the logs file but keep console logs, set `LOG_LEVEL_FILE` config to none. + ```text + dbt --log-level-file none + ``` ### Debug-level logging diff --git a/website/docs/reference/project-configs/query-comment.md b/website/docs/reference/project-configs/query-comment.md index 7e654350306..f7f9472e947 100644 --- a/website/docs/reference/project-configs/query-comment.md +++ b/website/docs/reference/project-configs/query-comment.md @@ -30,7 +30,7 @@ query-comment: ## Definition -A string to inject as a comment in each query that dbt runs against your database. This comment can be used to attribute SQL statements to specific dbt resources like models and tests. +A string to inject as a comment in each query that dbt runs against your database. This comment can attribute SQL statements to specific dbt resources like models and tests. The `query-comment` configuration can also call a macro that returns a string. @@ -51,7 +51,7 @@ create view analytics.analytics.orders as ( ## Using the dictionary syntax The dictionary syntax includes two keys: - * `comment` (optional, see above for default): The string to be injected to a query as a comment. + * `comment` (optional, for more information, refer to the [default](#default) section): The string to be injected into a query as a comment. * `append` (optional, default=`false`): Whether a comment should be appended (added to the bottom of a query) or not (i.e. added to the top of a query). By default, comments are added to the top of queries (i.e. `append: false`). This syntax is useful on databases like Snowflake which [remove leading SQL comments](https://docs.snowflake.com/en/release-notes/2017-04.html#queries-leading-comments-removed-during-execution). @@ -275,4 +275,6 @@ The following context variables are available when generating a query comment: | var | See [var](/reference/dbt-jinja-functions/var) | | target | See [target](/reference/dbt-jinja-functions/target) | | connection_name | A string representing the internal name for the connection. This string is generated by dbt. | -| node | A dictionary representation of the parsed node object. Use `node.unique_id`, `node.database`, `node.schema`, etc | +| node | A dictionary representation of the parsed node object. Use `node.unique_id`, `node.database`, `node.schema`, and so on. | + +Note: The `var()` function in `query-comment` macros only access variables passed through the `--vars` argument in the CLI. Variables defined in the vars block of your `dbt_project.yml` are not accessible when generating query comments. diff --git a/website/docs/reference/resource-configs/schema.md b/website/docs/reference/resource-configs/schema.md index 1e2ff47729c..b239e26bd87 100644 --- a/website/docs/reference/resource-configs/schema.md +++ b/website/docs/reference/resource-configs/schema.md @@ -108,7 +108,9 @@ This would result in the test results being stored in the `test_results` schema. Refer to [Usage](#usage) for more examples. ## Definition -Optionally specify a custom schema for a [model](/docs/build/sql-models) or [seed](/docs/build/seeds). (To specify a schema for a [snapshot](/docs/build/snapshots), use the [`target_schema` config](/reference/resource-configs/target_schema)). +Optionally specify a custom schema for a [model](/docs/build/sql-models), [seed](/docs/build/seeds), [snapshot](/docs/build/snapshots), [saved query](/docs/build/saved-queries), or [test](/docs/build/data-tests). + +For users on dbt Cloud v1.8 or earlier, use the [`target_schema` config](/reference/resource-configs/target_schema) to specify a custom schema for a snapshot. When dbt creates a relation (/) in a database, it creates it as: `{{ database }}.{{ schema }}.{{ identifier }}`, e.g. `analytics.finance.payments` diff --git a/website/snippets/_enterprise-permissions-table.md b/website/snippets/_enterprise-permissions-table.md index a5b825d34d2..b39337697c1 100644 --- a/website/snippets/_enterprise-permissions-table.md +++ b/website/snippets/_enterprise-permissions-table.md @@ -104,7 +104,7 @@ Key: | Custom env. variables | W | W | W | W | W | W | - | R | - | - | R | W | - | | Data platform configs | W | W | W | W | R | W | - | - | - | - | R | R | - | | Develop (IDE or CLI) | W | W | - | W | - | - | - | - | - | - | - | - | - | -| Environments | W | R* | R* | R* | R* | W | - | R | - | - | R | R* | - | +| Environments | W | R | R | R | R | W | - | R | - | - | R | R | - | | Jobs | W | R* | R* | R* | R* | W | R | R | - | - | R | R* | - | | Metadata GraphQL API access| R | R | R | R | R | R | - | R | R | - | R | R | - | | Permissions | W | - | R | R | R | - | - | - | - | - | - | R | - | diff --git a/website/snippets/_sl-measures-parameters.md b/website/snippets/_sl-measures-parameters.md index 728d63c6b4f..8d6b84a71dd 100644 --- a/website/snippets/_sl-measures-parameters.md +++ b/website/snippets/_sl-measures-parameters.md @@ -1,11 +1,11 @@ -| Parameter | Description | | -| --- | --- | --- | -| [`name`](/docs/build/measures#name) | Provide a name for the measure, which must be unique and can't be repeated across all semantic models in your dbt project. | Required | -| [`description`](/docs/build/measures#description) | Describes the calculated measure. | Optional | -| [`agg`](/docs/build/measures#aggregation) | dbt supports the following aggregations: `sum`, `max`, `min`, `average`, `median`, `count_distinct`, `percentile`, and `sum_boolean`. | Required | -| [`expr`](/docs/build/measures#expr) | Either reference an existing column in the table or use a SQL expression to create or derive a new one. | Optional | -| [`non_additive_dimension`](/docs/build/measures#non-additive-dimensions) | Non-additive dimensions can be specified for measures that cannot be aggregated over certain dimensions, such as bank account balances, to avoid producing incorrect results. | Optional | -| `agg_params` | Specific aggregation properties, such as a percentile. | Optional | -| `agg_time_dimension` | The time field. Defaults to the default agg time dimension for the semantic model. | Optional | 1.6 and higher | -| `label` | String that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as orders_total or "orders_total"). Available in dbt version 1.7 or higher. | Optional -| `create_metric` | Create a `simple` metric from a measure by setting `create_metric: True`. The `label` and `description` attributes will be automatically propagated to the created metric. Available in dbt version 1.7 or higher. | Optional | +| Parameter | Description | Required | Type | +| --- | --- | --- | --- | +| [`name`](/docs/build/measures#name) | Provide a name for the measure, which must be unique and can't be repeated across all semantic models in your dbt project. | Required | String | +| [`description`](/docs/build/measures#description) | Describes the calculated measure. | Optional | String | +| [`agg`](/docs/build/measures#aggregation) | dbt supports the following aggregations: `sum`, `max`, `min`, `average`, `median`, `count_distinct`, `percentile`, and `sum_boolean`. | Required | String | +| [`expr`](/docs/build/measures#expr) | Either reference an existing column in the table or use a SQL expression to create or derive a new one. | Optional | String | +| [`non_additive_dimension`](/docs/build/measures#non-additive-dimensions) | Non-additive dimensions can be specified for measures that cannot be aggregated over certain dimensions, such as bank account balances, to avoid producing incorrect results. | Optional | String | +| `agg_params` | Specific aggregation properties, such as a percentile. | Optional | Dict | +| `agg_time_dimension` | The time field. Defaults to the default agg time dimension for the semantic model. | Optional | String | +| `label` | String that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). Available in dbt version 1.7 or higher. | Optional | String | +| `create_metric` | Create a `simple` metric from a measure by setting `create_metric: True`. The `label` and `description` attributes will be automatically propagated to the created metric. Available in dbt version 1.7 or higher. | Optional | Boolean | diff --git a/website/snippets/_sso-docs-mt-available.md b/website/snippets/_sso-docs-mt-available.md index e56403988a4..fdcdc8249ba 100644 --- a/website/snippets/_sso-docs-mt-available.md +++ b/website/snippets/_sso-docs-mt-available.md @@ -2,6 +2,6 @@ This guide describes a feature of the dbt Cloud Enterprise plan. If you’re interested in learning more about an Enterprise plan, contact us at [sales@getdbt.com](mailto:sales@getdbt.com). -These SSO configuration documents apply to multi-tenant Enterprise deployments only. [Single-tenant](/docs/cloud/about-cloud/tenancy#single-tenant) Virtual Private users can [email dbt Cloud Support](mailto:support@getdbt.com) to set up or update their SSO configuration. +These SSO configuration documents apply to multi-tenant Enterprise deployments only. ::: diff --git a/website/snippets/core-versions-table.md b/website/snippets/core-versions-table.md index 743b59c6bb7..c1fa718e83e 100644 --- a/website/snippets/core-versions-table.md +++ b/website/snippets/core-versions-table.md @@ -2,7 +2,8 @@ | dbt Core | Initial release | Support level and end date | |:-------------------------------------------------------------:|:---------------:|:-------------------------------------:| -| [**v1.8**](/docs/dbt-versions/core-upgrade/upgrading-to-v1.8) | May 9 2024 | Active Support — May 8, 2025 | +| [**v1.9**](/docs/dbt-versions/core-upgrade/upgrading-to-v1.9) | Release candidate | TBA | +| [**v1.8**](/docs/dbt-versions/core-upgrade/upgrading-to-v1.8) | May 9 2024 | Active Support — May 8, 2025| | [**v1.7**](/docs/dbt-versions/core-upgrade/upgrading-to-v1.7) | Nov 2, 2023 |
**dbt Core and dbt Cloud Developer & Team customers:** End of Life
**dbt Cloud Enterprise customers:** Critical Support until further notice 1
| | [**v1.6**](/docs/dbt-versions/core-upgrade/upgrading-to-v1.6) | Jul 31, 2023 | End of Life ⚠️ | | [**v1.5**](/docs/dbt-versions/core-upgrade/upgrading-to-v1.5) | Apr 27, 2023 | End of Life ⚠️ | diff --git a/website/static/img/docs/dbt-cloud/cloud-ide/dbt-assist-toggle.jpg b/website/static/img/docs/dbt-cloud/cloud-ide/dbt-assist-toggle.jpg deleted file mode 100644 index 50dfbe7f51a..00000000000 Binary files a/website/static/img/docs/dbt-cloud/cloud-ide/dbt-assist-toggle.jpg and /dev/null differ diff --git a/website/static/img/docs/dbt-cloud/cloud-ide/dbt-assist.gif b/website/static/img/docs/dbt-cloud/cloud-ide/dbt-assist.gif deleted file mode 100644 index be3236a5123..00000000000 Binary files a/website/static/img/docs/dbt-cloud/cloud-ide/dbt-assist.gif and /dev/null differ diff --git a/website/static/img/docs/dbt-cloud/cloud-ide/dbt-copilot-doc.gif b/website/static/img/docs/dbt-cloud/cloud-ide/dbt-copilot-doc.gif index cca8db37a0a..2e4d42e2efe 100644 Binary files a/website/static/img/docs/dbt-cloud/cloud-ide/dbt-copilot-doc.gif and b/website/static/img/docs/dbt-cloud/cloud-ide/dbt-copilot-doc.gif differ