From 56d389991f9144f8d750afc98ef4cc42dd2395b5 Mon Sep 17 00:00:00 2001 From: Patrick Yost Date: Fri, 8 Nov 2024 13:15:36 -0800 Subject: [PATCH 1/2] InProgress: Add tags to SavedQueries --- .../dbt/artifacts/resources/v1/saved_query.py | 8 ++++- core/dbt/contracts/graph/nodes.py | 4 +++ core/dbt/contracts/graph/unparsed.py | 9 ++++++ core/dbt/parser/schema_yaml_readers.py | 13 ++++++++ core/setup.py | 2 +- editable-requirements.txt | 1 + schemas/dbt/catalog/v1.json | 2 +- schemas/dbt/manifest/v12.json | 30 +++++++++++++++++-- schemas/dbt/run-results/v6.json | 2 +- schemas/dbt/sources/v3.json | 2 +- 10 files changed, 66 insertions(+), 7 deletions(-) diff --git a/core/dbt/artifacts/resources/v1/saved_query.py b/core/dbt/artifacts/resources/v1/saved_query.py index e1d056d0422..8d51845755b 100644 --- a/core/dbt/artifacts/resources/v1/saved_query.py +++ b/core/dbt/artifacts/resources/v1/saved_query.py @@ -2,16 +2,18 @@ import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional, Union from dbt.artifacts.resources.base import GraphResource from dbt.artifacts.resources.types import NodeType from dbt.artifacts.resources.v1.components import DependsOn, RefArgs +from dbt.artifacts.resources.v1.config import list_str, metas from dbt.artifacts.resources.v1.semantic_layer_components import ( SourceFileMetadata, WhereFilterIntersection, ) from dbt_common.contracts.config.base import BaseConfig, CompareBehavior, MergeBehavior +from dbt_common.contracts.config.metadata import ShowBehavior from dbt_common.dataclass_schema import dbtClassMixin from dbt_semantic_interfaces.type_enums.export_destination_type import ( ExportDestinationType, @@ -95,6 +97,10 @@ class SavedQuery(SavedQueryMandatory): depends_on: DependsOn = field(default_factory=DependsOn) created_at: float = field(default_factory=lambda: time.time()) refs: List[RefArgs] = field(default_factory=list) + tags: Union[List[str], str] = field( + default_factory=list_str, + metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), + ) @property def metrics(self) -> List[str]: diff --git a/core/dbt/contracts/graph/nodes.py b/core/dbt/contracts/graph/nodes.py index 262ec83735e..90d97c26e74 100644 --- a/core/dbt/contracts/graph/nodes.py +++ b/core/dbt/contracts/graph/nodes.py @@ -1603,6 +1603,9 @@ def same_exports(self, old: "SavedQuery") -> bool: return True + def same_tags(self, old: "SavedQuery") -> bool: + return self.tags == old.tags + def same_contents(self, old: Optional["SavedQuery"]) -> bool: # existing when it didn't before is a change! # metadata/tags changes are not "changes" @@ -1618,6 +1621,7 @@ def same_contents(self, old: Optional["SavedQuery"]) -> bool: and self.same_config(old) and self.same_group(old) and self.same_exports(old) + and self.same_tags(old) and True ) diff --git a/core/dbt/contracts/graph/unparsed.py b/core/dbt/contracts/graph/unparsed.py index 5e63c487e89..373558adcb9 100644 --- a/core/dbt/contracts/graph/unparsed.py +++ b/core/dbt/contracts/graph/unparsed.py @@ -27,8 +27,11 @@ UnitTestOutputFixture, UnitTestOverrides, ) +from dbt.artifacts.resources.v1.config import list_str, metas from dbt.exceptions import ParsingError from dbt.node_types import NodeType +from dbt_common.contracts.config.base import CompareBehavior, MergeBehavior +from dbt_common.contracts.config.metadata import ShowBehavior from dbt_common.contracts.config.properties import AdditionalPropertiesMixin from dbt_common.contracts.util import Mergeable from dbt_common.dataclass_schema import ( @@ -740,6 +743,12 @@ class UnparsedSavedQuery(dbtClassMixin): label: Optional[str] = None exports: List[UnparsedExport] = field(default_factory=list) config: Dict[str, Any] = field(default_factory=dict) + # Note: the order of the types is critical; it's the order that they will be checked against inputs. + # if reversed, a single-string tag like `tag: "good"` becomes ['g','o','o','d'] + tags: Union[str, List[str]] = field( + default_factory=list_str, + metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude), + ) def normalize_date(d: Optional[datetime.date]) -> Optional[datetime.datetime]: diff --git a/core/dbt/parser/schema_yaml_readers.py b/core/dbt/parser/schema_yaml_readers.py index 9b4a550b5d3..5ce62fe7f9a 100644 --- a/core/dbt/parser/schema_yaml_readers.py +++ b/core/dbt/parser/schema_yaml_readers.py @@ -816,6 +816,18 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: rendered=False, ) + # The parser handles plain strings just fine, but we need to be able + # to join two lists, remove duplicates, and sort, so we have to wrap things here. + def wrap_tags(s: Union[List[str], str]) -> List[str]: + if s is None: + return [] + return [s] if isinstance(s, str) else s + + config_tags = wrap_tags(config.get("tags")) + unparsed_tags = wrap_tags(unparsed.tags) + tags = list(set([*unparsed_tags, *config_tags])) + tags.sort() + parsed = SavedQuery( description=unparsed.description, label=unparsed.label, @@ -831,6 +843,7 @@ def parse_saved_query(self, unparsed: UnparsedSavedQuery) -> None: config=config, unrendered_config=unrendered_config, group=config.group, + tags=tags, ) for export in parsed.exports: diff --git a/core/setup.py b/core/setup.py index 456e5fedc15..e631968aa65 100644 --- a/core/setup.py +++ b/core/setup.py @@ -69,7 +69,7 @@ # These are major-version-0 packages also maintained by dbt-labs. # Accept patches but avoid automatically updating past a set minor version range. "dbt-extractor>=0.5.0,<=0.6", - "dbt-semantic-interfaces>=0.7.4,<0.8", + # "dbt-semantic-interfaces>=0.7.4,<0.8", # Minor versions for these are expected to be backwards-compatible "dbt-common>=1.11.0,<2.0", "dbt-adapters>=1.8.0,<2.0", diff --git a/editable-requirements.txt b/editable-requirements.txt index 6f81820bb96..5cf12a85ea4 100644 --- a/editable-requirements.txt +++ b/editable-requirements.txt @@ -1 +1,2 @@ -e ./core +-e /Users/patricky/git/dbt-semantic-interfaces diff --git a/schemas/dbt/catalog/v1.json b/schemas/dbt/catalog/v1.json index f104c5b977f..e5c3887c057 100644 --- a/schemas/dbt/catalog/v1.json +++ b/schemas/dbt/catalog/v1.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.9.0b2" + "default": "1.9.0b4" }, "generated_at": { "type": "string" diff --git a/schemas/dbt/manifest/v12.json b/schemas/dbt/manifest/v12.json index 4d66f65f234..841fffae32a 100644 --- a/schemas/dbt/manifest/v12.json +++ b/schemas/dbt/manifest/v12.json @@ -13,7 +13,7 @@ }, "dbt_version": { "type": "string", - "default": "1.9.0b2" + "default": "1.9.0b4" }, "generated_at": { "type": "string" @@ -19969,6 +19969,19 @@ "name" ] } + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] } }, "additionalProperties": false, @@ -21524,6 +21537,19 @@ "name" ] } + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "string" + } + ] } }, "additionalProperties": false, @@ -22648,4 +22674,4 @@ "unit_tests" ], "$id": "https://schemas.getdbt.com/dbt/manifest/v12.json" -} +} \ No newline at end of file diff --git a/schemas/dbt/run-results/v6.json b/schemas/dbt/run-results/v6.json index 1bf1cf75e83..28f5ee69058 100644 --- a/schemas/dbt/run-results/v6.json +++ b/schemas/dbt/run-results/v6.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.9.0b2" + "default": "1.9.0b4" }, "generated_at": { "type": "string" diff --git a/schemas/dbt/sources/v3.json b/schemas/dbt/sources/v3.json index df2784f1a81..36f52bbf7dc 100644 --- a/schemas/dbt/sources/v3.json +++ b/schemas/dbt/sources/v3.json @@ -12,7 +12,7 @@ }, "dbt_version": { "type": "string", - "default": "1.9.0b2" + "default": "1.9.0b4" }, "generated_at": { "type": "string" From 752f6392e90b9f0ba244013360687bf16b0be293 Mon Sep 17 00:00:00 2001 From: Patrick Yost Date: Thu, 14 Nov 2024 13:47:58 -0800 Subject: [PATCH 2/2] Add tests for tags in saved queries --- tests/functional/saved_queries/fixtures.py | 24 +++++++++++++ .../functional/saved_queries/test_configs.py | 34 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/tests/functional/saved_queries/fixtures.py b/tests/functional/saved_queries/fixtures.py index 58ed73c81b0..ac45f8aeb76 100644 --- a/tests/functional/saved_queries/fixtures.py +++ b/tests/functional/saved_queries/fixtures.py @@ -164,3 +164,27 @@ export_as: table schema: my_export_schema_name """ + +saved_query_with_tags_defined_yml = """ +saved_queries: + - name: test_saved_query + description: "{{ doc('saved_query_description') }}" + label: Test Saved Query + tags: + - tag_a + - tag_c + query_params: + metrics: + - simple_metric + group_by: + - "Dimension('id__ds')" + where: + - "{{ TimeDimension('id__ds', 'DAY') }} <= now()" + - "{{ TimeDimension('id__ds', 'DAY') }} >= '2023-01-01'" + exports: + - name: my_export + config: + alias: my_export_alias + export_as: table + schema: my_export_schema_name +""" diff --git a/tests/functional/saved_queries/test_configs.py b/tests/functional/saved_queries/test_configs.py index df4be7aa5b6..943f4819fcd 100644 --- a/tests/functional/saved_queries/test_configs.py +++ b/tests/functional/saved_queries/test_configs.py @@ -14,6 +14,7 @@ saved_query_with_cache_configs_defined_yml, saved_query_with_export_configs_defined_at_saved_query_level_yml, saved_query_with_extra_config_attributes_yml, + saved_query_with_tags_defined_yml, saved_query_without_export_configs_defined_yml, ) from tests.functional.semantic_models.fixtures import ( @@ -322,3 +323,36 @@ def test_override_saved_query_config( result = runner.invoke(["parse"]) assert result.success assert saved_query.config.cache.enabled is True + + +# the tags defined in project yaml for the SavedQuery is additive to the query's +class TestSavedQueryTagsAdditiveWithConfig(BaseConfigProject): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "saved-queries": {"+tags": ["tag_b", "tag_c"]}, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "saved_queries.yml": saved_query_with_tags_defined_yml, + "schema.yml": schema_yml, + "fct_revenue.sql": fct_revenue_sql, + "metricflow_time_spine.sql": metricflow_time_spine_sql, + "docs.md": saved_query_description, + } + + def test_saved_query_tags_are_additive_unique_and_sorted( + self, + project, + ): + runner = dbtTestRunner() + + # parse with default fixture project config + result = runner.invoke(["parse"]) + assert result.success + assert isinstance(result.result, Manifest) + assert len(result.result.saved_queries) == 1 + saved_query = result.result.saved_queries["saved_query.test.test_saved_query"] + assert saved_query.tags == ["tag_a", "tag_b", "tag_c"]