From e91a1290843b57f99c0ae7d80e716ce0afe77a24 Mon Sep 17 00:00:00 2001 From: Devon Fulcher <24593113+DevonFulcher@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:35:03 -0600 Subject: [PATCH] Merge semantic model meta config (#367) ### Description This PR implements the meta config merge behavior described [here](https://docs.getdbt.com/reference/configs-and-properties#combining-configs) and referenced [here](https://github.com/dbt-labs/dbt-semantic-interfaces/issues/362). I decided to stack this PR on top to keep it distinct from the other PRs, but I don't intend to merge this one into main directly. I will merge it into its parent branch first. ### Checklist - [x] I have read [the contributing guide](https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/CONTRIBUTING.md) and understand what's expected of me - [x] I have signed the [CLA](https://docs.getdbt.com/docs/contributor-license-agreements) - [x] This PR includes tests, or tests are not required/relevant for this PR - [ ] ~~I have run `changie new` to [create a changelog entry](https://github.com/dbt-labs/dbt-semantic-interfaces/blob/main/CONTRIBUTING.md#adding-a-changelog-entry)~~ I'm not going to merge this directly into main, so we don't need a separate changelog entry. --- .../parsing/dir_to_model.py | 23 +++++- tests/parsing/test_semantic_model_parsing.py | 82 +++++++++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/dbt_semantic_interfaces/parsing/dir_to_model.py b/dbt_semantic_interfaces/parsing/dir_to_model.py index 418ca254..c347b8a4 100644 --- a/dbt_semantic_interfaces/parsing/dir_to_model.py +++ b/dbt_semantic_interfaces/parsing/dir_to_model.py @@ -3,11 +3,17 @@ import traceback from dataclasses import dataclass from string import Template -from typing import Dict, List, Optional, Type, Union +from typing import Dict, List, Optional, Sequence, Type, Union from jsonschema import exceptions from dbt_semantic_interfaces.errors import ParsingException +from dbt_semantic_interfaces.implementations.element_config import ( + PydanticSemanticLayerElementConfig, +) +from dbt_semantic_interfaces.implementations.elements.dimension import PydanticDimension +from dbt_semantic_interfaces.implementations.elements.entity import PydanticEntity +from dbt_semantic_interfaces.implementations.elements.measure import PydanticMeasure from dbt_semantic_interfaces.implementations.metric import PydanticMetric from dbt_semantic_interfaces.implementations.project_configuration import ( PydanticProjectConfiguration, @@ -334,7 +340,20 @@ def parse_config_yaml( results.append(metric_class.parse_obj(object_cfg)) elif document_type == SEMANTIC_MODEL_TYPE: semantic_model_validator.validate(config_document[document_type]) - results.append(semantic_model_class.parse_obj(object_cfg)) + sm = semantic_model_class.parse_obj(object_cfg) + # Combine configs according to the behavior documented here https://docs.getdbt.com/reference/configs-and-properties#combining-configs + elements: Sequence[Union[PydanticDimension, PydanticEntity, PydanticMeasure]] = [ + *sm.dimensions, + *sm.entities, + *sm.measures, + ] + for element in elements: + if sm.config is not None: + if element.config is None: + element.config = PydanticSemanticLayerElementConfig(meta=sm.config.meta) + else: + element.config.meta = {**sm.config.meta, **element.config.meta} + results.append(sm) elif document_type == PROJECT_CONFIGURATION_TYPE: project_configuration_validator.validate(config_document[document_type]) results.append(project_configuration_class.parse_obj(object_cfg)) diff --git a/tests/parsing/test_semantic_model_parsing.py b/tests/parsing/test_semantic_model_parsing.py index 34ad851f..61839360 100644 --- a/tests/parsing/test_semantic_model_parsing.py +++ b/tests/parsing/test_semantic_model_parsing.py @@ -538,3 +538,85 @@ def test_semantic_model_dimension_validity_params_parsing() -> None: assert end_dimension.type_params.validity_params is not None assert end_dimension.type_params.validity_params.is_start is False assert end_dimension.type_params.validity_params.is_end is True + + +def test_semantic_model_element_config_merging() -> None: + """Test for merging element config metadata from semantic model into dimension, entity, and measure objects.""" + yaml_contents = textwrap.dedent( + """\ + semantic_model: + name: sm + config: + meta: + sm_metadata: asdf + node_relation: + alias: source_table + schema_name: some_schema + dimensions: + - name: dim_0 + type: time + type_params: + time_granularity: day + config: + meta: + sm_metadata: qwer + dim_metadata: fdsa + - name: dim_1 + type: time + type_params: + time_granularity: day + config: + meta: + dim_metadata: mlkj + sm_metadata: zxcv + - name: dim_2 + type: time + type_params: + time_granularity: day + - name: dim_3 + type: time + type_params: + time_granularity: day + config: + meta: + dim_metadata: gfds + entities: + - name: entity_0 + type: primary + config: + meta: + sm_metadata: hjkl + measures: + - name: measure_0 + agg: count_distinct + config: + meta: + sm_metadata: ijkl + """ + ) + file = YamlConfigFile(filepath="test_dir/inline_for_test", contents=yaml_contents) + + build_result = parse_yaml_files_to_semantic_manifest(files=[file, EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE]) + + assert len(build_result.semantic_manifest.semantic_models) == 1 + semantic_model = build_result.semantic_manifest.semantic_models[0] + assert semantic_model.config is not None + assert semantic_model.config.meta["sm_metadata"] == "asdf" + assert len(semantic_model.dimensions) == 4 + assert semantic_model.dimensions[0].config is not None + assert semantic_model.dimensions[0].config.meta["sm_metadata"] == "qwer" + assert semantic_model.dimensions[0].config.meta["dim_metadata"] == "fdsa" + assert semantic_model.dimensions[1].config is not None + assert semantic_model.dimensions[1].config.meta["sm_metadata"] == "zxcv" + assert semantic_model.dimensions[1].config.meta["dim_metadata"] == "mlkj" + assert semantic_model.dimensions[2].config is not None + assert semantic_model.dimensions[2].config.meta["sm_metadata"] == "asdf" + assert semantic_model.dimensions[3].config is not None + assert semantic_model.dimensions[3].config.meta["dim_metadata"] == "gfds" + assert semantic_model.dimensions[3].config.meta["sm_metadata"] == "asdf" + assert len(semantic_model.entities) == 1 + assert semantic_model.entities[0].config is not None + assert semantic_model.entities[0].config.meta["sm_metadata"] == "hjkl" + assert len(semantic_model.measures) == 1 + assert semantic_model.measures[0].config is not None + assert semantic_model.measures[0].config.meta["sm_metadata"] == "ijkl"