Skip to content

Commit 791bfaa

Browse files
psousa50psousa50
committed
Add some tests for final model query
1 parent 2f99d0f commit 791bfaa

27 files changed

+151
-128
lines changed

‎integration-tests/macros/test_helpers.sql

+36-8
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,47 @@
11
{% macro test_should_fail (model_name, test_description) %}
2-
{% set mocks_and_expectations_json_str = caller() %}
32
{{ dbt_unit_testing.ref_tested_model(model_name) }}
43
{% if execute %}
5-
{% set test_configuration = {
6-
"model_name": model_name,
7-
"description": test_description,
8-
"options": {"hide_errors": true}}
9-
%}
4+
{% set mocks_and_expectations_json_str = caller() %}
5+
{% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_name, test_description, {}, mocks_and_expectations_json_str) %}
6+
{% set test_report = dbt_unit_testing.build_test_report(test_configuration, test_queries) %}
107

11-
{% do test_configuration.update (dbt_unit_testing.build_mocks_and_expectations(test_configuration, mocks_and_expectations_json_str)) %}
12-
{% set test_report = dbt_unit_testing.build_test_report(test_configuration) %}
8+
{{ dbt_unit_testing.verbose("-------------------- " ~ test_configuration.model_name ~ " --------------------" ) }}
9+
{{ dbt_unit_testing.verbose(test_queries.test_query) }}
10+
{{ dbt_unit_testing.verbose("----------------------------------------" ) }}
1311

1412
{% if test_report.succeeded %}
1513
{{ dbt_unit_testing.println("{RED}Test: " ~ "{YELLOW}" ~ test_description ~ " {RED}should have FAILED")}}
1614
{% endif %}
1715
select 1 from (select 1) as t where {{ test_report.succeeded }}
1816
{% endif %}
1917
{% endmacro %}
18+
19+
{% macro test_condition_on_model_query (model_name, test_description, options, condition) %}
20+
{{ dbt_unit_testing.ref_tested_model(model_name) }}
21+
{% if execute %}
22+
{% set mocks_and_expectations_json_str = caller() %}
23+
{% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_name, test_description, options, mocks_and_expectations_json_str) %}
24+
25+
{% set model_query = test_queries["model_query"] %}
26+
27+
{{ dbt_unit_testing.verbose("-------------------- " ~ test_configuration.model_name ~ " --------------------" ) }}
28+
{{ dbt_unit_testing.verbose(test_queries.test_query) }}
29+
{{ dbt_unit_testing.verbose("----------------------------------------" ) }}
30+
31+
{% set succeeded = condition(model_query, *varargs) %}
32+
{% if not succeeded %}
33+
{{ dbt_unit_testing.println("{RED}Test: " ~ "{YELLOW}" ~ test_description ~ " {RED}FAILED")}}
34+
{% endif %}
35+
36+
select 1 from (select 1) as t where {{ not succeeded }}
37+
{% endif %}
38+
{% endmacro %}
39+
40+
{% macro assert_should_contain(s, should_be_there) %}
41+
{{ return (should_be_there in s) }}
42+
{% endmacro %}
43+
44+
{% macro assert_should_not_contain(s, should_not_be_there) %}
45+
{{ return (not should_not_be_there in s) }}
46+
{% endmacro %}
47+

‎integration-tests/models/complex_hierarchy/m11.sql

-3
This file was deleted.

‎integration-tests/models/complex_hierarchy/m12.sql

-3
This file was deleted.

‎integration-tests/models/complex_hierarchy/m21.sql

-3
This file was deleted.

‎integration-tests/models/complex_hierarchy/m22.sql

-3
This file was deleted.

‎integration-tests/models/complex_hierarchy/m31.sql

-5
This file was deleted.

‎integration-tests/models/complex_hierarchy/sources.yml

-8
This file was deleted.

‎integration-tests/models/model_11.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
select * from {{ dbt_unit_testing.source('dbt_unit_testing','source_1') }}

‎integration-tests/models/model_12.sql

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
select * from {{ dbt_unit_testing.source('dbt_unit_testing','source_2') }}

‎integration-tests/models/model_21.sql

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
select *
2+
from {{ dbt_unit_testing.ref('model_11') }}
3+
left join {{ dbt_unit_testing.ref('model_12') }} using(id)

‎integration-tests/models/model_22.sql

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
select *
2+
from {{ dbt_unit_testing.ref('model_12') }}

‎integration-tests/models/model_31.sql

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
select *
2+
from {{ dbt_unit_testing.ref('model_21') }}
3+
left join {{ dbt_unit_testing.ref('model_22') }} using(id)

‎integration-tests/models/model_in_database.sql

+2
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1+
{{ config( tags = ['add-to-database'] )}}
2+
13
select 1 as a, 2 as b

‎integration-tests/models/sources/sources.yml

+2
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ sources:
1717
identifier: true
1818
- name: sample_source_name
1919
identifier: sample_source_identifier
20+
- name: source_1
21+
- name: source_2
2022

2123
- name: source_from_schema_1
2224
schema: dbt_unit_testing_schema_1

‎integration-tests/seeds/existing_sources/s1.csv

-4
This file was deleted.

‎integration-tests/seeds/existing_sources/s2.csv

-4
This file was deleted.

‎integration-tests/seeds/existing_sources/s3.csv

-4
This file was deleted.

‎integration-tests/seeds/existing_sources/source_1.csv

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
id,a
2+
1,'a'
3+
2,'b'
4+
3,'c'

‎integration-tests/seeds/existing_sources/source_2.csv

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
id
2+
1
3+
2
4+
3

‎integration-tests/tests/unit/complex_hierarchy/m31_no_database.sql

-38
This file was deleted.

‎integration-tests/tests/unit/complex_hierarchy/m31_with_database.sql

-26
This file was deleted.

‎integration-tests/tests/unit/model_31.sql

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{{
2+
config(
3+
tags=['unit-test', 'bigquery', 'snowflake', 'postgres', 'db-dependency']
4+
)
5+
}}
6+
7+
{% call test_condition_on_model_query('model_31', "should not include model_11 if model_21 is mocked", {},
8+
assert_should_not_contain, "model_11") %}
9+
{% call dbt_unit_testing.mock_ref ('model_21') %}
10+
select 1 as id
11+
{% endcall %}
12+
{% call dbt_unit_testing.expect() %}
13+
select 1 as id
14+
{% endcall %}
15+
{% endcall %}
16+
17+
UNION ALL
18+
19+
{% call dbt_unit_testing.test('model_31', "unless we need to fetch missiog columns") %}
20+
{% call dbt_unit_testing.mock_ref ('model_21', {"include_missing_columns": true}) %}
21+
select 1 as id
22+
{% endcall %}
23+
{% call dbt_unit_testing.expect() %}
24+
select 1 as id
25+
{% endcall %}
26+
{% endcall %}
27+
28+
UNION ALL
29+
30+
{% call test_condition_on_model_query('model_31', "should not include source_1 if model_11 is mocked", {},
31+
assert_should_not_contain, "source_1") %}
32+
{% call dbt_unit_testing.mock_ref ('model_11') %}
33+
select 1 as id
34+
{% endcall %}
35+
{% call dbt_unit_testing.expect() %}
36+
select 1 as id
37+
{% endcall %}
38+
{% endcall %}
39+
40+
UNION ALL
41+
42+
{% call test_condition_on_model_query('model_31', "should not include model_11 if using database models",
43+
{"use_database_models": true},
44+
assert_should_not_contain, "model_11") %}
45+
{% call dbt_unit_testing.mock_ref ('model_22') %}
46+
select 1 as id
47+
{% endcall %}
48+
{% call dbt_unit_testing.expect() %}
49+
select 1 as id
50+
{% endcall %}
51+
{% endcall %}
52+
53+
UNION ALL
54+
55+
{% call test_condition_on_model_query('model_31', "should not include model_12 if using database models",
56+
{"use_database_models": true},
57+
assert_should_not_contain, "model_12") %}
58+
{% call dbt_unit_testing.mock_ref ('model_22') %}
59+
select 1 as id
60+
{% endcall %}
61+
{% call dbt_unit_testing.expect() %}
62+
select 1 as id
63+
{% endcall %}
64+
{% endcall %}
65+

‎jaffle-shop/tests/unit/tests.sql

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{{
22
config(
3-
tags=['unit-test', 'no-db-dependency']
3+
tags=['unit-test']
44
)
55
}}
66

‎macros/tests.sql

+19-12
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,9 @@
22
{{ dbt_unit_testing.ref_tested_model(model_name) }}
33

44
{% if execute %}
5-
{% set test_configuration = {
6-
"model_name": model_name,
7-
"description": test_description,
8-
"options": dbt_unit_testing.merge_configs([options])}
9-
%}
105
{% set mocks_and_expectations_json_str = caller() %}
11-
12-
{{ dbt_unit_testing.verbose("CONFIG: " ~ test_configuration) }}
13-
14-
{% do test_configuration.update (dbt_unit_testing.build_mocks_and_expectations(test_configuration, mocks_and_expectations_json_str)) %}
15-
{% set test_report = dbt_unit_testing.build_test_report(test_configuration) %}
6+
{% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_name, test_description, options, mocks_and_expectations_json_str) %}
7+
{% set test_report = dbt_unit_testing.build_test_report(test_configuration, test_queries) %}
168

179
{% if not test_report.succeeded %}
1810
{{ dbt_unit_testing.show_test_report(test_configuration, test_report) }}
@@ -22,6 +14,21 @@
2214
{% endif %}
2315
{% endmacro %}
2416

17+
{% macro build_configuration_and_test_queries(model_name, test_description, options, mocks_and_expectations_json_str) %}
18+
{% set test_configuration = {
19+
"model_name": model_name,
20+
"description": test_description,
21+
"options": dbt_unit_testing.merge_configs([options])}
22+
%}
23+
24+
{{ dbt_unit_testing.verbose("CONFIG: " ~ test_configuration) }}
25+
26+
{% do test_configuration.update (dbt_unit_testing.build_mocks_and_expectations(test_configuration, mocks_and_expectations_json_str)) %}
27+
{% set test_queries = dbt_unit_testing.build_test_queries(test_configuration) %}
28+
29+
{{ return ((test_configuration, test_queries)) }}
30+
{% endmacro %}
31+
2532
{% macro build_mocks_and_expectations(test_configuration, mocks_and_expectations_json_str) %}
2633
{% set mocks_and_expectations = dbt_unit_testing.split_json_str(mocks_and_expectations_json_str) %}
2734

@@ -50,9 +57,8 @@
5057
{{ return (mocks_and_expectations_json) }}
5158
{% endmacro %}
5259

53-
{% macro build_test_report(test_configuration) %}
60+
{% macro build_test_report(test_configuration, test_queries) %}
5461

55-
{% set test_queries = dbt_unit_testing.build_test_queries(test_configuration) %}
5662
{% set test_report = dbt_unit_testing.run_test_query(test_configuration, test_queries) %}
5763

5864
{{ dbt_unit_testing.verbose("-------------------- " ~ test_configuration.model_name ~ " --------------------" ) }}
@@ -105,6 +111,7 @@
105111
{%- endset -%}
106112

107113
{% set test_queries = {
114+
"model_query": model_complete_sql,
108115
"actual_query": actual_query,
109116
"expectations_query": expectations_query,
110117
"test_query": test_query

‎macros/utils.sql

+5-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{% macro run_query(query) %}
22
{% set start_time = modules.datetime.datetime.now() %}
33
{{ dbt_unit_testing.verbose('Running query => ' ~ dbt_unit_testing.sanitize(query)) }}
4-
{% set results = run_query(query) %}
4+
{% set results = dbt.run_query(query) %}
55
{% set end_time = modules.datetime.datetime.now() - start_time %}
66
{{ dbt_unit_testing.verbose('Execution time => ' ~ end_time) }}
77
{{ dbt_unit_testing.verbose('==============================================================') }}
@@ -48,7 +48,9 @@
4848
{% endmacro %}
4949

5050
{% macro debug(s) %}
51-
{{ dbt_unit_testing.log_info (s, only_on_execute=true) }}
51+
{% if var('debug', dbt_unit_testing.config_is_true('debug')) %}
52+
{{ dbt_unit_testing.log_info (s, only_on_execute=true) }}
53+
{% endif %}
5254
{% endmacro %}
5355

5456
{% macro verbose(s) %}
@@ -169,4 +171,4 @@
169171

170172
{% macro raise_error(error_message) %}
171173
{{ exceptions.raise_compiler_error('\x1b[31m' ~ error_message ~ '\x1b[0m') }}
172-
{% endmacro %}
174+
{% endmacro %}

‎run-integration-tests.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE" --exclude tag
2626
# create sources in the database
2727
dbt seed --target "$PROFILE" --select seeds/existing_sources
2828
# create models in the database for tests that depends on database models
29-
dbt run --target "$PROFILE" --select models/complex_hierarchy
30-
dbt run --target "$PROFILE" --select models/model_in_database.sql
29+
dbt run --target "$PROFILE" --select tag:add-to-database
30+
3131
# run tests with database dependency
3232
dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE",tag:db-dependency

‎run-jaffle-shop-tests.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ if [[ ! -e ~/.dbt/profiles.yml ]]; then
1515
fi
1616

1717
dbt deps --target "$PROFILE"
18-
dbt test --target "$PROFILE" --models tag:unit-test,tag:no-db-dependency
18+
dbt test --target "$PROFILE" --models tag:unit-test, --exclude tag:db-dependency
1919
# create seeds in the database
2020
dbt seed --target "$PROFILE"
2121
# run tests that leverages from the created sources

0 commit comments

Comments
 (0)