Skip to content

Commit 56d3c93

Browse files
authored
config.meta and config.tags propagation to top-level on sources and tables (#11839)
1 parent 1fcce44 commit 56d3c93

File tree

7 files changed

+76
-19
lines changed

7 files changed

+76
-19
lines changed
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
kind: Fixes
2+
body: Propagate config.meta and config.tags to top-level on source nodes
3+
time: 2025-07-16T16:45:35.683199-04:00
4+
custom:
5+
Author: michelleark
6+
Issue: "11839"

core/dbt/artifacts/resources/v1/source_definition.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
HasRelationMetadata,
1111
Quoting,
1212
)
13-
from dbt.artifacts.resources.v1.config import BaseConfig
13+
from dbt.artifacts.resources.v1.config import BaseConfig, MergeBehavior
1414
from dbt_common.contracts.config.properties import AdditionalPropertiesAllowed
1515
from dbt_common.contracts.util import Mergeable
1616
from dbt_common.exceptions import CompilationError
@@ -23,6 +23,8 @@ class SourceConfig(BaseConfig):
2323
freshness: Optional[FreshnessThreshold] = field(default_factory=FreshnessThreshold)
2424
loaded_at_field: Optional[str] = None
2525
loaded_at_query: Optional[str] = None
26+
meta: Dict[str, Any] = field(default_factory=dict, metadata=MergeBehavior.Update.meta())
27+
tags: List[str] = field(default_factory=list)
2628

2729

2830
@dataclass

core/dbt/parser/sources.py

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -132,13 +132,8 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition:
132132
source_description = source.description or ""
133133

134134
quoting = source.quoting.merged(table.quoting)
135-
# path = block.path.original_file_path
136-
table_meta = table.meta or {}
137-
source_meta = source.meta or {}
138-
meta = {**source_meta, **table_meta}
139-
140-
# make sure we don't do duplicate tags from source + table
141-
tags = sorted(set(itertools.chain(source.tags, table.tags)))
135+
# Retain original source meta prior to merge with table meta
136+
source_meta = {**source.meta, **source.config.get("meta", {})}
142137

143138
config = self._generate_source_config(
144139
target=target,
@@ -176,15 +171,15 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition:
176171
source_name=source.name,
177172
source_description=source_description,
178173
source_meta=source_meta,
179-
meta=meta,
174+
meta=config.meta,
180175
loader=source.loader,
181176
loaded_at_field=config.loaded_at_field,
182177
loaded_at_query=config.loaded_at_query,
183178
freshness=config.freshness,
184179
quoting=quoting,
185180
resource_type=NodeType.Source,
186181
fqn=target.fqn,
187-
tags=tags,
182+
tags=config.tags,
188183
config=config,
189184
unrendered_config=unrendered_config,
190185
)
@@ -313,6 +308,13 @@ def _generate_source_config(self, target: UnpatchedSourceDefinition, rendered: b
313308
precedence_configs["loaded_at_field"] = precedence_loaded_at_field
314309
precedence_configs["loaded_at_query"] = precedence_loaded_at_query
315310

311+
# Handle merges across source, table, and config for meta and tags
312+
precedence_meta = self.calculate_meta_from_raw_target(target)
313+
precedence_configs["meta"] = precedence_meta
314+
315+
precedence_tags = self.calculate_tags_from_raw_target(target)
316+
precedence_configs["tags"] = precedence_tags
317+
316318
# Because freshness is a "object" config, the freshness from the dbt_project.yml and the freshness
317319
# from the schema file _won't_ get merged by this process. The result will be that the freshness will
318320
# come from the schema file if provided, and if not, it'll fall back to the dbt_project.yml freshness.
@@ -472,6 +474,34 @@ def calculate_loaded_at_field_query_from_raw_target(
472474

473475
return loaded_at_field, loaded_at_query
474476

477+
def calculate_meta_from_raw_target(self, target: UnpatchedSourceDefinition) -> Dict[str, Any]:
478+
source_meta = target.source.meta or {}
479+
source_config_meta = target.source.config.get("meta", {})
480+
source_config_meta = source_config_meta if isinstance(source_config_meta, dict) else {}
481+
482+
table_meta = target.table.meta or {}
483+
table_config_meta = target.table.config.get("meta", {})
484+
table_config_meta = table_config_meta if isinstance(table_config_meta, dict) else {}
485+
486+
return {**source_meta, **source_config_meta, **table_meta, **table_config_meta}
487+
488+
def calculate_tags_from_raw_target(self, target: UnpatchedSourceDefinition) -> List[str]:
489+
source_tags = target.source.tags or []
490+
source_config_tags = target.source.config.get("tags", [])
491+
source_config_tags = (
492+
source_config_tags if isinstance(source_config_tags, list) else [source_config_tags]
493+
)
494+
495+
table_tags = target.table.tags or []
496+
table_config_tags = target.table.config.get("tags", [])
497+
table_config_tags = (
498+
table_config_tags if isinstance(table_config_tags, list) else [table_config_tags]
499+
)
500+
501+
return sorted(
502+
set(itertools.chain(source_tags, source_config_tags, table_tags, table_config_tags))
503+
)
504+
475505

476506
def merge_freshness_time_thresholds(
477507
base: Optional[Time], update: Optional[Time]

tests/functional/artifacts/expected_manifest.py

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -823,6 +823,8 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
823823
},
824824
"loaded_at_query": None,
825825
"loaded_at_field": None,
826+
"meta": {},
827+
"tags": [],
826828
},
827829
"quoting": {
828830
"database": None,
@@ -859,7 +861,12 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
859861
"tags": [],
860862
"unique_id": "source.test.my_source.my_table",
861863
"fqn": ["test", "my_source", "my_table"],
862-
"unrendered_config": {"loaded_at_field": None, "loaded_at_query": None},
864+
"unrendered_config": {
865+
"loaded_at_field": None,
866+
"loaded_at_query": None,
867+
"meta": {},
868+
"tags": [],
869+
},
863870
"unrendered_database": None,
864871
"unrendered_schema": "{{ var('test_schema') }}",
865872
"doc_blocks": [],
@@ -1366,6 +1373,8 @@ def expected_references_manifest(project):
13661373
},
13671374
"loaded_at_field": None,
13681375
"loaded_at_query": None,
1376+
"meta": {},
1377+
"tags": [],
13691378
},
13701379
"quoting": {
13711380
"database": False,
@@ -1401,7 +1410,12 @@ def expected_references_manifest(project):
14011410
"tags": [],
14021411
"unique_id": "source.test.my_source.my_table",
14031412
"fqn": ["test", "my_source", "my_table"],
1404-
"unrendered_config": {"loaded_at_field": None, "loaded_at_query": None},
1413+
"unrendered_config": {
1414+
"loaded_at_field": None,
1415+
"loaded_at_query": None,
1416+
"meta": {},
1417+
"tags": [],
1418+
},
14051419
"unrendered_database": None,
14061420
"unrendered_schema": "{{ var('test_schema') }}",
14071421
"doc_blocks": ["doc.test.table_info"],

tests/functional/fixtures/happy_path_project/models/schema.yml

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,8 @@ sources:
141141
event_time: column_name
142142
loaded_at_field: column_name
143143
meta:
144-
test: 1
144+
source_meta: 1
145+
tags: ["source_tag"]
145146
freshness:
146147
warn_after:
147148
count: 1
@@ -187,14 +188,14 @@ sources:
187188
config:
188189
meta:
189190
test: 1
190-
tags: ["tags"]
191+
tags: ["column_tag"]
191192
config:
192193
enabled: true
193194
event_time: column_name
194195
loaded_at_field: column_name
195196
meta:
196-
test: 1
197-
tags: ["tag"]
197+
table_meta: 1
198+
tags: ["table_tag"]
198199
freshness:
199200
warn_after:
200201
count: 1

tests/functional/list/test_list.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -642,8 +642,8 @@ def expect_source_output(self):
642642
},
643643
"filter": "column_name = 1",
644644
},
645-
"meta": {"test": 1},
646-
"tags": ["tag"],
645+
"meta": {"source_meta": 1, "table_meta": 1},
646+
"tags": ["source_tag", "table_tag"],
647647
"loaded_at_query": None,
648648
"loaded_at_field": "column_name",
649649
},
@@ -653,7 +653,7 @@ def expect_source_output(self):
653653
"name": "my_table",
654654
"source_name": "my_source",
655655
"resource_type": "source",
656-
"tags": [],
656+
"tags": ["source_tag", "table_tag"],
657657
},
658658
"path": self.dir("models/schema.yml"),
659659
}

tests/unit/contracts/graph/test_nodes_parsed.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1904,6 +1904,8 @@ def basic_parsed_source_definition_dict():
19041904
"warn_after": {},
19051905
"error_after": {},
19061906
},
1907+
"tags": [],
1908+
"meta": {},
19071909
},
19081910
"unrendered_config": {},
19091911
"doc_blocks": [],
@@ -1939,6 +1941,8 @@ def complex_parsed_source_definition_dict():
19391941
"warn_after": {},
19401942
"error_after": {},
19411943
},
1944+
"tags": [],
1945+
"meta": {},
19421946
},
19431947
"freshness": {"warn_after": {"period": "hour", "count": 1}, "error_after": {}},
19441948
"loaded_at_field": "loaded_at",

0 commit comments

Comments
 (0)