Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changes/unreleased/Features-20261011-092619.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
kind: Features
body: Add a data_governance_tags column property that sets one BigQuery governance tag from a key and value
time: 2026-10-11T09:26:19.330713+02:00
custom:
author: AxelThevenot
issue: "16693"
project: dbt-core
2 changes: 2 additions & 0 deletions crates/dbt-adapter/src/metadata/bigquery/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,7 @@ pub fn nest_column_data_types(
meta: column.meta.clone(),
tags: column.tags.clone(),
policy_tags: column.policy_tags.clone(),
data_governance_tags: column.data_governance_tags.clone(),
classifiers: column.classifiers.clone(),
databricks_tags: column.databricks_tags.clone(),
column_mask: column.column_mask.clone(),
Expand All @@ -478,6 +479,7 @@ pub fn nest_column_data_types(
meta: IndexMap::new(),
tags: vec![],
policy_tags: None,
data_governance_tags: None,
classifiers: None,
databricks_tags: None,
column_mask: None,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@
{% endfor %}
{% set filtered_columns = validate_doc_columns(relation, model.columns, valid_names) %}
{% do alter_column_comment(relation, filtered_columns) %}
{% do bigquery__alter_column_data_governance_tags(relation, filtered_columns) %}
{% endif %}
{% endmacro %}

Expand All @@ -177,6 +178,55 @@
{% do adapter.update_columns(relation, column_dict) %}
{% endmacro %}

{% macro bigquery__alter_column_data_governance_tags(relation, column_dict) -%}
{% set sql = bigquery__get_alter_column_data_governance_tags_sql(relation, column_dict) %}
{% if sql %}
{% do run_query(sql) %}
{% endif %}
{% endmacro %}

{% macro bigquery__get_alter_column_data_governance_tags_sql(relation, column_dict) %}
{%- set ns = namespace(actions=[]) -%}
{%- for col_name in column_dict -%}
{%- set col = column_dict[col_name] -%}
{%- set gov = col['data_governance_tags'] -%}
{%- if gov is defined and gov is not none -%}
{%- if relation.type != 'table' -%}
{{ exceptions.raise_compiler_error(
"data_governance_tags are only supported on tables."
) }}
{%- endif -%}
{%- set dtype = col['data_type'] | default('', true) | lower -%}
{%- if dtype in ['record', 'struct'] or 'struct<' in dtype or 'record<' in dtype -%}
{{ exceptions.raise_compiler_error(
"data_governance_tags on column '" ~ col_name ~ "' cannot be set on a RECORD or STRUCT parent. Set the tag on a leaf field."
) }}
{%- endif -%}
{%- if '.' in col_name -%}
{{ exceptions.raise_compiler_error(
"data_governance_tags on column '" ~ col_name ~ "' cannot target a nested field. BigQuery ALTER COLUMN does not accept a dotted path."
) }}
{%- endif -%}
{%- set key_text = gov['key'] | default('', true) | string -%}
{%- set value_text = gov['value'] | default('', true) | string -%}
{%- set parts = key_text.split('/') -%}
{%- if parts | length != 2 or parts[0] | length == 0 or parts[1] | length == 0 or value_text | length == 0 -%}
{{ exceptions.raise_compiler_error(
"Column '" ~ col_name ~ "' data_governance_tags must set key to '{parent_id}/{short_name}' and value to a non-empty short name."
) }}
{%- endif -%}
{%- set quoted_name = adapter.quote(col_name) -%}
{%- set key_literal = '"' ~ (key_text | replace('\\', '\\\\') | replace('"', '\\"')) ~ '"' -%}
{%- set value_literal = '"' ~ (value_text | replace('\\', '\\\\') | replace('"', '\\"')) ~ '"' -%}
{%- set ns.actions = ns.actions + ["alter column " ~ quoted_name ~ " set options (data_governance_tags=[(" ~ key_literal ~ ", " ~ value_literal ~ ")])"] -%}
{%- endif -%}
{%- endfor -%}
{%- if ns.actions | length == 0 -%}
{{ return('') }}
{%- endif -%}
{{ return('alter ' ~ relation.type ~ ' ' ~ relation ~ '\n' ~ (ns.actions | join(',\n'))) }}
{% endmacro %}

{% macro bigquery__alter_relation_add_columns(relation, add_columns) %}

{% set sql -%}
Expand Down
161 changes: 161 additions & 0 deletions crates/dbt-loader/tests/macros/bigquery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -253,3 +253,164 @@ fn generate_schema_name_composes_on_a_projects_own_default() {
"sales_catalog.custom_staging"
);
}

fn render_governance_tags(
relation_type: RelationType,
column_dict: Value,
) -> dbt_common::FsResult<String> {
let harness = build_harness();
harness.mock().on("quote", |args| {
Ok(Value::from(format!(
"`{}`",
args[0].as_str().expect("column name")
)))
});
let relation = harness.relation("test-db", "test_schema", "customers", Some(relation_type));
let ctx = BTreeMap::from([
(
"relation".to_string(),
RelationObject::new(relation).into_value(),
),
("column_dict".to_string(), column_dict),
]);
harness.render(
"{{ bigquery__get_alter_column_data_governance_tags_sql(relation, column_dict) }}",
ctx,
)
}

fn column_dict(columns: BTreeMap<&str, Value>) -> Value {
Value::from_serialize(columns)
}

fn governance_tag(key: &str, value: &str) -> Value {
Value::from_serialize(BTreeMap::from([("key", key), ("value", value)]))
}

#[test]
fn data_governance_tags_alter_sets_one_tag() {
let mut email = BTreeMap::new();
email.insert(
"data_governance_tags".to_string(),
governance_tag("my-project/data_class", "email"),
);
let mut plain = BTreeMap::new();
plain.insert("description".to_string(), Value::from("untagged column"));

let rendered = render_governance_tags(
RelationType::Table,
column_dict(BTreeMap::from([
("email", Value::from_serialize(email)),
("name", Value::from_serialize(plain)),
])),
)
.expect("governance tag SQL should render");

let sql = rendered.trim();
assert!(
sql.starts_with("alter table "),
"expected one ALTER TABLE, got: {sql}"
);
assert!(
sql.contains("alter column `email` set options (data_governance_tags=[(\"my-project/data_class\", \"email\")])"),
"missing set clause, got: {sql}"
);
assert!(
!sql.contains("`name`"),
"a column without data_governance_tags must be left alone, got: {sql}"
);
}

#[test]
fn data_governance_tags_alter_escapes_literals_and_rejects_nested_path() {
let mut email = BTreeMap::new();
email.insert("data_type".to_string(), Value::from("STRING"));
email.insert(
"data_governance_tags".to_string(),
governance_tag("my-project/data_class", "say \"hi\""),
);
let rendered = render_governance_tags(
RelationType::Table,
column_dict(BTreeMap::from([("email", Value::from_serialize(email))])),
)
.expect("quoted value should render");
let sql = rendered.trim();
assert!(
sql.contains("alter column `email` set options (data_governance_tags=[(\"my-project/data_class\", \"say \\\"hi\\\"\")])"),
"unexpected escaped SQL: {sql}"
);

let mut nested = BTreeMap::new();
nested.insert("data_type".to_string(), Value::from("STRING"));
nested.insert(
"data_governance_tags".to_string(),
governance_tag("my-project/data_class", "email"),
);
let err = render_governance_tags(
RelationType::Table,
column_dict(BTreeMap::from([(
"address.email",
Value::from_serialize(nested),
)])),
)
.expect_err("a dotted field path must fail");
assert!(
err.to_string().contains("does not accept a dotted path"),
"unexpected error: {err}"
);
}

#[test]
fn data_governance_tags_alter_rejects_struct_parent_and_bad_key() {
let mut parent = BTreeMap::new();
parent.insert(
"data_type".to_string(),
Value::from("ARRAY<STRUCT<email STRING>>"),
);
parent.insert(
"data_governance_tags".to_string(),
governance_tag("my-project/data_class", "email"),
);
let err = render_governance_tags(
RelationType::Table,
column_dict(BTreeMap::from([("address", Value::from_serialize(parent))])),
)
.expect_err("STRUCT parent must fail");
assert!(
err.to_string().contains("RECORD or STRUCT parent"),
"unexpected error: {err}"
);

let mut bad_key = BTreeMap::new();
bad_key.insert(
"data_governance_tags".to_string(),
governance_tag("data_class", "email"),
);
let err = render_governance_tags(
RelationType::Table,
column_dict(BTreeMap::from([("email", Value::from_serialize(bad_key))])),
)
.expect_err("a key without a namespace must fail");
assert!(
err.to_string().contains("{parent_id}/{short_name}"),
"unexpected error: {err}"
);

let mut view_email = BTreeMap::new();
view_email.insert(
"data_governance_tags".to_string(),
governance_tag("my-project/data_class", "email"),
);
let err = render_governance_tags(
RelationType::View,
column_dict(BTreeMap::from([(
"email",
Value::from_serialize(view_email),
)])),
)
.expect_err("a view must fail");
assert!(
err.to_string().contains("only supported on tables"),
"unexpected error: {err}"
);
}
Loading
Loading