Skip to content

Fixes #30678: propagate inherited domain to descendants in search on asset move #31137

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sonika-shah wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fixes #30678: propagate inherited domain to descendants in search on asset move #31137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 180 additions & 0 deletions ...data-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,6 +723,34 @@ private Table createTestTable(TestNamespace ns, String suffix, Domain domain) {
return SdkClients.adminClient().tables().create(createTable);
}

private org.openmetadata.schema.entity.data.DatabaseSchema createSchemaWithDomain(
TestNamespace ns, String suffix, Domain domain) {
DatabaseService service = getOrCreateDatabaseService(ns);
org.openmetadata.schema.entity.data.Database database =
getOrCreateDatabase(ns, service.getFullyQualifiedName());
org.openmetadata.schema.api.data.CreateDatabaseSchema create =
new org.openmetadata.schema.api.data.CreateDatabaseSchema()
.withName(ns.prefix(suffix))
.withDatabase(database.getFullyQualifiedName())
.withDomains(List.of(domain.getFullyQualifiedName()));
return SdkClients.adminClient().databaseSchemas().create(create);
}

private Table createChildTable(
TestNamespace ns,
String suffix,
org.openmetadata.schema.entity.data.DatabaseSchema schema,
Domain domain) {
CreateTable createTable =
new CreateTable()
.withName(ns.prefix(suffix))
.withDatabaseSchema(schema.getFullyQualifiedName());
if (domain != null) {
createTable.withDomains(List.of(domain.getFullyQualifiedName()));
}
return SdkClients.adminClient().tables().create(createTable);
}

private Dashboard createTestDashboard(TestNamespace ns, String suffix, Domain domain) {
DashboardService service = DashboardServiceTestFactory.createMetabase(ns);

Expand Down Expand Up @@ -860,6 +888,40 @@ private List<EntityReference> getEntityReferencesFromSearchIndex(
return JsonUtils.readObjects(fieldNode.toString(), EntityReference.class);
}

private void assertSingleSearchDomain(UUID tableId, Domain expected, String message)
throws Exception {
assertSingleSearchDomain(tableId, expected, message, "table_search_index");
}

private void assertSingleSearchDomain(
UUID entityId, Domain expected, String message, String indexName) throws Exception {
List<EntityReference> searchDomains =
getEntityReferencesFromSearchIndex(entityId, indexName, "domains");
assertNotNull(searchDomains, message);
assertEquals(1, searchDomains.size(), message);
assertEquals(expected.getId(), searchDomains.getFirst().getId(), message);
}

private org.openmetadata.schema.entity.data.Database createDatabaseWithDomain(
TestNamespace ns, String suffix, Domain domain) {
DatabaseService service = getOrCreateDatabaseService(ns);
org.openmetadata.schema.api.data.CreateDatabase create =
new org.openmetadata.schema.api.data.CreateDatabase()
.withName(ns.prefix(suffix))
.withService(service.getFullyQualifiedName())
.withDomains(List.of(domain.getFullyQualifiedName()));
return SdkClients.adminClient().databases().create(create);
}

private org.openmetadata.schema.entity.data.DatabaseSchema createSchemaUnderDatabase(
TestNamespace ns, String suffix, org.openmetadata.schema.entity.data.Database database) {
org.openmetadata.schema.api.data.CreateDatabaseSchema create =
new org.openmetadata.schema.api.data.CreateDatabaseSchema()
.withName(ns.prefix(suffix))
.withDatabase(database.getFullyQualifiedName());
return SdkClients.adminClient().databaseSchemas().create(create);
}

// ===================================================================
// RENAME + CONSOLIDATION TESTS
// Tests that verify assets are preserved when:
Expand Down Expand Up @@ -1393,6 +1455,124 @@ void test_changeDataProductDomain_withAssetMigration(TestNamespace ns) throws Ex
});
}

@Test
void test_changeDataProductDomain_propagatesInheritedDomainToChildTablesInSearch(TestNamespace ns)
throws Exception {
Domain finance = createTestDomain(ns, "dp_inherit_finance");
Domain hr = createTestDomain(ns, "dp_inherit_hr");
Domain marketing = createTestDomain(ns, "dp_inherit_marketing");

// Schema with an explicit Finance domain, assigned as the data product's only asset
org.openmetadata.schema.entity.data.DatabaseSchema schema =
createSchemaWithDomain(ns, "dp_inherit_schema", finance);

// Children under the schema: t1 inherits Finance, t2 is explicitly Finance, t3 is Marketing
Table inheritedChild = createChildTable(ns, "dp_inherit_t1", schema, null);
Table explicitSameDomainChild = createChildTable(ns, "dp_inherit_t2", schema, finance);
Table explicitOtherDomainChild = createChildTable(ns, "dp_inherit_t3", schema, marketing);

CreateDataProduct create =
new CreateDataProduct()
.withName(ns.prefix("dp_inherit"))
.withDescription("Data product for inherited-domain search propagation test")
.withDomains(List.of(finance.getFullyQualifiedName()));
DataProduct dataProduct = createEntity(create);

bulkAddAssets(
dataProduct.getFullyQualifiedName(),
new BulkAssets().withAssets(List.of(schema.getEntityReference())));

Awaitility.await("Wait for schema asset to be linked")
.pollInterval(Duration.ofSeconds(1))
.atMost(Duration.ofSeconds(15))
.untilAsserted(
() -> assertEquals(1, getAssets(dataProduct.getId(), 10, 0).getPaging().getTotal()));

// Move the data product, and with it the schema asset, from Finance to HR
dataProduct.setDomains(List.of(hr.getEntityReference()));
patchEntity(dataProduct.getId().toString(), dataProduct);

Awaitility.await("Wait for inherited-domain search propagation to child tables")
.atMost(Duration.ofSeconds(30))
.pollDelay(Duration.ofMillis(500))
.pollInterval(Duration.ofSeconds(2))
.ignoreExceptions()
.untilAsserted(
() -> {
assertSingleSearchDomain(
inheritedChild.getId(),
hr,
"Child table that inherits its domain should follow the data product to HR in search");
assertSingleSearchDomain(
explicitSameDomainChild.getId(),
finance,
"Child table with an explicit Finance domain should stay in Finance in search");
assertSingleSearchDomain(
explicitOtherDomainChild.getId(),
marketing,
"Child table with an explicit Marketing domain should keep it in search");
});
}

@Test
void test_changeDataProductDomain_multiLevelDescendantsFollowInSearch(TestNamespace ns)
throws Exception {
Domain finance = createTestDomain(ns, "dp_ml_finance");
Domain hr = createTestDomain(ns, "dp_ml_hr");
Domain marketing = createTestDomain(ns, "dp_ml_marketing");

// Database (explicit Finance) is the asset; the schema and one table inherit through it, two
// levels down, so the move must reach descendants keyed on database.id in search.
org.openmetadata.schema.entity.data.Database database =
createDatabaseWithDomain(ns, "dp_ml_db", finance);
org.openmetadata.schema.entity.data.DatabaseSchema schema =
createSchemaUnderDatabase(ns, "dp_ml_schema", database);
Table inheritedTable = createChildTable(ns, "dp_ml_t1", schema, null);
Table explicitTable = createChildTable(ns, "dp_ml_t2", schema, marketing);

CreateDataProduct create =
new CreateDataProduct()
.withName(ns.prefix("dp_ml"))
.withDescription("Data product for multi-level domain propagation test")
.withDomains(List.of(finance.getFullyQualifiedName()));
DataProduct dataProduct = createEntity(create);

bulkAddAssets(
dataProduct.getFullyQualifiedName(),
new BulkAssets().withAssets(List.of(database.getEntityReference())));

Awaitility.await("Wait for database asset to be linked")
.pollInterval(Duration.ofSeconds(1))
.atMost(Duration.ofSeconds(15))
.untilAsserted(
() -> assertEquals(1, getAssets(dataProduct.getId(), 10, 0).getPaging().getTotal()));

dataProduct.setDomains(List.of(hr.getEntityReference()));
patchEntity(dataProduct.getId().toString(), dataProduct);

Awaitility.await("Wait for multi-level inherited-domain search propagation")
.atMost(Duration.ofSeconds(30))
.pollDelay(Duration.ofMillis(500))
.pollInterval(Duration.ofSeconds(2))
.ignoreExceptions()
.untilAsserted(
() -> {
assertSingleSearchDomain(
schema.getId(),
hr,
"Inheriting schema (child of the moved database) should follow to HR in search",
"database_schema_search_index");
assertSingleSearchDomain(
inheritedTable.getId(),
hr,
"Inheriting grandchild table should follow to HR in search");
assertSingleSearchDomain(
explicitTable.getId(),
marketing,
"Grandchild table with an explicit domain should keep it in search");
});
}

@Test
void test_changeDataProductDomain_multipleAssets(TestNamespace ns) throws Exception {
// Create two domains
Expand Down
Loading
Loading