From 2aeda5b193c12e397a5bd17db1fa74c2a0db3a41 Mon Sep 17 00:00:00 2001 From: mohitdeuex Date: Thu, 6 Aug 2026 22:01:43 +0530 Subject: [PATCH] fix(search): don't fail the reindex when the embedding provider is unreachable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #30364 un-gated the staged chunk recreate, so every job-driven full recreate now calls beginStagedChunkRecreate(), which opens with a pre-flight embed. The pre-flight rethrows, and that exception fails the whole SearchIndexApp run — so a reindex that needs no embeddings at all is now blocked by an optional AI provider being unreachable. That is not a narrow case. Collate ships semanticSearchEnabled=true by default while llmConfiguration.embeddings.provider defaults to bedrock, and BedrockEmbeddingClient's constructor never calls AWS — it validates the model id, dimension and region, then builds the SDK client. So on any deployment where a region resolves (anything on AWS) but bedrock:InvokeModel was never granted, the client constructs happily, initializeVectorSearchService reports success, and the first full reindex dies: User: arn:aws:sts::...:assumed-role/... is not authorized to perform: bedrock:InvokeModel on resource: .../amazon.titan-embed-text-v2:0 (Status Code: 403) surfacing as status='failed' within seconds with an empty failureContext. The misconfiguration is otherwise invisible: live indexing logs embedding errors and carries on, so the deployment looks healthy right up until someone reindexes. This is what has been failing the nightly Java IT suites on main and 2.0. Treat an unavailable provider as "do not stage" rather than "fail": return null, which the caller already handles as the partial-recreate outcome — existing chunks stay live and are swept by the next recreate that runs with a working provider. markEntityTypeReindexed already ignores marks from a run without staging, so this reuses a supported state rather than inventing one. Genuine staging failures — an indeterminate live-target probe, a failed index create — still throw, because those mean continuing could destroy live chunks. --- .../vector/OpenSearchVectorService.java | 34 +++++++++++++++---- ...enSearchVectorServiceChunkStagingTest.java | 23 +++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/vector/OpenSearchVectorService.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/vector/OpenSearchVectorService.java index 8159a382298a..ee5cf96d8ceb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/vector/OpenSearchVectorService.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/vector/OpenSearchVectorService.java @@ -434,11 +434,27 @@ public List searchChunksByParent(String parentId, String query, int k) { /** * Begins a staged recreate for a full-recreate run: pre-flights the embedding client (a broken * client would make the whole run pointless), sweeps generations orphaned by crashed runs, and - * creates the next generation bare — no aliases, so reads keep hitting the old chunks. Throws on - * any failure; nothing has been destroyed at that point. + * creates the next generation bare — no aliases, so reads keep hitting the old chunks. + * + *

Returns {@code null} when the embedding client cannot serve a request, meaning "not staged": + * the caller carries on with a normal in-place reindex and the existing chunks stay live, exactly + * as they do for a partial recreate. An unreachable embedding provider is a reason not to stage a + * generation we could never finish — it is not a reason to fail the entity reindex, which does + * not need embeddings at all. Note the provider is only exercised here: client construction makes + * no call to it, and live indexing logs and continues on embedding errors, so a deployment with + * semantic search enabled but no working provider looks healthy right up until a reindex. + * + *

Still throws on genuine staging failures (indeterminate live-target probe, index create) — + * those mean the cluster is in a state where continuing could destroy live chunks. */ public String beginStagedChunkRecreate() { - preflightEmbedding(); + if (!isEmbeddingAvailable()) { + LOG.warn( + "Embedding pre-flight failed — skipping the staged chunk recreate. The entity reindex " + + "continues and existing chunks stay live; orphaned chunks, if any, are swept by the " + + "next recreate that runs with a working embedding provider."); + return null; + } synchronized (stagedChunkLock) { String base = getChunkIndexName(); String liveTarget = requireResolvedLiveChunkTarget(base); @@ -479,12 +495,18 @@ public void clearStagedChunkState(String generation) { } } - private void preflightEmbedding() { + /** + * Whether the embedding client can actually serve a request. Logged rather than thrown: the only + * caller treats an unavailable provider as "do not stage", and the stack trace belongs in the log + * next to the provider's own error, not wrapped in a reindex failure. + */ + private boolean isEmbeddingAvailable() { try { embeddingClient.embedQuery("chunk index recreate pre-flight"); + return true; } catch (Exception e) { - throw new RuntimeException( - "Refusing to start a staged chunk-index recreate: embedding client pre-flight failed", e); + LOG.warn("Embedding client pre-flight failed: {}", e.getMessage(), e); + return false; } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/vector/OpenSearchVectorServiceChunkStagingTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/vector/OpenSearchVectorServiceChunkStagingTest.java index 71d9f5779568..5d233262e49f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/vector/OpenSearchVectorServiceChunkStagingTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/vector/OpenSearchVectorServiceChunkStagingTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -113,4 +114,26 @@ void beginStagedChunkRecreate_abortsWhenTheLiveTargetProbeIsIndeterminate() thro "abort must name the unresolved live target. Got: " + failure.getMessage()); verify(client, never()).generic(); } + + @Test + void beginStagedChunkRecreate_skipsStagingWhenTheEmbeddingProviderIsUnavailable() + throws IOException { + // An unreachable embedding provider must not fail the reindex. Semantic search ships enabled + // with the provider defaulting to bedrock, and the client constructs without ever calling it, + // so any deployment that never set Bedrock up reaches this pre-flight — and the entity reindex + // it would abort does not need embeddings at all. Skip staging, leave the old chunks live, and + // touch nothing in the cluster. + OpenSearchClient client = mock(OpenSearchClient.class); + EmbeddingClient embeddingClient = mock(EmbeddingClient.class); + when(embeddingClient.embedQuery(any(String.class))) + .thenThrow(new RuntimeException("Bedrock embedding generation failed (AWS service error)")); + + OpenSearchVectorService service = new OpenSearchVectorService(client, embeddingClient); + + assertNull( + service.beginStagedChunkRecreate(), + "an unavailable embedding provider must read as 'not staged', not as a failure"); + verify(client, never()).indices(); + verify(client, never()).generic(); + } }