From 7758ee3263174e82cbe03ee49f06638375fafc72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E5=B0=8F=E9=9D=92?= Date: Fri, 31 Jul 2026 17:50:34 +0800 Subject: [PATCH] HIVE-29584: Invalidate RawStore after Direct SQL connection failures Preserve JDBC connection errors and remove the cached RawStore before shutdown, allowing RetryingHMSHandler to retry with a fresh PersistenceManager. Make post-shutdown rollback idempotent so outer handler cleanup cannot mask the original failure. Co-Authored-By: GPT-5 Codex --- .../hive/metastore/DatabaseProduct.java | 24 +++++++++++ .../hive/metastore/HMSHandlerContext.java | 15 +++++++ .../hadoop/hive/metastore/ObjectStore.java | 14 ++++-- .../directsql/MetaStoreDirectSql.java | 4 +- .../hive/metastore/metastore/GetHelper.java | 27 +++++++++++- .../hive/metastore/TestObjectStore.java | 43 ++++++++++++++++++- 6 files changed, 119 insertions(+), 8 deletions(-) diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java index cd78a18bf53f..7e00e109c946 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DatabaseProduct.java @@ -21,6 +21,9 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.SQLNonTransientConnectionException; +import java.sql.SQLRecoverableException; +import java.sql.SQLTransientConnectionException; import java.sql.SQLTransactionRollbackException; import java.sql.Timestamp; import java.util.ArrayList; @@ -217,6 +220,27 @@ public static boolean isRecoverableException(Throwable t) { .allMatch(ex -> ExceptionUtils.indexOfType(t, ex) < 0); } + public static boolean isConnectionException(Throwable t) { + while (t != null) { + if (t instanceof SQLNonTransientConnectionException + || t instanceof SQLRecoverableException + || t instanceof SQLTransientConnectionException) { + return true; + } + if (t instanceof SQLException sqlException) { + for (SQLException current = sqlException; current != null; + current = current.getNextException()) { + String sqlState = current.getSQLState(); + if (sqlState != null && sqlState.startsWith("08")) { + return true; + } + } + } + t = t.getCause(); + } + return false; + } + public final boolean isDERBY() { return dbType == DbType.DERBY; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandlerContext.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandlerContext.java index 90dd0a4a7c97..35bc3c6bfd81 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandlerContext.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandlerContext.java @@ -155,6 +155,21 @@ public static void setRawStore(RawStore rawStore) { context.get().rawStore = rawStore; } + /** + * Remove and shut down the RawStore associated with the current handler thread. + * + * @return true if a RawStore was present + */ + public static boolean invalidateRawStore() { + RawStore rawStore = context.get().rawStore; + context.get().rawStore = null; + if (rawStore == null) { + return false; + } + rawStore.shutdown(); + return true; + } + public static void setTxnStore(TxnStore txnStore) { context.get().txnStore = txnStore; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 9a42ba9f01a5..3dd1cea433b9 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -377,9 +377,13 @@ public PersistenceManager getPersistenceManager() { @Override public void shutdown() { LOG.info("RawStore: {}, with PersistenceManager: {} will be shutdown", this, pm); - if (pm != null) { - pm.close(); - pm = null; + PersistenceManager persistenceManager = pm; + pm = null; + currentTransaction = null; + openTrasactionCalls = 0; + transactionStatus = TXN_STATUS.NO_STATE; + if (persistenceManager != null) { + persistenceManager.close(); } } @@ -548,7 +552,9 @@ public void rollbackTransaction() { // remove all detached objects from the cache, since the transaction is // being rolled back they are no longer relevant, and this prevents them // from reattaching in future transactions - pm.evictAll(); + if (pm != null) { + pm.evictAll(); + } } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/MetaStoreDirectSql.java index 7c95577e2614..8b2de2069d25 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/directsql/MetaStoreDirectSql.java @@ -1983,7 +1983,9 @@ public void prepareTxn() throws MetaException { assert pm.currentTransaction().isActive(); // must be inside tx together with queries executeNoResult(stmt); } catch (SQLException sqlEx) { - throw new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + MetaException metaException = new MetaException("Error setting ansi quotes: " + sqlEx.getMessage()); + metaException.initCause(sqlEx); + throw metaException; } } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java index 4ddc43083a08..451381da9362 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metastore/GetHelper.java @@ -21,6 +21,7 @@ import com.codahale.metrics.Counter; import com.google.common.annotations.VisibleForTesting; +import javax.jdo.JDODataStoreException; import javax.jdo.JDOException; import javax.jdo.PersistenceManager; import java.util.List; @@ -28,6 +29,7 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.DatabaseProduct; import org.apache.hadoop.hive.metastore.ExceptionHandler; +import org.apache.hadoop.hive.metastore.HMSHandlerContext; import org.apache.hadoop.hive.metastore.directsql.MetaStoreDirectSql; import org.apache.hadoop.hive.metastore.RawStore; import org.apache.hadoop.hive.metastore.api.InvalidInputException; @@ -59,6 +61,7 @@ public abstract class GetHelper { protected final List partitionFields; protected final A argument; private boolean success = false; + private boolean storeInvalidated = false; protected T results = null; public GetHelper(RawStoreBundle rsb, A args) throws MetaException { @@ -148,6 +151,14 @@ private void start(boolean initTable) throws MetaException, NoSuchObjectExceptio } private void handleDirectSqlError(Exception ex, String savePoint) throws MetaException, NoSuchObjectException { + if (DatabaseProduct.isConnectionException(ex)) { + LOG.warn("Direct SQL failed because the metastore database connection is not usable", ex); + directSqlErrors.inc(); + invalidateRawStore(ex); + throw ExceptionHandler.newMetaException(new JDODataStoreException( + "Direct SQL failed because the metastore database connection is not usable", ex)); + } + String message = null; try { message = generateShorterMessage(ex); @@ -195,6 +206,18 @@ private void handleDirectSqlError(Exception ex, String savePoint) throws MetaExc doUseDirectSql = false; } + private void invalidateRawStore(Exception originalException) { + storeInvalidated = true; + try { + if (!HMSHandlerContext.invalidateRawStore()) { + baseStore.shutdown(); + } + } catch (Exception cleanupException) { + originalException.addSuppressed(cleanupException); + LOG.warn("Failed to shut down RawStore after a Direct SQL connection error", cleanupException); + } + } + public void setTransactionSavePoint(String savePoint) { if (savePoint != null) { ((JDOTransaction) pm.currentTransaction()).setSavepoint(savePoint); @@ -252,7 +275,7 @@ private T commit() { } private void close() { - if (!success) { + if (!success && !storeInvalidated) { baseStore.rollbackTransaction(); } } @@ -278,4 +301,4 @@ public static Counter setDirectSqlErrors(Counter counter) { directSqlErrors = counter; return counter; } -} \ No newline at end of file +} diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java index e2f87ee4fdb4..d34761396d25 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -100,6 +100,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.jdo.JDODataStoreException; import javax.jdo.Query; import java.sql.Connection; import java.sql.DriverManager; @@ -1424,6 +1425,47 @@ protected Database getJdoResult() throws MetaException, Assert.assertEquals(1, directSqlErrors.getCount()); } + @Test + public void testDirectSqlConnectionErrorInvalidatesRawStore() throws Exception { + AtomicBoolean runJdo = new AtomicBoolean(false); + MetaException directSqlFailure = null; + objectStore.openTransaction(); + HMSHandlerContext.setRawStore(objectStore); + try { + new GetHelper(objectStore.createRawStoreBundle(), + new DatabaseName(DEFAULT_CATALOG_NAME, "foo")) { + @Override + protected Database getSqlResult() throws MetaException { + MetaException ex = new MetaException("Direct SQL connection failure"); + ex.initCause(new SQLException("Communication link failure", "08S01")); + throw ex; + } + + @Override + protected String describeResult() { + return ""; + } + + @Override + protected Database getJdoResult() { + runJdo.set(true); + return null; + } + }.run(false); + Assert.fail("Expected a connection-level Direct SQL failure"); + } catch (MetaException ex) { + directSqlFailure = ex; + Assert.assertFalse(HMSHandlerContext.getRawStore().isPresent()); + } finally { + // Handlers that own an outer transaction still execute this cleanup after GetHelper fails. + objectStore.rollbackTransaction(); + HMSHandlerContext.clear(); + } + + Assert.assertTrue(directSqlFailure.getCause() instanceof JDODataStoreException); + Assert.assertFalse(runJdo.get()); + } + @Test(expected = MetaException.class) public void testLockDbTableThrowsExceptionWhenTableIsNotAllowedToLock() throws Exception { MetaStoreDirectSql metaStoreDirectSql = new MetaStoreDirectSql(objectStore.getPersistenceManager(), conf, null); @@ -2195,4 +2237,3 @@ AutoCloseable directsql(boolean tryDirectSql) { }; } } -