HIVE-29668: Add -rebuildIndexes utility to reconstruct backend Metast…

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java

@@ -42,7 +42,7 @@ public class HiveSchemaHelper {
   public static final String DB_HIVE = "hive";
   public static final String DB_MSSQL = "mssql";
   public static final String DB_MYSQL = "mysql";
-  public static final String DB_POSTGRACE = "postgres";
+  public static final String DB_POSTGRES = "postgres";
   public static final String DB_ORACLE = "oracle";
   public static final String EMBEDDED_HS2_URL =
       "jdbc:hive2://?hive.conf.restricted.list=;hive.security.authorization.sqlstd.confwhitelist=.*;"
@@ -581,7 +581,7 @@ public static NestedScriptParser getDbCommandParser(String dbName,
       return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
     } else if (dbName.equalsIgnoreCase(DB_MYSQL)) {
       return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
-    } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) {
+    } else if (dbName.equalsIgnoreCase(DB_POSTGRES)) {
       return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);
     } else if (dbName.equalsIgnoreCase(DB_ORACLE)) {
       return new OracleCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine);

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java

@@ -477,6 +477,8 @@ public int run(String metastoreHome, String[] args, OptionGroup additionalOption
         task = new SchemaToolTaskDrop();
       } else if (cmdLine.hasOption("createLogsTable")) {
         task = new SchemaToolTaskCreateLogsTable();
+      } else if (cmdLine.hasOption("rebuildIndexes")) {
+        task = new SchemaToolTaskRebuildIndexes();
       } else {
         throw new HiveMetaException("No task defined!");
       }

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java

@@ -82,6 +82,8 @@
       .hasArg()
       .withDescription("Create table for Hive warehouse/compute logs")
       .create("createLogsTable");
+    Option rebuildIndexesOpt = new Option("rebuildIndexes",
+        "Rebuild indexes in the metastore backend database.");
 
     OptionGroup optGroup = new OptionGroup();
     optGroup
@@ -100,7 +102,8 @@
       .addOption(moveDatabase)
       .addOption(moveTable)
       .addOption(createUserOpt)
-      .addOption(createLogsTable);
+      .addOption(createLogsTable)
+      .addOption(rebuildIndexesOpt);
     optGroup.setRequired(true);
 
     Option userNameOpt = OptionBuilder.withArgName("user")
@@ -240,10 +243,10 @@
 
   private static final Set<String> VALID_DB_TYPES = ImmutableSet.of(HiveSchemaHelper.DB_DERBY,
       HiveSchemaHelper.DB_HIVE, HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL,
-      HiveSchemaHelper.DB_POSTGRACE, HiveSchemaHelper.DB_ORACLE);
+      HiveSchemaHelper.DB_POSTGRES, HiveSchemaHelper.DB_ORACLE);
 
   private static final Set<String> VALID_META_DB_TYPES = ImmutableSet.of(HiveSchemaHelper.DB_DERBY,
-      HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL, HiveSchemaHelper.DB_POSTGRACE,
+      HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL, HiveSchemaHelper.DB_POSTGRES,
       HiveSchemaHelper.DB_ORACLE);
 
   private void validate() throws ParseException {

standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskRebuildIndexes.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools.schematool;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class SchemaToolTaskRebuildIndexes extends SchemaToolTask {
+
+  private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskRebuildIndexes.class);
+  static final String REBUILD_INDEXES_FILE_PREFIX = "rebuild-indexes";
+
+  @Override
+  void setCommandLineArguments(SchemaToolCommandLine cl) {
+  }
+
+  @Override
+  void execute() throws HiveMetaException {
+    String dbType = schemaTool.getDbType();
+    String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir();
+    String scriptFile = REBUILD_INDEXES_FILE_PREFIX + "." + dbType + ".sql";
+    File script = new File(scriptDir, scriptFile);
+
+    if (!script.exists()) {
+      throw new HiveMetaException(
+          "-rebuildIndexes is not supported for -dbType " + dbType + ". "
+              + "Expected script not found: " + script.getAbsolutePath());
+    }
+
+    if (schemaTool.isDryRun()) {
+      try {
+        LOG.info("Dry run: would execute {}", script.getAbsolutePath());
+        LOG.info(new String(Files.readAllBytes(script.toPath())));
+      } catch (IOException e) {
+        throw new HiveMetaException("Failed to read rebuild-indexes script", e);
+      }
+      return;
+    }
+
+    LOG.info("Starting index rebuild using {}", scriptFile);
+    try {
+      schemaTool.execSql(scriptDir, scriptFile);
+    } catch (IOException e) {
+      throw new HiveMetaException("Index rebuild failed", e);
+    }
+    LOG.info("Index rebuild complete.");
+  }
+}

standalone-metastore/metastore-server/src/main/sql/mssql/rebuild-indexes.mssql.sql

@@ -0,0 +1,164 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to you under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+-- Rebuilds all HMS indexes on MSSQL using CREATE ... WITH (DROP_EXISTING = ON).
+-- Update this file whenever a new upgrade script adds an index.
+
+CREATE INDEX BUCKETING_COLS_N49 ON BUCKETING_COLS (SD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX COLUMNS_V2_N49 ON COLUMNS_V2 (CD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS (CC_DATABASE,CC_TABLE,CC_PARTITION) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX CONSTRAINTS_CONSTRAINT_TYPE_INDEX ON KEY_CONSTRAINTS (CONSTRAINT_TYPE) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX CONSTRAINTS_PARENT_TBL_ID__INDEX ON KEY_CONSTRAINTS (PARENT_TBL_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX DATABASE_PARAMS_N49 ON DATABASE_PARAMS (DB_ID) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX DBPRIVILEGEINDEX ON DB_PRIVS (AUTHORIZER,DB_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,DB_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX DB_PRIVS_N49 ON DB_PRIVS (DB_ID) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX DCPRIVILEGEINDEX ON DC_PRIVS (AUTHORIZER,NAME,PRINCIPAL_NAME,PRINCIPAL_TYPE,DC_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX DC_PRIVS_N49 ON DC_PRIVS (NAME) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX "DUMP_IDX" ON "REPLICATION_METRICS" ("RM_DUMP_EXECUTION_ID") WITH (DROP_EXISTING = ON);
+
+CREATE INDEX FUNCS_N49 ON FUNCS (DB_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX GLOBALPRIVILEGEINDEX ON GLOBAL_PRIVS (AUTHORIZER,PRINCIPAL_NAME,PRINCIPAL_TYPE,USER_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS (CREATE_TIME) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX IDX_SCHEDULED_EX_LAST_UPDATE ON "SCHEDULED_EXECUTIONS" ("LAST_UPDATE_TIME") WITH (DROP_EXISTING = ON);
+
+CREATE INDEX IDX_SCHEDULED_EX_SQ_ID ON "SCHEDULED_EXECUTIONS" ("SCHEDULED_QUERY_ID") WITH (DROP_EXISTING = ON);
+
+CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX MV_UNIQUE_TABLE ON MV_CREATION_METADATA (TBL_NAME,DB_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX NOTIFICATION_LOG_EVENT_ID ON NOTIFICATION_LOG (EVENT_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (AUTHORIZER,PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PARTITIONS_N49 ON PARTITIONS (SD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PARTITION_KEYS_N49 ON PARTITION_KEYS (TBL_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PARTITION_KEY_VALS_N49 ON PARTITION_KEY_VALS (PART_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PARTITION_PARAMS_N49 ON PARTITION_PARAMS (PART_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PARTPRIVILEGEINDEX ON PART_PRIVS (AUTHORIZER,PART_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PART_COL_PRIVS_N49 ON PART_COL_PRIVS (PART_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PART_PRIVS_N49 ON PART_PRIVS (PART_ID) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX PART_TABLE_PK ON SEQUENCE_TABLE (SEQUENCE_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (PART_ID,COLUMN_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX "POLICY_IDX" ON "REPLICATION_METRICS" ("RM_POLICY") WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX ROLEENTITYINDEX ON ROLES (ROLE_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX ROLE_MAP_N49 ON ROLE_MAP (ROLE_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SDS_N49 ON SDS (SERDE_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SDS_N50 ON SDS (CD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SD_PARAMS_N49 ON SD_PARAMS (SD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SERDE_PARAMS_N49 ON SERDE_PARAMS (SERDE_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SKEWED_COL_NAMES_N49 ON SKEWED_COL_NAMES (SD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SKEWED_COL_VALUE_LOC_MAP_N49 ON SKEWED_COL_VALUE_LOC_MAP (SD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SKEWED_COL_VALUE_LOC_MAP_N50 ON SKEWED_COL_VALUE_LOC_MAP (STRING_LIST_ID_KID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SKEWED_STRING_LIST_VALUES_N49 ON SKEWED_STRING_LIST_VALUES (STRING_LIST_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SKEWED_VALUES_N49 ON SKEWED_VALUES (STRING_LIST_ID_EID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SKEWED_VALUES_N50 ON SKEWED_VALUES (SD_ID_OID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX SORT_COLS_N49 ON SORT_COLS (SD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TABLECOLUMNPRIVILEGEINDEX ON TBL_COL_PRIVS (AUTHORIZER,TBL_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_COL_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TABLEPRIVILEGEINDEX ON TBL_PRIVS (AUTHORIZER,TBL_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TABLE_PARAMS_N49 ON TABLE_PARAMS (TBL_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (TBL_ID, COLUMN_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TBLS_N50 ON TBLS (SD_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TBL_COL_PRIVS_N49 ON TBL_COL_PRIVS (TBL_ID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TBL_PRIVS_N49 ON TBL_PRIVS (TBL_ID) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID) WITH (DROP_EXISTING = ON);
+
+CREATE INDEX TYPE_FIELDS_N49 ON TYPE_FIELDS (TYPE_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUEDATABASE ON DBS ("NAME", "CTLG_NAME") WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUEFUNCTION ON FUNCS (FUNC_NAME,DB_ID) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (TBL_ID,PART_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX "UNIQUEPKG" ON "PACKAGES" ("NAME", "DB_ID") WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX "UNIQUESTOREDPROC" ON "STORED_PROCS" ("NAME", "DB_ID") WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUETYPE ON TYPES (TYPE_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUE_CTLG ON CTLGS ("NAME") WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUE_WM_MAPPING ON WM_MAPPING (RP_ID, ENTITY_TYPE, ENTITY_NAME) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUE_WM_POOL ON WM_POOL (RP_ID, PATH) WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NS", "NAME") WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX UNIQUE_WM_TRIGGER ON WM_TRIGGER (RP_ID, "NAME") WITH (DROP_EXISTING = ON);
+
+CREATE UNIQUE INDEX USERROLEMAPINDEX ON ROLE_MAP (PRINCIPAL_NAME,ROLE_ID,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON);
+