diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java index 47000cc41b20..041ef39e924e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/HiveSchemaHelper.java @@ -42,7 +42,7 @@ public class HiveSchemaHelper { public static final String DB_HIVE = "hive"; public static final String DB_MSSQL = "mssql"; public static final String DB_MYSQL = "mysql"; - public static final String DB_POSTGRACE = "postgres"; + public static final String DB_POSTGRES = "postgres"; public static final String DB_ORACLE = "oracle"; public static final String EMBEDDED_HS2_URL = "jdbc:hive2://?hive.conf.restricted.list=;hive.security.authorization.sqlstd.confwhitelist=.*;" @@ -581,7 +581,7 @@ public static NestedScriptParser getDbCommandParser(String dbName, return new MSSQLCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); } else if (dbName.equalsIgnoreCase(DB_MYSQL)) { return new MySqlCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); - } else if (dbName.equalsIgnoreCase(DB_POSTGRACE)) { + } else if (dbName.equalsIgnoreCase(DB_POSTGRES)) { return new PostgresCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); } else if (dbName.equalsIgnoreCase(DB_ORACLE)) { return new OracleCommandParser(dbOpts, msUsername, msPassword, conf, usingSqlLine); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java index 2dc07cb319bf..acaadac7915d 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/MetastoreSchemaTool.java @@ -477,6 +477,8 @@ public int run(String metastoreHome, String[] args, OptionGroup additionalOption task = new SchemaToolTaskDrop(); } else if (cmdLine.hasOption("createLogsTable")) { task = new SchemaToolTaskCreateLogsTable(); + } else if (cmdLine.hasOption("rebuildIndexes")) { + task = new SchemaToolTaskRebuildIndexes(); } else { throw new HiveMetaException("No task defined!"); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java index 72c1e1ac4884..6ea53d33867f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolCommandLine.java @@ -82,6 +82,8 @@ private Options createOptions(OptionGroup additionalOptions) { .hasArg() .withDescription("Create table for Hive warehouse/compute logs") .create("createLogsTable"); + Option rebuildIndexesOpt = new Option("rebuildIndexes", + "Rebuild indexes in the metastore backend database."); OptionGroup optGroup = new OptionGroup(); optGroup @@ -100,7 +102,8 @@ private Options createOptions(OptionGroup additionalOptions) { .addOption(moveDatabase) .addOption(moveTable) .addOption(createUserOpt) - .addOption(createLogsTable); + .addOption(createLogsTable) + .addOption(rebuildIndexesOpt); optGroup.setRequired(true); Option userNameOpt = OptionBuilder.withArgName("user") @@ -240,10 +243,10 @@ private CommandLine getCommandLine(String[] args) throws ParseException { private static final Set VALID_DB_TYPES = ImmutableSet.of(HiveSchemaHelper.DB_DERBY, HiveSchemaHelper.DB_HIVE, HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL, - HiveSchemaHelper.DB_POSTGRACE, HiveSchemaHelper.DB_ORACLE); + HiveSchemaHelper.DB_POSTGRES, HiveSchemaHelper.DB_ORACLE); private static final Set VALID_META_DB_TYPES = ImmutableSet.of(HiveSchemaHelper.DB_DERBY, - HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL, HiveSchemaHelper.DB_POSTGRACE, + HiveSchemaHelper.DB_MSSQL, HiveSchemaHelper.DB_MYSQL, HiveSchemaHelper.DB_POSTGRES, HiveSchemaHelper.DB_ORACLE); private void validate() throws ParseException { diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskRebuildIndexes.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskRebuildIndexes.java new file mode 100644 index 000000000000..6b496ab44b3e --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/tools/schematool/SchemaToolTaskRebuildIndexes.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools.schematool; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +import org.apache.hadoop.hive.metastore.HiveMetaException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class SchemaToolTaskRebuildIndexes extends SchemaToolTask { + + private static final Logger LOG = LoggerFactory.getLogger(SchemaToolTaskRebuildIndexes.class); + static final String REBUILD_INDEXES_FILE_PREFIX = "rebuild-indexes"; + + @Override + void setCommandLineArguments(SchemaToolCommandLine cl) { + } + + @Override + void execute() throws HiveMetaException { + String dbType = schemaTool.getDbType(); + String scriptDir = schemaTool.getMetaStoreSchemaInfo().getMetaStoreScriptDir(); + String scriptFile = REBUILD_INDEXES_FILE_PREFIX + "." + dbType + ".sql"; + File script = new File(scriptDir, scriptFile); + + if (!script.exists()) { + throw new HiveMetaException( + "-rebuildIndexes is not supported for -dbType " + dbType + ". " + + "Expected script not found: " + script.getAbsolutePath()); + } + + if (schemaTool.isDryRun()) { + try { + LOG.info("Dry run: would execute {}", script.getAbsolutePath()); + LOG.info(new String(Files.readAllBytes(script.toPath()))); + } catch (IOException e) { + throw new HiveMetaException("Failed to read rebuild-indexes script", e); + } + return; + } + + LOG.info("Starting index rebuild using {}", scriptFile); + try { + schemaTool.execSql(scriptDir, scriptFile); + } catch (IOException e) { + throw new HiveMetaException("Index rebuild failed", e); + } + LOG.info("Index rebuild complete."); + } +} \ No newline at end of file diff --git a/standalone-metastore/metastore-server/src/main/sql/mssql/rebuild-indexes.mssql.sql b/standalone-metastore/metastore-server/src/main/sql/mssql/rebuild-indexes.mssql.sql new file mode 100644 index 000000000000..4fd52ff999b2 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/sql/mssql/rebuild-indexes.mssql.sql @@ -0,0 +1,164 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to you under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + + +-- Rebuilds all HMS indexes on MSSQL using CREATE ... WITH (DROP_EXISTING = ON). +-- Update this file whenever a new upgrade script adds an index. + +CREATE INDEX BUCKETING_COLS_N49 ON BUCKETING_COLS (SD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX COLUMNS_V2_N49 ON COLUMNS_V2 (CD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS (CC_DATABASE,CC_TABLE,CC_PARTITION) WITH (DROP_EXISTING = ON); + +CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION) WITH (DROP_EXISTING = ON); + +CREATE INDEX CONSTRAINTS_CONSTRAINT_TYPE_INDEX ON KEY_CONSTRAINTS (CONSTRAINT_TYPE) WITH (DROP_EXISTING = ON); + +CREATE INDEX CONSTRAINTS_PARENT_TBL_ID__INDEX ON KEY_CONSTRAINTS (PARENT_TBL_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX DATABASE_PARAMS_N49 ON DATABASE_PARAMS (DB_ID) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX DBPRIVILEGEINDEX ON DB_PRIVS (AUTHORIZER,DB_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,DB_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON); + +CREATE INDEX DB_PRIVS_N49 ON DB_PRIVS (DB_ID) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX DCPRIVILEGEINDEX ON DC_PRIVS (AUTHORIZER,NAME,PRINCIPAL_NAME,PRINCIPAL_TYPE,DC_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON); + +CREATE INDEX DC_PRIVS_N49 ON DC_PRIVS (NAME) WITH (DROP_EXISTING = ON); + +CREATE INDEX "DUMP_IDX" ON "REPLICATION_METRICS" ("RM_DUMP_EXECUTION_ID") WITH (DROP_EXISTING = ON); + +CREATE INDEX FUNCS_N49 ON FUNCS (DB_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX GLOBALPRIVILEGEINDEX ON GLOBAL_PRIVS (AUTHORIZER,PRINCIPAL_NAME,PRINCIPAL_TYPE,USER_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON); + +CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS (CREATE_TIME) WITH (DROP_EXISTING = ON); + +CREATE INDEX IDX_SCHEDULED_EX_LAST_UPDATE ON "SCHEDULED_EXECUTIONS" ("LAST_UPDATE_TIME") WITH (DROP_EXISTING = ON); + +CREATE INDEX IDX_SCHEDULED_EX_SQ_ID ON "SCHEDULED_EXECUTIONS" ("SCHEDULED_QUERY_ID") WITH (DROP_EXISTING = ON); + +CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID) WITH (DROP_EXISTING = ON); + +CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID) WITH (DROP_EXISTING = ON); + +CREATE INDEX MV_UNIQUE_TABLE ON MV_CREATION_METADATA (TBL_NAME,DB_NAME) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX NOTIFICATION_LOG_EVENT_ID ON NOTIFICATION_LOG (EVENT_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (AUTHORIZER,PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON); + +CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME) WITH (DROP_EXISTING = ON); + +CREATE INDEX PARTITIONS_N49 ON PARTITIONS (SD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX PARTITION_KEYS_N49 ON PARTITION_KEYS (TBL_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX PARTITION_KEY_VALS_N49 ON PARTITION_KEY_VALS (PART_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX PARTITION_PARAMS_N49 ON PARTITION_PARAMS (PART_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX PARTPRIVILEGEINDEX ON PART_PRIVS (AUTHORIZER,PART_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON); + +CREATE INDEX PART_COL_PRIVS_N49 ON PART_COL_PRIVS (PART_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX PART_PRIVS_N49 ON PART_PRIVS (PART_ID) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX PART_TABLE_PK ON SEQUENCE_TABLE (SEQUENCE_NAME) WITH (DROP_EXISTING = ON); + +CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (PART_ID,COLUMN_NAME) WITH (DROP_EXISTING = ON); + +CREATE INDEX "POLICY_IDX" ON "REPLICATION_METRICS" ("RM_POLICY") WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX ROLEENTITYINDEX ON ROLES (ROLE_NAME) WITH (DROP_EXISTING = ON); + +CREATE INDEX ROLE_MAP_N49 ON ROLE_MAP (ROLE_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SDS_N49 ON SDS (SERDE_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SDS_N50 ON SDS (CD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SD_PARAMS_N49 ON SD_PARAMS (SD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SERDE_PARAMS_N49 ON SERDE_PARAMS (SERDE_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SKEWED_COL_NAMES_N49 ON SKEWED_COL_NAMES (SD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SKEWED_COL_VALUE_LOC_MAP_N49 ON SKEWED_COL_VALUE_LOC_MAP (SD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SKEWED_COL_VALUE_LOC_MAP_N50 ON SKEWED_COL_VALUE_LOC_MAP (STRING_LIST_ID_KID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SKEWED_STRING_LIST_VALUES_N49 ON SKEWED_STRING_LIST_VALUES (STRING_LIST_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SKEWED_VALUES_N49 ON SKEWED_VALUES (STRING_LIST_ID_EID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SKEWED_VALUES_N50 ON SKEWED_VALUES (SD_ID_OID) WITH (DROP_EXISTING = ON); + +CREATE INDEX SORT_COLS_N49 ON SORT_COLS (SD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX TABLECOLUMNPRIVILEGEINDEX ON TBL_COL_PRIVS (AUTHORIZER,TBL_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_COL_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON); + +CREATE INDEX TABLEPRIVILEGEINDEX ON TBL_PRIVS (AUTHORIZER,TBL_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_PRIV,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON); + +CREATE INDEX TABLE_PARAMS_N49 ON TABLE_PARAMS (TBL_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (TBL_ID, COLUMN_NAME) WITH (DROP_EXISTING = ON); + +CREATE INDEX TBLS_N50 ON TBLS (SD_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX TBL_COL_PRIVS_N49 ON TBL_COL_PRIVS (TBL_ID) WITH (DROP_EXISTING = ON); + +CREATE INDEX TBL_PRIVS_N49 ON TBL_PRIVS (TBL_ID) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID) WITH (DROP_EXISTING = ON); + +CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID) WITH (DROP_EXISTING = ON); + +CREATE INDEX TYPE_FIELDS_N49 ON TYPE_FIELDS (TYPE_NAME) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUEDATABASE ON DBS ("NAME", "CTLG_NAME") WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUEFUNCTION ON FUNCS (FUNC_NAME,DB_ID) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (TBL_ID,PART_NAME) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX "UNIQUEPKG" ON "PACKAGES" ("NAME", "DB_ID") WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX "UNIQUESTOREDPROC" ON "STORED_PROCS" ("NAME", "DB_ID") WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUETYPE ON TYPES (TYPE_NAME) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUE_CTLG ON CTLGS ("NAME") WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUE_WM_MAPPING ON WM_MAPPING (RP_ID, ENTITY_TYPE, ENTITY_NAME) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUE_WM_POOL ON WM_POOL (RP_ID, PATH) WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN ("NS", "NAME") WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX UNIQUE_WM_TRIGGER ON WM_TRIGGER (RP_ID, "NAME") WITH (DROP_EXISTING = ON); + +CREATE UNIQUE INDEX USERROLEMAPINDEX ON ROLE_MAP (PRINCIPAL_NAME,ROLE_ID,GRANTOR,GRANTOR_TYPE) WITH (DROP_EXISTING = ON); + diff --git a/standalone-metastore/metastore-server/src/main/sql/mysql/rebuild-indexes.mysql.sql b/standalone-metastore/metastore-server/src/main/sql/mysql/rebuild-indexes.mysql.sql new file mode 100644 index 000000000000..df270d5bbc30 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/sql/mysql/rebuild-indexes.mysql.sql @@ -0,0 +1,170 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to you under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + + +-- Rebuilds all HMS indexes on MySQL using ALTER TABLE for atomicity. +-- Update this file whenever a new upgrade script adds an index. + +ALTER TABLE `BUCKETING_COLS` DROP INDEX `BUCKETING_COLS_N49`, ADD INDEX `BUCKETING_COLS_N49` (`SD_ID`); + +ALTER TABLE `COLUMNS_V2` DROP INDEX `COLUMNS_V2_N49`, ADD INDEX `COLUMNS_V2_N49` (`CD_ID`); + +ALTER TABLE `COMPLETED_COMPACTIONS` DROP INDEX `COMPLETED_COMPACTIONS_RES`, ADD INDEX `COMPLETED_COMPACTIONS_RES` (CC_DATABASE,CC_TABLE,CC_PARTITION); + +ALTER TABLE `COMPLETED_TXN_COMPONENTS` DROP INDEX `COMPLETED_TXN_COMPONENTS_IDX`, ADD INDEX `COMPLETED_TXN_COMPONENTS_IDX` (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); + +ALTER TABLE `KEY_CONSTRAINTS` DROP INDEX `CONSTRAINTS_CONSTRAINT_TYPE_INDEX`, ADD INDEX `CONSTRAINTS_CONSTRAINT_TYPE_INDEX` (`CONSTRAINT_TYPE`); + +ALTER TABLE `KEY_CONSTRAINTS` DROP INDEX `CONSTRAINTS_PARENT_TABLE_ID_INDEX`, ADD INDEX `CONSTRAINTS_PARENT_TABLE_ID_INDEX` (`PARENT_TBL_ID`); + +ALTER TABLE `DATABASE_PARAMS` DROP INDEX `DATABASE_PARAMS_N49`, ADD INDEX `DATABASE_PARAMS_N49` (`DB_ID`); + +ALTER TABLE `DB_PRIVS` DROP INDEX `DBPRIVILEGEINDEX`, ADD UNIQUE INDEX `DBPRIVILEGEINDEX` (`AUTHORIZER`,`DB_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`DB_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); + +ALTER TABLE `DB_PRIVS` DROP INDEX `DB_PRIVS_N49`, ADD INDEX `DB_PRIVS_N49` (`DB_ID`); + +ALTER TABLE `DC_PRIVS` DROP INDEX `DCPRIVILEGEINDEX`, ADD UNIQUE INDEX `DCPRIVILEGEINDEX` (`AUTHORIZER`,`NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`DC_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); + +ALTER TABLE `DC_PRIVS` DROP INDEX `DC_PRIVS_N49`, ADD INDEX `DC_PRIVS_N49` (`NAME`); + +ALTER TABLE `REPLICATION_METRICS` DROP INDEX `DUMP_IDX`, ADD INDEX `DUMP_IDX` (RM_DUMP_EXECUTION_ID); + +ALTER TABLE `FUNCS` DROP INDEX `FUNCS_N49`, ADD INDEX `FUNCS_N49` (`DB_ID`); + +ALTER TABLE `GLOBAL_PRIVS` DROP INDEX `GLOBALPRIVILEGEINDEX`, ADD UNIQUE INDEX `GLOBALPRIVILEGEINDEX` (`AUTHORIZER`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`USER_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); + +ALTER TABLE `HIVE_LOCKS` DROP INDEX `HIVE_LOCK_TXNID_INDEX`, ADD INDEX `HIVE_LOCK_TXNID_INDEX` (HL_TXNID); + +ALTER TABLE `HIVE_LOCKS` DROP INDEX `HL_TXNID_IDX`, ADD INDEX `HL_TXNID_IDX` (HL_TXNID); + +ALTER TABLE `RUNTIME_STATS` DROP INDEX `IDX_RUNTIME_STATS_CREATE_TIME`, ADD INDEX `IDX_RUNTIME_STATS_CREATE_TIME` (CREATE_TIME); + +ALTER TABLE `SCHEDULED_EXECUTIONS` DROP INDEX `IDX_SCHEDULED_EXECUTIONS_LAST_UPDATE_TIME`, ADD INDEX `IDX_SCHEDULED_EXECUTIONS_LAST_UPDATE_TIME` (LAST_UPDATE_TIME); + +ALTER TABLE `SCHEDULED_EXECUTIONS` DROP INDEX `IDX_SCHEDULED_EXECUTIONS_SCHEDULED_QUERY_ID`, ADD INDEX `IDX_SCHEDULED_EXECUTIONS_SCHEDULED_QUERY_ID` (SCHEDULED_QUERY_ID); + +ALTER TABLE `MIN_HISTORY_LEVEL` DROP INDEX `MIN_HISTORY_LEVEL_IDX`, ADD INDEX `MIN_HISTORY_LEVEL_IDX` (MHL_MIN_OPEN_TXNID); + +ALTER TABLE `MIN_HISTORY_WRITE_ID` DROP INDEX `MIN_HISTORY_WRITE_ID_IDX`, ADD INDEX `MIN_HISTORY_WRITE_ID_IDX` (MH_DATABASE, MH_TABLE, MH_WRITEID); + +ALTER TABLE `MV_CREATION_METADATA` DROP INDEX `MV_UNIQUE_TABLE`, ADD INDEX `MV_UNIQUE_TABLE` (TBL_NAME, DB_NAME); + +ALTER TABLE `NEXT_WRITE_ID` DROP INDEX `NEXT_WRITE_ID_IDX`, ADD UNIQUE INDEX `NEXT_WRITE_ID_IDX` (NWI_DATABASE, NWI_TABLE); + +ALTER TABLE `NOTIFICATION_LOG` DROP INDEX `NOTIFICATION_LOG_EVENT_ID`, ADD UNIQUE INDEX `NOTIFICATION_LOG_EVENT_ID` (`EVENT_ID`); + +ALTER TABLE `PART_COL_PRIVS` DROP INDEX `PARTITIONCOLUMNPRIVILEGEINDEX`, ADD INDEX `PARTITIONCOLUMNPRIVILEGEINDEX` (`AUTHORIZER`,`PART_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); + +ALTER TABLE `PARTITION_EVENTS` DROP INDEX `PARTITIONEVENTINDEX`, ADD INDEX `PARTITIONEVENTINDEX` (`PARTITION_NAME`); + +ALTER TABLE `PARTITIONS` DROP INDEX `PARTITIONS_N50`, ADD INDEX `PARTITIONS_N50` (`SD_ID`); + +ALTER TABLE `PARTITION_KEYS` DROP INDEX `PARTITION_KEYS_N49`, ADD INDEX `PARTITION_KEYS_N49` (`TBL_ID`); + +ALTER TABLE `PARTITION_KEY_VALS` DROP INDEX `PARTITION_KEY_VALS_N49`, ADD INDEX `PARTITION_KEY_VALS_N49` (`PART_ID`); + +ALTER TABLE `PARTITION_PARAMS` DROP INDEX `PARTITION_PARAMS_N49`, ADD INDEX `PARTITION_PARAMS_N49` (`PART_ID`); + +ALTER TABLE `PART_PRIVS` DROP INDEX `PARTPRIVILEGEINDEX`, ADD INDEX `PARTPRIVILEGEINDEX` (`AUTHORIZER`,`PART_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`PART_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); + +ALTER TABLE `PART_COL_PRIVS` DROP INDEX `PART_COL_PRIVS_N49`, ADD INDEX `PART_COL_PRIVS_N49` (`PART_ID`); + +ALTER TABLE `PART_PRIVS` DROP INDEX `PART_PRIVS_N49`, ADD INDEX `PART_PRIVS_N49` (`PART_ID`); + +ALTER TABLE `PART_COL_STATS` DROP INDEX `PCS_STATS_IDX`, ADD INDEX `PCS_STATS_IDX` (PART_ID,COLUMN_NAME); + +ALTER TABLE `REPLICATION_METRICS` DROP INDEX `POLICY_IDX`, ADD INDEX `POLICY_IDX` (RM_POLICY); + +ALTER TABLE `ROLES` DROP INDEX `ROLEENTITYINDEX`, ADD UNIQUE INDEX `ROLEENTITYINDEX` (`ROLE_NAME`); + +ALTER TABLE `ROLE_MAP` DROP INDEX `ROLE_MAP_N49`, ADD INDEX `ROLE_MAP_N49` (`ROLE_ID`); + +ALTER TABLE `SDS` DROP INDEX `SDS_N49`, ADD INDEX `SDS_N49` (`SERDE_ID`); + +ALTER TABLE `SDS` DROP INDEX `SDS_N50`, ADD INDEX `SDS_N50` (`CD_ID`); + +ALTER TABLE `SD_PARAMS` DROP INDEX `SD_PARAMS_N49`, ADD INDEX `SD_PARAMS_N49` (`SD_ID`); + +ALTER TABLE `SERDE_PARAMS` DROP INDEX `SERDE_PARAMS_N49`, ADD INDEX `SERDE_PARAMS_N49` (`SERDE_ID`); + +ALTER TABLE `SKEWED_COL_NAMES` DROP INDEX `SKEWED_COL_NAMES_N49`, ADD INDEX `SKEWED_COL_NAMES_N49` (`SD_ID`); + +ALTER TABLE `SKEWED_COL_VALUE_LOC_MAP` DROP INDEX `SKEWED_COL_VALUE_LOC_MAP_N49`, ADD INDEX `SKEWED_COL_VALUE_LOC_MAP_N49` (`STRING_LIST_ID_KID`); + +ALTER TABLE `SKEWED_COL_VALUE_LOC_MAP` DROP INDEX `SKEWED_COL_VALUE_LOC_MAP_N50`, ADD INDEX `SKEWED_COL_VALUE_LOC_MAP_N50` (`SD_ID`); + +ALTER TABLE `SKEWED_STRING_LIST_VALUES` DROP INDEX `SKEWED_STRING_LIST_VALUES_N49`, ADD INDEX `SKEWED_STRING_LIST_VALUES_N49` (`STRING_LIST_ID`); + +ALTER TABLE `SKEWED_VALUES` DROP INDEX `SKEWED_VALUES_N49`, ADD INDEX `SKEWED_VALUES_N49` (`STRING_LIST_ID_EID`); + +ALTER TABLE `SKEWED_VALUES` DROP INDEX `SKEWED_VALUES_N50`, ADD INDEX `SKEWED_VALUES_N50` (`SD_ID_OID`); + +ALTER TABLE `SORT_COLS` DROP INDEX `SORT_COLS_N49`, ADD INDEX `SORT_COLS_N49` (`SD_ID`); + +ALTER TABLE `TBL_COL_PRIVS` DROP INDEX `TABLECOLUMNPRIVILEGEINDEX`, ADD INDEX `TABLECOLUMNPRIVILEGEINDEX` (`AUTHORIZER`,`TBL_ID`,`COLUMN_NAME`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`TBL_COL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); + +ALTER TABLE `TBL_PRIVS` DROP INDEX `TABLEPRIVILEGEINDEX`, ADD INDEX `TABLEPRIVILEGEINDEX` (`AUTHORIZER`,`TBL_ID`,`PRINCIPAL_NAME`,`PRINCIPAL_TYPE`,`TBL_PRIV`,`GRANTOR`,`GRANTOR_TYPE`); + +ALTER TABLE `TABLE_PARAMS` DROP INDEX `TABLE_PARAMS_N49`, ADD INDEX `TABLE_PARAMS_N49` (`TBL_ID`); + +ALTER TABLE `TAB_COL_STATS` DROP INDEX `TAB_COL_STATS_IDX`, ADD INDEX `TAB_COL_STATS_IDX` (TBL_ID, COLUMN_NAME); + +ALTER TABLE `TBLS` DROP INDEX `TBLS_N50`, ADD INDEX `TBLS_N50` (`SD_ID`); + +ALTER TABLE `TBL_COL_PRIVS` DROP INDEX `TBL_COL_PRIVS_N49`, ADD INDEX `TBL_COL_PRIVS_N49` (`TBL_ID`); + +ALTER TABLE `TBL_PRIVS` DROP INDEX `TBL_PRIVS_N49`, ADD INDEX `TBL_PRIVS_N49` (`TBL_ID`); + +ALTER TABLE `TXN_TO_WRITE_ID` DROP INDEX `TBL_TO_TXN_ID_IDX`, ADD UNIQUE INDEX `TBL_TO_TXN_ID_IDX` (T2W_DATABASE, T2W_TABLE, T2W_TXNID); + +ALTER TABLE `TXN_TO_WRITE_ID` DROP INDEX `TBL_TO_WRITE_ID_IDX`, ADD UNIQUE INDEX `TBL_TO_WRITE_ID_IDX` (T2W_DATABASE, T2W_TABLE, T2W_WRITEID); + +ALTER TABLE `TXN_COMPONENTS` DROP INDEX `TC_TXNID_INDEX`, ADD INDEX `TC_TXNID_INDEX` (TC_TXNID); + +ALTER TABLE `TYPE_FIELDS` DROP INDEX `TYPE_FIELDS_N49`, ADD INDEX `TYPE_FIELDS_N49` (`TYPE_NAME`); + +ALTER TABLE `FUNCS` DROP INDEX `UNIQUEFUNCTION`, ADD UNIQUE INDEX `UNIQUEFUNCTION` (`FUNC_NAME`, `DB_ID`); + +ALTER TABLE `PARTITIONS` DROP INDEX `UNIQUEPARTITION`, ADD UNIQUE INDEX `UNIQUEPARTITION` (`TBL_ID`, `PART_NAME`); + +ALTER TABLE `PACKAGES` DROP INDEX `UNIQUEPKG`, ADD UNIQUE INDEX `UNIQUEPKG` (NAME, DB_ID); + +ALTER TABLE `STORED_PROCS` DROP INDEX `UNIQUESTOREDPROC`, ADD UNIQUE INDEX `UNIQUESTOREDPROC` (NAME, DB_ID); + +ALTER TABLE `TBLS` DROP INDEX `UNIQUETABLE`, ADD UNIQUE INDEX `UNIQUETABLE` (`DB_ID`,`TBL_NAME`); + +ALTER TABLE `CTLGS` DROP INDEX `UNIQUE_CATALOG`, ADD UNIQUE INDEX `UNIQUE_CATALOG` (`NAME`); + +ALTER TABLE `DBS` DROP INDEX `UNIQUE_DATABASE`, ADD UNIQUE INDEX `UNIQUE_DATABASE` (`NAME`, `CTLG_NAME`); + +ALTER TABLE `I_SCHEMA` DROP INDEX `UNIQUE_NAME`, ADD INDEX `UNIQUE_NAME` (`NAME`); + +ALTER TABLE `SCHEDULED_EXECUTIONS` DROP INDEX `UNIQUE_SCHEDULED_EXECUTIONS_ID`, ADD UNIQUE INDEX `UNIQUE_SCHEDULED_EXECUTIONS_ID` (SCHEDULED_EXECUTION_ID); + +ALTER TABLE `TYPES` DROP INDEX `UNIQUE_TYPE`, ADD UNIQUE INDEX `UNIQUE_TYPE` (`TYPE_NAME`); + +ALTER TABLE `SCHEMA_VERSION` DROP INDEX `UNIQUE_VERSION`, ADD INDEX `UNIQUE_VERSION` (`SCHEMA_ID`, `VERSION`); + +ALTER TABLE `WM_MAPPING` DROP INDEX `UNIQUE_WM_MAPPING`, ADD UNIQUE INDEX `UNIQUE_WM_MAPPING` (`RP_ID`, `ENTITY_TYPE`, `ENTITY_NAME`); + +ALTER TABLE `WM_POOL` DROP INDEX `UNIQUE_WM_POOL`, ADD UNIQUE INDEX `UNIQUE_WM_POOL` (`RP_ID`, `PATH`); + +ALTER TABLE `WM_RESOURCEPLAN` DROP INDEX `UNIQUE_WM_RESOURCEPLAN`, ADD UNIQUE INDEX `UNIQUE_WM_RESOURCEPLAN` (`NAME`, `NS`); + +ALTER TABLE `WM_TRIGGER` DROP INDEX `UNIQUE_WM_TRIGGER`, ADD UNIQUE INDEX `UNIQUE_WM_TRIGGER` (`RP_ID`, `NAME`); + +ALTER TABLE `ROLE_MAP` DROP INDEX `USERROLEMAPINDEX`, ADD UNIQUE INDEX `USERROLEMAPINDEX` (`PRINCIPAL_NAME`,`ROLE_ID`,`GRANTOR`,`GRANTOR_TYPE`); + diff --git a/standalone-metastore/metastore-server/src/main/sql/oracle/rebuild-indexes.oracle.sql b/standalone-metastore/metastore-server/src/main/sql/oracle/rebuild-indexes.oracle.sql new file mode 100644 index 000000000000..9e7c5eb3f491 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/sql/oracle/rebuild-indexes.oracle.sql @@ -0,0 +1,222 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to you under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + + +-- Oracle has no DROP INDEX IF EXISTS; this script uses plain DROP INDEX. +-- If an index is already missing, comment out its DROP line before running. +-- Update this file whenever a new upgrade script adds an index. + +DROP INDEX BUCKETING_COLS_N49; +CREATE INDEX BUCKETING_COLS_N49 ON BUCKETING_COLS (SD_ID); + +DROP INDEX COLUMNS_V2_N49; +CREATE INDEX COLUMNS_V2_N49 ON COLUMNS_V2 (CD_ID); + +DROP INDEX COMPLETED_COMPACTIONS_RES; +CREATE INDEX COMPLETED_COMPACTIONS_RES ON COMPLETED_COMPACTIONS (CC_DATABASE,CC_TABLE,CC_PARTITION); + +DROP INDEX COMPLETED_TXN_COMPONENTS_INDEX; +CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON COMPLETED_TXN_COMPONENTS (CTC_DATABASE, CTC_TABLE, CTC_PARTITION); + +DROP INDEX CONSTRAINTS_CT_INDEX; +CREATE INDEX CONSTRAINTS_CT_INDEX ON KEY_CONSTRAINTS(CONSTRAINT_TYPE); + +DROP INDEX CONSTRAINTS_PT_INDEX; +CREATE INDEX CONSTRAINTS_PT_INDEX ON KEY_CONSTRAINTS(PARENT_TBL_ID); + +DROP INDEX CTLG_NAME_DBS; +CREATE INDEX CTLG_NAME_DBS ON DBS(CTLG_NAME); + +DROP INDEX DATABASE_PARAMS_N49; +CREATE INDEX DATABASE_PARAMS_N49 ON DATABASE_PARAMS (DB_ID); + +DROP INDEX DBPRIVILEGEINDEX; +CREATE UNIQUE INDEX DBPRIVILEGEINDEX ON DB_PRIVS (AUTHORIZER,DB_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,DB_PRIV,GRANTOR,GRANTOR_TYPE); + +DROP INDEX DB_PRIVS_N49; +CREATE INDEX DB_PRIVS_N49 ON DB_PRIVS (DB_ID); + +DROP INDEX DCPRIVILEGEINDEX; +CREATE UNIQUE INDEX DCPRIVILEGEINDEX ON DC_PRIVS (AUTHORIZER,NAME,PRINCIPAL_NAME,PRINCIPAL_TYPE,DC_PRIV,GRANTOR, GRANTOR_TYPE); + +DROP INDEX DC_PRIVS_N49; +CREATE INDEX DC_PRIVS_N49 ON DC_PRIVS (NAME); + +DROP INDEX DUMP_IDX; +CREATE INDEX DUMP_IDX ON "REPLICATION_METRICS" ("RM_DUMP_EXECUTION_ID"); + +DROP INDEX FUNCS_N49; +CREATE INDEX FUNCS_N49 ON FUNCS (DB_ID); + +DROP INDEX FUNC_RU_N49; +CREATE INDEX FUNC_RU_N49 ON FUNC_RU (FUNC_ID); + +DROP INDEX GLOBALPRIVILEGEINDEX; +CREATE UNIQUE INDEX GLOBALPRIVILEGEINDEX ON GLOBAL_PRIVS (AUTHORIZER,PRINCIPAL_NAME,PRINCIPAL_TYPE,USER_PRIV,GRANTOR,GRANTOR_TYPE); + +DROP INDEX HL_TXNID_INDEX; +CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); + +DROP INDEX IDX_RUNTIME_STATS_CREATE_TIME; +CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); + +DROP INDEX IDX_SCHEDULED_EX_LAST_UPDATE; +CREATE INDEX IDX_SCHEDULED_EX_LAST_UPDATE ON "SCHEDULED_EXECUTIONS" ("LAST_UPDATE_TIME"); + +DROP INDEX IDX_SCHEDULED_EX_SQ_ID; +CREATE INDEX IDX_SCHEDULED_EX_SQ_ID ON "SCHEDULED_EXECUTIONS" ("SCHEDULED_QUERY_ID"); + +DROP INDEX MIN_HISTORY_LEVEL_IDX; +CREATE INDEX MIN_HISTORY_LEVEL_IDX ON MIN_HISTORY_LEVEL (MHL_MIN_OPEN_TXNID); + +DROP INDEX MIN_HISTORY_WRITE_ID_IDX; +CREATE INDEX MIN_HISTORY_WRITE_ID_IDX ON MIN_HISTORY_WRITE_ID (MH_DATABASE, MH_TABLE, MH_WRITEID); + +DROP INDEX NEXT_WRITE_ID_IDX; +CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE); + +DROP INDEX NOTIFICATION_LOG_EVENT_ID; +CREATE UNIQUE INDEX NOTIFICATION_LOG_EVENT_ID ON NOTIFICATION_LOG(EVENT_ID); + +DROP INDEX PARTITIONCOLUMNPRIVILEGEINDEX; +CREATE INDEX PARTITIONCOLUMNPRIVILEGEINDEX ON PART_COL_PRIVS (AUTHORIZER,PART_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_COL_PRIV,GRANTOR,GRANTOR_TYPE); + +DROP INDEX PARTITIONEVENTINDEX; +CREATE INDEX PARTITIONEVENTINDEX ON PARTITION_EVENTS (PARTITION_NAME); + +DROP INDEX PARTITIONS_N49; +CREATE INDEX PARTITIONS_N49 ON PARTITIONS (SD_ID); + +DROP INDEX PARTITION_KEYS_N49; +CREATE INDEX PARTITION_KEYS_N49 ON PARTITION_KEYS (TBL_ID); + +DROP INDEX PARTITION_KEY_VALS_N49; +CREATE INDEX PARTITION_KEY_VALS_N49 ON PARTITION_KEY_VALS (PART_ID); + +DROP INDEX PARTITION_PARAMS_N49; +CREATE INDEX PARTITION_PARAMS_N49 ON PARTITION_PARAMS (PART_ID); + +DROP INDEX PARTPRIVILEGEINDEX; +CREATE INDEX PARTPRIVILEGEINDEX ON PART_PRIVS (AUTHORIZER,PART_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,PART_PRIV,GRANTOR,GRANTOR_TYPE); + +DROP INDEX PART_COL_PRIVS_N49; +CREATE INDEX PART_COL_PRIVS_N49 ON PART_COL_PRIVS (PART_ID); + +DROP INDEX PART_PRIVS_N49; +CREATE INDEX PART_PRIVS_N49 ON PART_PRIVS (PART_ID); + +DROP INDEX PCS_STATS_IDX; +CREATE INDEX PCS_STATS_IDX ON PART_COL_STATS (PART_ID,COLUMN_NAME); + +DROP INDEX POLICY_IDX; +CREATE INDEX POLICY_IDX ON "REPLICATION_METRICS" ("RM_POLICY"); + +DROP INDEX ROLEENTITYINDEX; +CREATE UNIQUE INDEX ROLEENTITYINDEX ON ROLES (ROLE_NAME); + +DROP INDEX ROLE_MAP_N49; +CREATE INDEX ROLE_MAP_N49 ON ROLE_MAP (ROLE_ID); + +DROP INDEX SDS_N49; +CREATE INDEX SDS_N49 ON SDS (SERDE_ID); + +DROP INDEX SDS_N50; +CREATE INDEX SDS_N50 ON SDS (CD_ID); + +DROP INDEX SD_PARAMS_N49; +CREATE INDEX SD_PARAMS_N49 ON SD_PARAMS (SD_ID); + +DROP INDEX SERDE_PARAMS_N49; +CREATE INDEX SERDE_PARAMS_N49 ON SERDE_PARAMS (SERDE_ID); + +DROP INDEX SORT_COLS_N49; +CREATE INDEX SORT_COLS_N49 ON SORT_COLS (SD_ID); + +DROP INDEX TABLECOLUMNPRIVILEGEINDEX; +CREATE INDEX TABLECOLUMNPRIVILEGEINDEX ON TBL_COL_PRIVS (AUTHORIZER,TBL_ID,"COLUMN_NAME",PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_COL_PRIV,GRANTOR,GRANTOR_TYPE); + +DROP INDEX TABLEPRIVILEGEINDEX; +CREATE INDEX TABLEPRIVILEGEINDEX ON TBL_PRIVS (AUTHORIZER,TBL_ID,PRINCIPAL_NAME,PRINCIPAL_TYPE,TBL_PRIV,GRANTOR,GRANTOR_TYPE); + +DROP INDEX TABLE_PARAMS_N49; +CREATE INDEX TABLE_PARAMS_N49 ON TABLE_PARAMS (TBL_ID); + +DROP INDEX TAB_COL_STATS_IDX; +CREATE INDEX TAB_COL_STATS_IDX ON TAB_COL_STATS (TBL_ID, COLUMN_NAME); + +DROP INDEX TBLS_N50; +CREATE INDEX TBLS_N50 ON TBLS (SD_ID); + +DROP INDEX TBL_COL_PRIVS_N49; +CREATE INDEX TBL_COL_PRIVS_N49 ON TBL_COL_PRIVS (TBL_ID); + +DROP INDEX TBL_PRIVS_N49; +CREATE INDEX TBL_PRIVS_N49 ON TBL_PRIVS (TBL_ID); + +DROP INDEX TBL_TO_TXN_ID_IDX; +CREATE UNIQUE INDEX TBL_TO_TXN_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_TXNID); + +DROP INDEX TBL_TO_WRITE_ID_IDX; +CREATE UNIQUE INDEX TBL_TO_WRITE_ID_IDX ON TXN_TO_WRITE_ID (T2W_DATABASE, T2W_TABLE, T2W_WRITEID); + +DROP INDEX TC_TXNID_INDEX; +CREATE INDEX TC_TXNID_INDEX ON TXN_COMPONENTS (TC_TXNID); + +DROP INDEX TXN_WRITE_NOTIFICATION_LOG_IDX; +CREATE INDEX TXN_WRITE_NOTIFICATION_LOG_IDX ON TXN_WRITE_NOTIFICATION_LOG (WNL_TXNID, WNL_DATABASE, WNL_TABLE, WNL_PARTITION); + +DROP INDEX TYPE_FIELDS_N49; +CREATE INDEX TYPE_FIELDS_N49 ON TYPE_FIELDS (TYPE_NAME); + +DROP INDEX UNIQUEFUNCTION; +CREATE UNIQUE INDEX UNIQUEFUNCTION ON FUNCS (FUNC_NAME, DB_ID); + +DROP INDEX UNIQUEPARTITION; +CREATE UNIQUE INDEX UNIQUEPARTITION ON PARTITIONS (TBL_ID, PART_NAME); + +DROP INDEX UNIQUEPKG; +CREATE UNIQUE INDEX UNIQUEPKG ON PACKAGES ("NAME", "DB_ID"); + +DROP INDEX UNIQUESTOREDPROC; +CREATE UNIQUE INDEX UNIQUESTOREDPROC ON STORED_PROCS ("NAME", "DB_ID"); + +DROP INDEX UNIQUETABLE; +CREATE UNIQUE INDEX UNIQUETABLE ON TBLS (DB_ID,TBL_NAME); + +DROP INDEX UNIQUE_DATABASE; +CREATE UNIQUE INDEX UNIQUE_DATABASE ON DBS ("NAME", CTLG_NAME); + +DROP INDEX UNIQUE_TABLE; +CREATE UNIQUE INDEX UNIQUE_TABLE ON MV_CREATION_METADATA ("DB_NAME", "TBL_NAME"); + +DROP INDEX UNIQUE_TYPE; +CREATE UNIQUE INDEX UNIQUE_TYPE ON TYPES (TYPE_NAME); + +DROP INDEX UNIQUE_WM_MAPPING; +CREATE UNIQUE INDEX UNIQUE_WM_MAPPING ON WM_MAPPING (RP_ID, ENTITY_TYPE, ENTITY_NAME); + +DROP INDEX UNIQUE_WM_POOL; +CREATE UNIQUE INDEX UNIQUE_WM_POOL ON WM_POOL (RP_ID, PATH); + +DROP INDEX UNIQUE_WM_RESOURCEPLAN; +CREATE UNIQUE INDEX UNIQUE_WM_RESOURCEPLAN ON WM_RESOURCEPLAN (NS, "NAME"); + +DROP INDEX UNIQUE_WM_TRIGGER; +CREATE UNIQUE INDEX UNIQUE_WM_TRIGGER ON WM_TRIGGER (RP_ID, "NAME"); + +DROP INDEX USERROLEMAPINDEX; +CREATE UNIQUE INDEX USERROLEMAPINDEX ON ROLE_MAP (PRINCIPAL_NAME,ROLE_ID,GRANTOR,GRANTOR_TYPE); + diff --git a/standalone-metastore/metastore-server/src/main/sql/postgres/rebuild-indexes.postgres.sql b/standalone-metastore/metastore-server/src/main/sql/postgres/rebuild-indexes.postgres.sql new file mode 100644 index 000000000000..3552b7c1b8c3 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/sql/postgres/rebuild-indexes.postgres.sql @@ -0,0 +1,176 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and + +-- Rebuilds all HMS indexes on PostgreSQL. +-- Update this file whenever a new upgrade script adds an index. + +DROP INDEX IF EXISTS "NOTIFICATION_LOG_EVENT_ID"; +CREATE UNIQUE INDEX "NOTIFICATION_LOG_EVENT_ID" ON "NOTIFICATION_LOG" USING btree ("EVENT_ID"); + +DROP INDEX IF EXISTS "BUCKETING_COLS_N49"; +CREATE INDEX "BUCKETING_COLS_N49" ON "BUCKETING_COLS" USING btree ("SD_ID"); + +DROP INDEX IF EXISTS "DATABASE_PARAMS_N49"; +CREATE INDEX "DATABASE_PARAMS_N49" ON "DATABASE_PARAMS" USING btree ("DB_ID"); + +DROP INDEX IF EXISTS "DB_PRIVS_N49"; +CREATE INDEX "DB_PRIVS_N49" ON "DB_PRIVS" USING btree ("DB_ID"); + +DROP INDEX IF EXISTS "DC_PRIVS_N49"; +CREATE INDEX "DC_PRIVS_N49" ON "DC_PRIVS" USING btree ("NAME"); + +DROP INDEX IF EXISTS "PARTITIONCOLUMNPRIVILEGEINDEX"; +CREATE INDEX "PARTITIONCOLUMNPRIVILEGEINDEX" ON "PART_COL_PRIVS" USING btree ("AUTHORIZER", "PART_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +DROP INDEX IF EXISTS "PARTITIONEVENTINDEX"; +CREATE INDEX "PARTITIONEVENTINDEX" ON "PARTITION_EVENTS" USING btree ("PARTITION_NAME"); + +DROP INDEX IF EXISTS "PARTITIONS_N50"; +CREATE INDEX "PARTITIONS_N50" ON "PARTITIONS" USING btree ("SD_ID"); + +DROP INDEX IF EXISTS "PARTITION_KEYS_N49"; +CREATE INDEX "PARTITION_KEYS_N49" ON "PARTITION_KEYS" USING btree ("TBL_ID"); + +DROP INDEX IF EXISTS "PARTITION_KEY_VALS_N49"; +CREATE INDEX "PARTITION_KEY_VALS_N49" ON "PARTITION_KEY_VALS" USING btree ("PART_ID"); + +DROP INDEX IF EXISTS "PARTITION_PARAMS_N49"; +CREATE INDEX "PARTITION_PARAMS_N49" ON "PARTITION_PARAMS" USING btree ("PART_ID"); + +DROP INDEX IF EXISTS "PARTPRIVILEGEINDEX"; +CREATE INDEX "PARTPRIVILEGEINDEX" ON "PART_PRIVS" USING btree ("AUTHORIZER", "PART_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "PART_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +DROP INDEX IF EXISTS "PART_COL_PRIVS_N49"; +CREATE INDEX "PART_COL_PRIVS_N49" ON "PART_COL_PRIVS" USING btree ("PART_ID"); + +DROP INDEX IF EXISTS "PART_PRIVS_N49"; +CREATE INDEX "PART_PRIVS_N49" ON "PART_PRIVS" USING btree ("PART_ID"); + +DROP INDEX IF EXISTS "PCS_STATS_IDX"; +CREATE INDEX "PCS_STATS_IDX" ON "PART_COL_STATS" USING btree ("PART_ID","COLUMN_NAME"); + +DROP INDEX IF EXISTS "ROLE_MAP_N49"; +CREATE INDEX "ROLE_MAP_N49" ON "ROLE_MAP" USING btree ("ROLE_ID"); + +DROP INDEX IF EXISTS "SDS_N49"; +CREATE INDEX "SDS_N49" ON "SDS" USING btree ("SERDE_ID"); + +DROP INDEX IF EXISTS "SDS_N50"; +CREATE INDEX "SDS_N50" ON "SDS" USING btree ("CD_ID"); + +DROP INDEX IF EXISTS "SD_PARAMS_N49"; +CREATE INDEX "SD_PARAMS_N49" ON "SD_PARAMS" USING btree ("SD_ID"); + +DROP INDEX IF EXISTS "SERDE_PARAMS_N49"; +CREATE INDEX "SERDE_PARAMS_N49" ON "SERDE_PARAMS" USING btree ("SERDE_ID"); + +DROP INDEX IF EXISTS "SORT_COLS_N49"; +CREATE INDEX "SORT_COLS_N49" ON "SORT_COLS" USING btree ("SD_ID"); + +DROP INDEX IF EXISTS "TABLECOLUMNPRIVILEGEINDEX"; +CREATE INDEX "TABLECOLUMNPRIVILEGEINDEX" ON "TBL_COL_PRIVS" USING btree ("AUTHORIZER", "TBL_ID", "COLUMN_NAME", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_COL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +DROP INDEX IF EXISTS "TABLEPRIVILEGEINDEX"; +CREATE INDEX "TABLEPRIVILEGEINDEX" ON "TBL_PRIVS" USING btree ("AUTHORIZER", "TBL_ID", "PRINCIPAL_NAME", "PRINCIPAL_TYPE", "TBL_PRIV", "GRANTOR", "GRANTOR_TYPE"); + +DROP INDEX IF EXISTS "TABLE_PARAMS_N49"; +CREATE INDEX "TABLE_PARAMS_N49" ON "TABLE_PARAMS" USING btree ("TBL_ID"); + +DROP INDEX IF EXISTS "TBLS_N50"; +CREATE INDEX "TBLS_N50" ON "TBLS" USING btree ("SD_ID"); + +DROP INDEX IF EXISTS "TBL_COL_PRIVS_N49"; +CREATE INDEX "TBL_COL_PRIVS_N49" ON "TBL_COL_PRIVS" USING btree ("TBL_ID"); + +DROP INDEX IF EXISTS "TAB_COL_STATS_IDX"; +CREATE INDEX "TAB_COL_STATS_IDX" ON "TAB_COL_STATS" USING btree ("TBL_ID","COLUMN_NAME"); + +DROP INDEX IF EXISTS "TBL_PRIVS_N49"; +CREATE INDEX "TBL_PRIVS_N49" ON "TBL_PRIVS" USING btree ("TBL_ID"); + +DROP INDEX IF EXISTS "TYPE_FIELDS_N49"; +CREATE INDEX "TYPE_FIELDS_N49" ON "TYPE_FIELDS" USING btree ("TYPE_NAME"); + +DROP INDEX IF EXISTS "UNIQUEFUNCTION"; +CREATE UNIQUE INDEX "UNIQUEFUNCTION" ON "FUNCS" ("FUNC_NAME", "DB_ID"); + +DROP INDEX IF EXISTS "FUNCS_N49"; +CREATE INDEX "FUNCS_N49" ON "FUNCS" ("DB_ID"); + +DROP INDEX IF EXISTS "FUNC_RU_N49"; +CREATE INDEX "FUNC_RU_N49" ON "FUNC_RU" ("FUNC_ID"); + +DROP INDEX IF EXISTS "CONSTRAINTS_PARENT_TBLID_INDEX"; +CREATE INDEX "CONSTRAINTS_PARENT_TBLID_INDEX" ON "KEY_CONSTRAINTS" USING BTREE ("PARENT_TBL_ID"); + +DROP INDEX IF EXISTS "CONSTRAINTS_CONSTRAINT_TYPE_INDEX"; +CREATE INDEX "CONSTRAINTS_CONSTRAINT_TYPE_INDEX" ON "KEY_CONSTRAINTS" USING BTREE ("CONSTRAINT_TYPE"); + +DROP INDEX IF EXISTS "MV_UNIQUE_TABLE"; +CREATE INDEX "MV_UNIQUE_TABLE" + ON "MV_CREATION_METADATA" USING btree ("TBL_NAME", "DB_NAME"); + +DROP INDEX IF EXISTS TC_TXNID_INDEX; +CREATE INDEX TC_TXNID_INDEX ON "TXN_COMPONENTS" USING hash ("TC_TXNID"); + +DROP INDEX IF EXISTS COMPLETED_TXN_COMPONENTS_INDEX; +CREATE INDEX COMPLETED_TXN_COMPONENTS_INDEX ON "COMPLETED_TXN_COMPONENTS" USING btree ("CTC_DATABASE", "CTC_TABLE", "CTC_PARTITION"); + +DROP INDEX IF EXISTS HL_TXNID_INDEX; +CREATE INDEX HL_TXNID_INDEX ON "HIVE_LOCKS" USING hash ("HL_TXNID"); + +DROP INDEX IF EXISTS "COMPLETED_COMPACTIONS_RES"; +CREATE INDEX "COMPLETED_COMPACTIONS_RES" ON "COMPLETED_COMPACTIONS" ("CC_DATABASE","CC_TABLE","CC_PARTITION"); + +DROP INDEX IF EXISTS "TBL_TO_TXN_ID_IDX"; +CREATE UNIQUE INDEX "TBL_TO_TXN_ID_IDX" ON "TXN_TO_WRITE_ID" ("T2W_DATABASE", "T2W_TABLE", "T2W_TXNID"); + +DROP INDEX IF EXISTS "TBL_TO_WRITE_ID_IDX"; +CREATE UNIQUE INDEX "TBL_TO_WRITE_ID_IDX" ON "TXN_TO_WRITE_ID" ("T2W_DATABASE", "T2W_TABLE", "T2W_WRITEID"); + +DROP INDEX IF EXISTS "NEXT_WRITE_ID_IDX"; +CREATE UNIQUE INDEX "NEXT_WRITE_ID_IDX" ON "NEXT_WRITE_ID" ("NWI_DATABASE", "NWI_TABLE"); + +DROP INDEX IF EXISTS "MIN_HISTORY_WRITE_ID_IDX"; +CREATE INDEX "MIN_HISTORY_WRITE_ID_IDX" ON "MIN_HISTORY_WRITE_ID" ("MH_DATABASE", "MH_TABLE", "MH_WRITEID"); + +DROP INDEX IF EXISTS "MIN_HISTORY_LEVEL_IDX"; +CREATE INDEX "MIN_HISTORY_LEVEL_IDX" ON "MIN_HISTORY_LEVEL" ("MHL_MIN_OPEN_TXNID"); + +DROP INDEX IF EXISTS "IDX_RUNTIME_STATS_CREATE_TIME"; +CREATE INDEX "IDX_RUNTIME_STATS_CREATE_TIME" ON "RUNTIME_STATS" ("CREATE_TIME"); + +DROP INDEX IF EXISTS IDX_SCHEDULED_EXECUTIONS_LAST_UPDATE_TIME; +CREATE INDEX IDX_SCHEDULED_EXECUTIONS_LAST_UPDATE_TIME ON "SCHEDULED_EXECUTIONS" ("LAST_UPDATE_TIME"); + +DROP INDEX IF EXISTS IDX_SCHEDULED_EXECUTIONS_SCHEDULED_QUERY_ID; +CREATE INDEX IDX_SCHEDULED_EXECUTIONS_SCHEDULED_QUERY_ID ON "SCHEDULED_EXECUTIONS" ("SCHEDULED_QUERY_ID"); + +DROP INDEX IF EXISTS UNIQUE_SCHEDULED_EXECUTIONS_ID; +CREATE UNIQUE INDEX UNIQUE_SCHEDULED_EXECUTIONS_ID ON "SCHEDULED_EXECUTIONS" ("SCHEDULED_EXECUTION_ID"); + +DROP INDEX IF EXISTS "POLICY_IDX"; +CREATE INDEX "POLICY_IDX" ON "REPLICATION_METRICS" ("RM_POLICY"); + +DROP INDEX IF EXISTS "DUMP_IDX"; +CREATE INDEX "DUMP_IDX" ON "REPLICATION_METRICS" ("RM_DUMP_EXECUTION_ID"); + +DROP INDEX IF EXISTS "UNIQUESTOREDPROC"; +CREATE UNIQUE INDEX "UNIQUESTOREDPROC" ON "STORED_PROCS" ("NAME", "DB_ID"); + +DROP INDEX IF EXISTS "UNIQUEPKG"; +CREATE UNIQUE INDEX "UNIQUEPKG" ON "PACKAGES" ("NAME", "DB_ID"); + + diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestRebuildIndexesScriptConsistency.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestRebuildIndexesScriptConsistency.java new file mode 100644 index 000000000000..bf12cf0487e4 --- /dev/null +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestRebuildIndexesScriptConsistency.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.tools.schematool; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.IMetaStoreSchemaInfo; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; +import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category(MetastoreUnitTest.class) +@RunWith(Parameterized.class) +public class TestRebuildIndexesScriptConsistency { + + private static final Pattern POSTGRES_PATTERN = Pattern.compile( + "CREATE\\s+(?:UNIQUE\\s+)?INDEX\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?\"?([A-Za-z0-9_]+)\"?", + Pattern.CASE_INSENSITIVE); + + // MySQL indexes show up as CREATE INDEX, inline KEY clauses inside CREATE TABLE, or ADD INDEX + // inside ALTER TABLE (used in the rebuild script for atomicity with FK constraints). + // The negative lookahead prevents matching "PRIMARY KEY AUTO_INCREMENT". + private static final Pattern MYSQL_PATTERN = Pattern.compile( + "(?:CREATE\\s+(?:UNIQUE\\s+)?INDEX\\s+|ADD\\s+(?:UNIQUE\\s+)?INDEX\\s+|(?:UNIQUE\\s+)?KEY\\s+(?!AUTO_INCREMENT\\b))`?([A-Za-z0-9_]+)`?", + Pattern.CASE_INSENSITIVE); + + private static final Pattern ORACLE_PATTERN = Pattern.compile( + "CREATE\\s+(?:UNIQUE\\s+)?INDEX\\s+\"?([A-Za-z0-9_]+)\"?", + Pattern.CASE_INSENSITIVE); + + private static final Pattern MSSQL_PATTERN = Pattern.compile( + "CREATE\\s+(?:UNIQUE\\s+)?INDEX\\s+\"?([A-Za-z0-9_]+)\"?", + Pattern.CASE_INSENSITIVE); + + private static final Map PATTERNS = new HashMap<>(); + static { + PATTERNS.put(HiveSchemaHelper.DB_POSTGRES, POSTGRES_PATTERN); + PATTERNS.put(HiveSchemaHelper.DB_MYSQL, MYSQL_PATTERN); + PATTERNS.put(HiveSchemaHelper.DB_ORACLE, ORACLE_PATTERN); + PATTERNS.put(HiveSchemaHelper.DB_MSSQL, MSSQL_PATTERN); + } + + private static final Pattern DROP_PATTERN = Pattern.compile( + "DROP\\s+INDEX\\s+(?:IF\\s+EXISTS\\s+)?[`\"']?([A-Za-z0-9_]+)[`\"']?", + Pattern.CASE_INSENSITIVE); + + @Parameterized.Parameters(name = "{0}") + public static Collection dbTypes() { + return Arrays.asList( + HiveSchemaHelper.DB_POSTGRES, + HiveSchemaHelper.DB_MYSQL, + HiveSchemaHelper.DB_ORACLE, + HiveSchemaHelper.DB_MSSQL); + } + + private final String dbType; + + public TestRebuildIndexesScriptConsistency(String dbType) { + this.dbType = dbType; + } + + @Test + public void rebuildScriptMatchesInitScript() throws Exception { + String metastoreHome = System.getProperty("test.tmp.dir", "target/tmp"); + IMetaStoreSchemaInfo schemaInfo = + MetaStoreSchemaInfoFactory.get(new Configuration(), metastoreHome, dbType); + + String scriptDir = schemaInfo.getMetaStoreScriptDir(); + String version = schemaInfo.getHiveSchemaVersion(); + + File schemaScript = new File(scriptDir, "hive-schema-" + version + "." + dbType + ".sql"); + File rebuildScript = new File(scriptDir, + SchemaToolTaskRebuildIndexes.REBUILD_INDEXES_FILE_PREFIX + "." + dbType + ".sql"); + + assertTrue("Schema script not found: " + schemaScript, schemaScript.exists()); + assertTrue("Rebuild script not found: " + rebuildScript, rebuildScript.exists()); + + Pattern pattern = PATTERNS.get(dbType); + Set initIndexes = extractIndexNames(schemaScript, pattern); + Set rebuildIndexes = extractIndexNames(rebuildScript, pattern); + + Set missing = new HashSet<>(initIndexes); + missing.removeAll(rebuildIndexes); + + Set extra = new HashSet<>(rebuildIndexes); + extra.removeAll(initIndexes); + + assertTrue("Indexes in init script missing from rebuild script: " + missing, missing.isEmpty()); + assertTrue("Indexes in rebuild script not found in init script: " + extra, extra.isEmpty()); + } + + @Test + public void rebuildScriptHasDropAndCreateForEachIndex() throws Exception { + // MSSQL uses CREATE INDEX ... WITH (DROP_EXISTING = ON) which has no explicit DROP lines. + // The rebuildScriptMatchesInitScript test already covers completeness for MSSQL. + if (dbType.equals(HiveSchemaHelper.DB_MSSQL)) { + return; + } + String metastoreHome = System.getProperty("test.tmp.dir", "target/tmp"); + IMetaStoreSchemaInfo schemaInfo = + MetaStoreSchemaInfoFactory.get(new Configuration(), metastoreHome, dbType); + + File rebuildScript = new File(schemaInfo.getMetaStoreScriptDir(), + SchemaToolTaskRebuildIndexes.REBUILD_INDEXES_FILE_PREFIX + "." + dbType + ".sql"); + + assertTrue("Rebuild script not found: " + rebuildScript, rebuildScript.exists()); + + List lines = Files.readAllLines(rebuildScript.toPath()); + Set created = extractIndexNames(lines, PATTERNS.get(dbType)); + Set dropped = extractIndexNames(lines, DROP_PATTERN); + + Set createdButNotDropped = new HashSet<>(created); + createdButNotDropped.removeAll(dropped); + + Set droppedButNotCreated = new HashSet<>(dropped); + droppedButNotCreated.removeAll(created); + + assertTrue("Indexes created but never dropped: " + createdButNotDropped, createdButNotDropped.isEmpty()); + assertTrue("Indexes dropped but never created: " + droppedButNotCreated, droppedButNotCreated.isEmpty()); + assertEquals("Mismatch between number of DROPs and CREATEs", dropped.size(), created.size()); + } + + private Set extractIndexNames(File file, Pattern pattern) throws IOException { + return extractIndexNames(Files.readAllLines(file.toPath()), pattern); + } + + private Set extractIndexNames(List lines, Pattern pattern) { + Set names = new HashSet<>(); + for (String line : lines) { + if (line.trim().startsWith("--")) { + continue; + } + Matcher m = pattern.matcher(line); + if (m.find()) { + names.add(m.group(1).toUpperCase()); + } + } + return names; + } +} + diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java index e302d24a95e9..e0f449effcad 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/tools/schematool/TestSchemaToolForMetastore.java @@ -486,6 +486,16 @@ public void testMetastoreDbPropertiesAfterUpgrade() throws HiveMetaException, IO validateMetastoreDbPropertiesTable(); } + @Test + public void testRebuildIndexes() throws HiveMetaException { + if (dbms.getDbType().startsWith("derby")) { + return; + } + schemaTool.setVerbose(true); + execute(new SchemaToolTaskInit(), "-initSchema"); + execute(new SchemaToolTaskRebuildIndexes(), "-rebuildIndexes"); + } + private File generateTestScript(String [] stmts) throws IOException { File testScriptFile = File.createTempFile("schematest", ".sql"); testScriptFile.deleteOnExit();