diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java index c833b7cdaa32..9b6eb188634d 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/TableConfigUtils.java @@ -109,6 +109,7 @@ import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.spi.utils.PinotMd5Mode; import org.apache.pinot.spi.utils.TimeUtils; +import org.apache.pinot.spi.utils.TimestampIndexUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1648,7 +1649,8 @@ private static void validateIndexingConfigAndFieldConfigList(TableConfig tableCo // Star-tree index config is not managed by FieldIndexConfigs, and we need to validate it separately. List starTreeIndexConfigs = indexingConfig.getStarTreeIndexConfigs(); if (CollectionUtils.isNotEmpty(starTreeIndexConfigs)) { - validateStarTreeIndexConfigs(starTreeIndexConfigs, indexConfigsMap, schema); + validateStarTreeIndexConfigs(starTreeIndexConfigs, indexConfigsMap, schema, + TimestampIndexUtils.extractColumnsWithGranularity(tableConfig)); } // TIMESTAMP index is not managed by FieldIndexConfigs, and we need to validate it separately. @@ -1750,14 +1752,23 @@ private static void validateIndexingConfigAndFieldConfigListCompatibility(Indexi /// - 'dimensionsSplitOrder' contains all dimensions in 'skipStarNodeCreationForDimensions' /// - Either functionColumnPairs or aggregationConfigs must be specified, but not both /// - All referenced columns exist in the schema and are single-valued + /// + /// `timestampIndexColumns` holds the TIMESTAMP-index derived columns (e.g. `$ts$DAY`) declared via + /// [TimestampConfig#getGranularities()]. These are materialized as dictionary-encoded single-value TIMESTAMP + /// columns at segment generation time (see [TimestampIndexUtils#applyTimestampIndex(TableConfig, Schema)]), so + /// they are absent from the schema at config-validation time and are accepted here without a schema lookup. private static void validateStarTreeIndexConfigs(List starTreeIndexConfigs, - Map indexConfigsMap, Schema schema) { + Map indexConfigsMap, Schema schema, Set timestampIndexColumns) { Set dimensionColumns = new HashSet<>(); for (StarTreeIndexConfig starTreeIndexConfig : starTreeIndexConfigs) { // Validate dimension columns are dictionary encoded List dimensionsSplitOrder = starTreeIndexConfig.getDimensionsSplitOrder(); assert CollectionUtils.isNotEmpty(dimensionsSplitOrder); for (String dimension : dimensionsSplitOrder) { + if (timestampIndexColumns.contains(dimension)) { + dimensionColumns.add(dimension); + continue; + } FieldIndexConfigs indexConfigs = indexConfigsMap.get(dimension); Preconditions.checkState(indexConfigs != null, "Failed to find dimension column: %s specified in star-tree index config in schema", dimension); @@ -1844,6 +1855,9 @@ private static void validateStarTreeIndexConfigs(List starT } for (String column : Iterables.concat(dimensionColumns, aggregatedColumns)) { + if (timestampIndexColumns.contains(column)) { + continue; + } FieldSpec fieldSpec = schema.getFieldSpecFor(column); Preconditions.checkState(fieldSpec != null, "Failed to find column: %s specified in star-tree index config in schema", column); @@ -1852,6 +1866,9 @@ private static void validateStarTreeIndexConfigs(List starT } for (String column : dimensionColumns) { + if (timestampIndexColumns.contains(column)) { + continue; + } FieldSpec fieldSpec = schema.getFieldSpecFor(column); Preconditions.checkState(fieldSpec.isSingleValueField(), "Star-tree dimension columns must be single-value, but found multi-value column: %s", column); diff --git a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java index 0776eeda342b..0fd36dc44762 100644 --- a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java +++ b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/utils/TableConfigUtilsTest.java @@ -52,6 +52,8 @@ import org.apache.pinot.spi.config.table.TagOverrideConfig; import org.apache.pinot.spi.config.table.TenantConfig; import org.apache.pinot.spi.config.table.TierConfig; +import org.apache.pinot.spi.config.table.TimestampConfig; +import org.apache.pinot.spi.config.table.TimestampIndexGranularity; import org.apache.pinot.spi.config.table.UpsertConfig; import org.apache.pinot.spi.config.table.assignment.InstanceAssignmentConfig; import org.apache.pinot.spi.config.table.assignment.InstancePartitionsType; @@ -2088,6 +2090,42 @@ public void testValidateIndexingConfig() { } } + @Test + public void testValidateStarTreeIndexWithTimestampIndexDerivedColumns() { + Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME) + .addDateTime("OrderDate", FieldSpec.DataType.TIMESTAMP, "TIMESTAMP", "1:MILLISECONDS") + .addMetric("value", FieldSpec.DataType.LONG) + .build(); + + // Derived TIMESTAMP-index columns ($OrderDate$DAY, ...) are declared via TimestampConfig granularities and are + // materialized only at segment generation time, so they are absent from the schema here. The star-tree config + // referencing them in dimensionsSplitOrder must still validate. + FieldConfig timestampFieldConfig = new FieldConfig.Builder("OrderDate").withTimestampConfig( + new TimestampConfig(List.of(TimestampIndexGranularity.DAY, TimestampIndexGranularity.WEEK, + TimestampIndexGranularity.MONTH))).build(); + StarTreeIndexConfig starTreeIndexConfig = new StarTreeIndexConfig( + List.of("$OrderDate$DAY", "$OrderDate$WEEK", "$OrderDate$MONTH"), null, List.of("COUNT__*"), null, 10000); + TableConfig tableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME) + .setFieldConfigList(List.of(timestampFieldConfig)) + .setStarTreeIndexConfigs(List.of(starTreeIndexConfig)) + .build(); + TableConfigUtils.validate(tableConfig, schema); + + // A derived-looking column whose granularity was NOT declared in TimestampConfig must still be rejected. + StarTreeIndexConfig undeclaredGranularity = new StarTreeIndexConfig( + List.of("$OrderDate$HOUR"), null, List.of("COUNT__*"), null, 10000); + TableConfig invalidTableConfig = new TableConfigBuilder(TableType.OFFLINE).setTableName(TABLE_NAME) + .setFieldConfigList(List.of(timestampFieldConfig)) + .setStarTreeIndexConfigs(List.of(undeclaredGranularity)) + .build(); + try { + TableConfigUtils.validate(invalidTableConfig, schema); + fail("Should fail for star-tree dimension referencing an undeclared timestamp-index granularity"); + } catch (Exception e) { + // expected + } + } + @Test public void testValidateStarTreeIndexDuplicateFunctionColumnPair() { Schema schema = new Schema.SchemaBuilder().setSchemaName(TABLE_NAME)