diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 4f05682a2db..bf686a205fc 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -52,6 +52,7 @@ public enum Key { /** Common Settings for SQL and PPL. */ QUERY_MEMORY_LIMIT("plugins.query.memory_limit"), QUERY_SIZE_LIMIT("plugins.query.size_limit"), + QUERY_PRUNING_ENABLED("plugins.query.pruning.enabled"), MAX_EXPRESSION_DEPTH("plugins.query.max_expression_depth"), DESERIALIZATION_MAX_DEPTH("plugins.query.deserialization.max_depth"), DESERIALIZATION_MAX_REFS("plugins.query.deserialization.max_refs"), diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index dafa8c84172..729a657d4ed 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -204,6 +204,55 @@ Result set:: } } +plugins.query.pruning.enabled (Experimental) +============================================ + +Version +------- +3.9 + +Description +----------- + +Prunes a wildcard index expression down to the concrete indices that can hold data in the query's ``@timestamp`` range, so fewer indices and shards are touched. The primary use currently is to avoid exhausting the open point-in-time (PIT) context limit when a query would otherwise open a reader context over many indices. + +Pruning only applies to a wildcard expression whose query filters on a ``@timestamp`` range, and never to one that names a remote cluster; anything else is left untouched, and any failure while probing the cluster falls back to querying the full expression. Weigh these limitations before enabling it: + +1. An index whose shards are all unavailable is pruned rather than reported, because ``_field_caps`` does not surface per-index failures. Such a query returns fewer rows instead of an error. +2. Pruning fixes the list of index names, so an index created or deleted between pruning and PIT creation, by a rollover or retention policy for instance, is missed or fails the query. The interval between the two is short, so this is unlikely in practice. +3. An expression that matches an alias or a data stream is never pruned, because a filtered alias contributes a filter and routing that are resolved from the expression itself and so would be silently dropped. +4. Pruning probes the cluster with the ``indices:admin/resolve/index`` and ``indices:data/read/field_caps`` actions, so a principal lacking either permission falls back to querying the full expression. + +Pruning is also skipped whenever it would not reduce the read: when no index is excluded, and when the list of matching indices is longer than an internal cap (50 indices after pruning). Both cases query the original wildcard expression, which reads the same indices when nothing was excluded and more when the cap declined. Here is an example:: + + >> curl -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings -d '{ + "transient" : { + "plugins.query.pruning.enabled" : true + } + }' + +Result set:: + + { + "acknowledged" : true, + "persistent" : { }, + "transient" : { + "plugins" : { + "query" : { + "pruning" : { + "enabled" : "true" + } + } + } + } + } + +Settings: + +1. The default value is false. +2. This setting is node scope. +3. This setting can be updated dynamically. + plugins.query.max_expression_depth ================================== diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/ppl/index_pruning.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/ppl/index_pruning.yml new file mode 100644 index 00000000000..86bed19612b --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/ppl/index_pruning.yml @@ -0,0 +1,369 @@ +# Index pruning: https://github.com/opensearch-project/sql/issues/5698 +# +# Asserted on point_in_time_total because no skipped shards or metrics are observable from the query +# response or the explain/profile API for now, and point_in_time_current is already gone by the time +# a non-paginated query responds. +# +# Each index has one shard, so a query on the PIT path takes point_in_time_total from 0 to 1 on +# every index it reads: a 0 proves pruning excluded that index, a 1 only says it was still read. +# Setup recreates the indices per test, so the counts never accumulate. +# +# max_result_window is 2 so `head 5` exceeds it and forces the PIT path that pruning runs on. + +setup: + - skip: + features: + - headers + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + # Without this a Calcite failure is silently answered by the v2 engine, which never prunes. + plugins.calcite.fallback.allowed: false + plugins.query.pruning.enabled: true + - do: + indices.put_index_template: + name: pruning-it + body: + index_patterns: ['pruning-it-*'] + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + max_result_window: 2 + mappings: + properties: + "@timestamp": {type: date} + status: {type: integer} + body: {type: text} + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "pruning-it-2026-01"}}' + - '{"@timestamp": "2026-01-15T12:00:00Z", "status": 1, "body": "january"}' + - '{"index": {"_index": "pruning-it-2026-02"}}' + - '{"@timestamp": "2026-02-15T12:00:00Z", "status": 2, "body": "february"}' + - '{"index": {"_index": "pruning-it-2026-03"}}' + - '{"@timestamp": "2026-03-15T12:00:00Z", "status": 3, "body": "march"}' + +--- +teardown: + # Data stream first: deleting a wildcard index expression does not remove backing indices. + - do: + indices.delete_data_stream: + name: 'pruning-ds-*' + ignore: 404 + - do: + indices.delete: + index: 'pruning-it-*,pruning-mixed-*,pruning-nomap-*' + ignore_unavailable: true + - do: + indices.delete_index_template: + name: pruning-it + ignore: 404 + - do: + indices.delete_index_template: + name: pruning-ds + ignore: 404 + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: null + plugins.calcite.fallback.allowed: null + plugins.query.pruning.enabled: null + +--- +"Prunes to the single index in range": + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where @timestamp >= '2026-02-01 00:00:00' and @timestamp <= '2026-02-28 23:59:59' | head 5" + - do: + indices.stats: {index: 'pruning-it-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-it-2026-01: {total: {search: {point_in_time_total: 0}}} + pruning-it-2026-02: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-03: {total: {search: {point_in_time_total: 0}}} + +--- +"Prunes to both indices in range": + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where @timestamp >= '2026-01-01 00:00:00' and @timestamp <= '2026-02-28 23:59:59' | head 5" + - do: + indices.stats: {index: 'pruning-it-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-it-2026-01: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-02: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-03: {total: {search: {point_in_time_total: 0}}} + +--- +"Prunes only the head of the pattern for an open ended range": + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where @timestamp >= '2026-02-01 00:00:00' | head 5" + - do: + indices.stats: {index: 'pruning-it-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-it-2026-01: {total: {search: {point_in_time_total: 0}}} + pruning-it-2026-02: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-03: {total: {search: {point_in_time_total: 1}}} + +--- +"Declines to prune when the filter has no range": + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where status = 2 | head 5" + - do: + indices.stats: {index: 'pruning-it-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-it-2026-01: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-02: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-03: {total: {search: {point_in_time_total: 1}}} + +--- +"Declines to prune when the range is not on the timestamp": + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where status >= 1 and status <= 100 | head 5" + - do: + indices.stats: {index: 'pruning-it-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-it-2026-01: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-02: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-03: {total: {search: {point_in_time_total: 1}}} + +--- +"Declines to prune when the setting is off": + - do: + query.settings: + body: + transient: + plugins.query.pruning.enabled: false + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where @timestamp >= '2026-02-01 00:00:00' and @timestamp <= '2026-02-28 23:59:59' | head 5" + - do: + indices.stats: {index: 'pruning-it-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-it-2026-01: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-02: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-03: {total: {search: {point_in_time_total: 1}}} + +--- +"Declines to prune an alias, so its filter still applies": + - do: + indices.put_alias: + index: 'pruning-it-*' + name: pruning-alias-all + body: + filter: + term: {status: 2} + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-alias-* | where @timestamp >= '2026-01-01 00:00:00' and @timestamp <= '2026-03-31 23:59:59' | fields body | head 5" + - match: {datarows: [["february"]]} + +--- +"Declines to prune a data stream": + - do: + indices.put_index_template: + name: pruning-ds + body: + index_patterns: ['pruning-ds-*'] + data_stream: {} + template: + settings: + number_of_shards: 1 + number_of_replicas: 0 + max_result_window: 2 + mappings: + properties: + "@timestamp": {type: date} + body: {type: text} + - do: + indices.create_data_stream: {name: pruning-ds-logs} + - do: + bulk: + index: pruning-ds-logs + refresh: true + body: + - '{"create": {}}' + - '{"@timestamp": "2026-01-15T12:00:00Z", "body": "january"}' + - do: + indices.rollover: {alias: pruning-ds-logs} + - do: + bulk: + index: pruning-ds-logs + refresh: true + body: + - '{"create": {}}' + - '{"@timestamp": "2026-03-15T12:00:00Z", "body": "march"}' + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-ds-* | where @timestamp >= '2026-01-01 00:00:00' and @timestamp <= '2026-01-31 23:59:59' | head 5" + - do: + indices.stats: {index: 'pruning-ds-*', metric: search, filter_path: '_all.total.search.point_in_time_total'} + - match: {_all.total.search.point_in_time_total: 2} + +--- +"A backfilled document stored outside its month is still returned": + - do: + bulk: + index: pruning-it-2026-03 + refresh: true + body: + - '{"index": {}}' + - '{"@timestamp": "2026-02-20T12:00:00Z", "status": 99, "body": "backfill"}' + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where @timestamp >= '2026-02-01 00:00:00' and @timestamp <= '2026-02-28 23:59:59' | sort @timestamp | fields body | head 5" + - match: {datarows: [["february"], ["backfill"]]} + +--- +"Pruning does not change the rows returned": + - do: + query.settings: + body: + transient: + plugins.query.pruning.enabled: false + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where @timestamp >= '2026-01-01 00:00:00' and @timestamp <= '2026-03-31 23:59:59' | sort @timestamp | fields body | head 5" + # Pin the baseline so the comparison below cannot pass by matching two empty result sets. + - match: {datarows: [["january"], ["february"], ["march"]]} + - set: {datarows: unpruned} + - do: + query.settings: + body: + transient: + plugins.query.pruning.enabled: true + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-* | where @timestamp >= '2026-01-01 00:00:00' and @timestamp <= '2026-03-31 23:59:59' | sort @timestamp | fields body | head 5" + - match: {datarows: $unpruned} + +--- +"Prunes across indices whose timestamp types conflict": + - do: + indices.create: + index: pruning-mixed-a + body: + settings: {number_of_shards: 1, number_of_replicas: 0, max_result_window: 2} + mappings: {properties: {"@timestamp": {type: date}, body: {type: keyword}}} + - do: + indices.create: + index: pruning-mixed-b + body: + settings: {number_of_shards: 1, number_of_replicas: 0, max_result_window: 2} + mappings: {properties: {"@timestamp": {type: date_nanos}, body: {type: keyword}}} + - do: + indices.create: + index: pruning-mixed-c + body: + settings: {number_of_shards: 1, number_of_replicas: 0, max_result_window: 2} + mappings: {properties: {"@timestamp": {type: date_nanos}, body: {type: keyword}}} + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "pruning-mixed-a"}}' + - '{"@timestamp": "2026-01-15T12:00:00Z", "body": "january"}' + - '{"index": {"_index": "pruning-mixed-b"}}' + - '{"@timestamp": "2026-02-15T12:00:00.000000000Z", "body": "february"}' + - '{"index": {"_index": "pruning-mixed-c"}}' + - '{"@timestamp": "2026-03-15T12:00:00.000000000Z", "body": "march"}' + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-mixed-* | where @timestamp >= '2026-01-01 00:00:00' and @timestamp <= '2026-02-28 23:59:59' | sort @timestamp | fields body | head 5" + # One row from each type proves the pruned expression spanned the conflict, not one side of it. + - match: {datarows: [["january"], ["february"]]} + - do: + indices.stats: {index: 'pruning-mixed-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-mixed-a: {total: {search: {point_in_time_total: 1}}} + pruning-mixed-b: {total: {search: {point_in_time_total: 1}}} + pruning-mixed-c: {total: {search: {point_in_time_total: 0}}} + +--- +"Prunes a comma separated index expression": + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-it-2026-01,pruning-it-2026-0* | where @timestamp >= '2026-02-01 00:00:00' and @timestamp <= '2026-02-28 23:59:59' | head 5" + - do: + indices.stats: {index: 'pruning-it-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-it-2026-01: {total: {search: {point_in_time_total: 0}}} + pruning-it-2026-02: {total: {search: {point_in_time_total: 1}}} + pruning-it-2026-03: {total: {search: {point_in_time_total: 0}}} + +--- +"Prunes an index that does not map the timestamp": + - do: + indices.create: + index: pruning-nomap-a + body: + settings: {number_of_shards: 1, number_of_replicas: 0, max_result_window: 2} + mappings: {properties: {"@timestamp": {type: date}, body: {type: keyword}}} + - do: + indices.create: + index: pruning-nomap-b + body: + settings: {number_of_shards: 1, number_of_replicas: 0, max_result_window: 2} + mappings: {properties: {body: {type: keyword}}} + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "pruning-nomap-a"}}' + - '{"@timestamp": "2026-02-15T12:00:00Z", "body": "february"}' + - '{"index": {"_index": "pruning-nomap-b"}}' + - '{"body": "untimed"}' + - do: + headers: {Content-Type: 'application/json'} + ppl: + body: + query: "source=pruning-nomap-* | where @timestamp >= '2026-02-01 00:00:00' | fields body | head 5" + - match: {datarows: [["february"]]} + - do: + indices.stats: {index: 'pruning-nomap-*', metric: search, filter_path: 'indices.*.total.search.point_in_time_total'} + - match: + indices: + pruning-nomap-a: {total: {search: {point_in_time_total: 1}}} + pruning-nomap-b: {total: {search: {point_in_time_total: 0}}} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/IndexPruner.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/IndexPruner.java new file mode 100644 index 00000000000..d206a12e23f --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/IndexPruner.java @@ -0,0 +1,166 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.request; + +import static org.opensearch.action.search.SearchRequest.DEFAULT_INDICES_OPTIONS; +import static org.opensearch.sql.calcite.plan.OpenSearchConstants.IMPLICIT_FIELD_TIMESTAMP; +import static org.opensearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.opensearch.action.admin.indices.resolve.ResolveIndexAction; +import org.opensearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.opensearch.common.regex.Regex; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.ConstantScoreQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.sql.opensearch.request.OpenSearchRequest.IndexName; +import org.opensearch.transport.client.node.NodeClient; + +/** + * Prunes a wildcard index expression down to the concrete indices that can match the query's + * filter, ahead of operations that do no pruning of their own, such as PIT creation. + */ +@Log4j2 +@RequiredArgsConstructor +public class IndexPruner { + + /** Caps the substituted expression, not what is read: the wildcard fallback reads no fewer. */ + private static final int MAX_PRUNED_INDICES = 50; + + /** Bounds each probe. The match probe can cost a round trip per shard. */ + private static final TimeValue PROBE_TIMEOUT = TimeValue.timeValueSeconds(5); + + /** Both probes are transport actions, so only the node client can issue them. */ + private final NodeClient node; + + /** + * Returns the index expression to read. When pruning is safe and narrows the read, that is the + * list of indices which can match the filter. Otherwise, and on any probe failure, it is the + * expression the query named. + * + * @param indexName index expression the query named + * @param filter filter pushed down to the search, or null when there is none + * @return expression to read, never null + */ + public IndexName prune(IndexName indexName, QueryBuilder filter) { + try { + IndexExpression indexExpr = new IndexExpression(indexName, node); + if (!isPrunable(indexExpr, filter)) { + log.info("Index pruning skipped: {}", indexExpr); + return indexName; + } + + String[] candidates = indexExpr.probeMatching(filter); + if (0 < candidates.length + && candidates.length <= MAX_PRUNED_INDICES + && indexExpr.isPrunedBy(candidates)) { + return new IndexName(String.join(",", candidates)); + } + log.info( + "Index pruning declined: {} of {} indices matched", + candidates.length, + indexExpr.resolved.getIndices().size()); + } catch (Exception e) { + log.warn("Index pruning failed; querying the full index expression", e); + } + return indexName; + } + + private static boolean isPrunable(IndexExpression expression, QueryBuilder filter) { + return expression.hasWildcard() + && !expression.isCrossCluster() + && containsTimeRange(filter) + // Keep these last: unlike the gates above, they resolve the expression. + && !expression.hasAlias() + && !expression.hasDataStream(); + } + + static boolean containsTimeRange(QueryBuilder query) { + if (query instanceof RangeQueryBuilder range) { + return IMPLICIT_FIELD_TIMESTAMP.equals(range.fieldName()); + } + if (query instanceof BoolQueryBuilder bool) { + return Stream.of(bool.must(), bool.filter(), bool.should()) + .flatMap(List::stream) + .anyMatch(IndexPruner::containsTimeRange); + } + if (query instanceof ConstantScoreQueryBuilder constantScore) { + return containsTimeRange(constantScore.innerQuery()); + } + return false; + } + + /** The index expression a query named, resolving itself the first time it is asked to. */ + static final class IndexExpression { + + private final IndexName indexName; + private final NodeClient node; + private ResolveIndexAction.Response resolved; + + IndexExpression(IndexName indexName, NodeClient node) { + this.indexName = indexName; + this.node = node; + } + + boolean hasWildcard() { + return Arrays.stream(indexName.getIndexNames()).anyMatch(Regex::isSimpleMatchPattern); + } + + boolean isCrossCluster() { + return Arrays.stream(indexName.getIndexNames()) + .anyMatch(name -> name.indexOf(REMOTE_CLUSTER_INDEX_SEPARATOR) >= 0); + } + + boolean hasAlias() { + return !resolved().getAliases().isEmpty(); + } + + boolean hasDataStream() { + return !resolved().getDataStreams().isEmpty(); + } + + boolean isPrunedBy(String[] candidates) { + return candidates.length < resolved().getIndices().size(); + } + + String[] probeMatching(QueryBuilder filter) { + FieldCapabilitiesRequest request = + new FieldCapabilitiesRequest() + .indices(indexName.getIndexNames()) + .fields(IMPLICIT_FIELD_TIMESTAMP) + .indexFilter(filter) + // Must expand as the search will, or candidates describe a different index set. + .indicesOptions(DEFAULT_INDICES_OPTIONS); + return node.fieldCaps(request).actionGet(PROBE_TIMEOUT).getIndices(); + } + + @Override + public String toString() { + return String.format( + "wildcard=%s, crossCluster=%s, alias=%s, dataStream=%s", + hasWildcard(), + isCrossCluster(), + // Guarded so neither a log nor a debugger inspection can fire a resolve probe. + resolved == null ? "n/a" : hasAlias(), + resolved == null ? "n/a" : hasDataStream()); + } + + private ResolveIndexAction.Response resolved() { + if (resolved == null) { + ResolveIndexAction.Request request = + new ResolveIndexAction.Request(indexName.getIndexNames(), DEFAULT_INDICES_OPTIONS); + resolved = node.execute(ResolveIndexAction.INSTANCE, request).actionGet(PROBE_TIMEOUT); + } + return resolved; + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java index a694e0fea06..f56044b033e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -22,6 +22,7 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.join.ScoreMode; import org.jetbrains.annotations.TestOnly; @@ -50,6 +51,7 @@ import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; /** OpenSearch search request builder. */ +@Log4j2 @EqualsAndHashCode @Getter @ToString @@ -74,6 +76,9 @@ public class OpenSearchRequestBuilder { @EqualsAndHashCode.Exclude @ToString.Exclude private final Settings settings; + /** Memoized because build() runs once per scan on the calling thread. */ + @EqualsAndHashCode.Exclude @ToString.Exclude private OpenSearchRequest.IndexName prunedIndexName; + public static class PushDownUnSupportedException extends RuntimeException { public PushDownUnSupportedException(String message) { super(message); @@ -135,9 +140,10 @@ private OpenSearchRequest buildRequestWithPit( if (startFrom + size > maxResultWindow) { sourceBuilder.size(maxResultWindow - startFrom); // Search with PIT request - String pitId = createPit(indexName, cursorKeepAlive, client); + OpenSearchRequest.IndexName prunedName = pruneIndexName(indexName, client); + String pitId = createPit(prunedName, cursorKeepAlive, client); return OpenSearchQueryRequest.pitOf( - indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); + prunedName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); } else { sourceBuilder.from(startFrom); sourceBuilder.size(size); @@ -150,10 +156,32 @@ private OpenSearchRequest buildRequestWithPit( } sourceBuilder.size(pageSize); // Search with PIT request - String pitId = createPit(indexName, cursorKeepAlive, client); + OpenSearchRequest.IndexName prunedName = pruneIndexName(indexName, client); + String pitId = createPit(prunedName, cursorKeepAlive, client); return OpenSearchQueryRequest.pitOf( - indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); + prunedName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); + } + } + + private OpenSearchRequest.IndexName pruneIndexName( + OpenSearchRequest.IndexName indexName, OpenSearchClient client) { + if (!Boolean.TRUE.equals(settings.getSettingValue(Settings.Key.QUERY_PRUNING_ENABLED))) { + return indexName; + } + + if (prunedIndexName == null) { + // Only the node client can issue the pruning probes, so the REST client never prunes. + prunedIndexName = + client + .getNodeClient() + .map( + nodeClient -> new IndexPruner(nodeClient).prune(indexName, sourceBuilder.query())) + .orElse(indexName); + log.info( + "Index expression has {} names after pruning", prunedIndexName.getIndexNames().length); + log.debug("Pruned index expression from {} to {}", indexName, prunedIndexName); } + return prunedIndexName; } private String createPit( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index daa71e44629..9d5b2f2fc8a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -210,6 +210,13 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting QUERY_PRUNING_ENABLED_SETTING = + Setting.boolSetting( + Key.QUERY_PRUNING_ENABLED.getKeyValue(), + false, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting MAX_EXPRESSION_DEPTH_SETTING = Setting.intSetting( Key.MAX_EXPRESSION_DEPTH.getKeyValue(), @@ -532,6 +539,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.QUERY_SIZE_LIMIT, QUERY_SIZE_LIMIT_SETTING, new Updater(Key.QUERY_SIZE_LIMIT)); + register( + settingBuilder, + clusterSettings, + Key.QUERY_PRUNING_ENABLED, + QUERY_PRUNING_ENABLED_SETTING, + new Updater(Key.QUERY_PRUNING_ENABLED)); register( settingBuilder, clusterSettings, @@ -758,6 +771,7 @@ public static List> pluginSettings() { .add(PPL_JOIN_SUBSEARCH_MAXOUT_SETTING) .add(QUERY_MEMORY_LIMIT_SETTING) .add(QUERY_SIZE_LIMIT_SETTING) + .add(QUERY_PRUNING_ENABLED_SETTING) .add(QUERY_BUCKET_SIZE_SETTING) .add(METRICS_ROLLING_WINDOW_SETTING) .add(METRICS_ROLLING_INTERVAL_SETTING) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/IndexExpressionTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/IndexExpressionTest.java new file mode 100644 index 00000000000..d240bd467a7 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/IndexExpressionTest.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.request; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.opensearch.sql.opensearch.request.IndexPruner.IndexExpression; +import org.opensearch.sql.opensearch.request.OpenSearchRequest.IndexName; + +class IndexExpressionTest { + + private static IndexExpression expression(String indexName) { + // Null node client: every predicate under test reads only the expression text. + return new IndexExpression(new IndexName(indexName), null); + } + + @Test + void concreteNameIsNotWildcard() { + assertFalse(expression("logs-2024").hasWildcard()); + } + + @Test + void starIsWildcard() { + assertTrue(expression("logs-*").hasWildcard()); + } + + @Test + void questionMarkIsNotWildcard() { + // OpenSearch resolves index expressions with Glob.globMatch, which honours only '*', so + // "logs-202?" names one literal index. Treating it as a pattern would prune a concrete name. + assertFalse(expression("logs-202?").hasWildcard()); + } + + @Test + void oneWildcardAmongConcreteNamesIsWildcard() { + assertTrue(expression("logs-2024,logs-*").hasWildcard()); + } + + @Test + void localNameIsNotCrossCluster() { + assertFalse(expression("logs-*").isCrossCluster()); + } + + @Test + void qualifiedNameIsCrossCluster() { + assertTrue(expression("remote:logs-*").isCrossCluster()); + } + + @Test + void oneQualifiedNameAmongLocalNamesIsCrossCluster() { + assertTrue(expression("logs-*,remote:logs-*").isCrossCluster()); + } + + @Test + void leadingSeparatorIsCrossCluster() { + // Pins the deliberately conservative reading: a colon anywhere makes the expression + // unprunable, rather than only a colon that follows a cluster name. + assertTrue(expression(":logs-*").isCrossCluster()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/IndexPrunerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/IndexPrunerTest.java new file mode 100644 index 00000000000..7f2b96b4e4d --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/IndexPrunerTest.java @@ -0,0 +1,352 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.request; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.opensearch.index.query.QueryBuilders.boolQuery; +import static org.opensearch.index.query.QueryBuilders.constantScoreQuery; +import static org.opensearch.index.query.QueryBuilders.queryStringQuery; +import static org.opensearch.index.query.QueryBuilders.rangeQuery; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.action.admin.indices.resolve.ResolveIndexAction; +import org.opensearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.opensearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.opensearch.action.search.SearchRequest; +import org.opensearch.common.action.ActionFuture; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.sql.opensearch.request.OpenSearchRequest.IndexName; +import org.opensearch.transport.client.node.NodeClient; + +@ExtendWith(MockitoExtension.class) +class IndexPrunerTest { + + @Mock private NodeClient node; + @Mock private ActionFuture matchFuture; + @Mock private ActionFuture resolveFuture; + @Mock private ResolveIndexAction.Response resolveResponse; + + @Nested + class FilterShapes { + + @ParameterizedTest(name = "{0}") + @MethodSource("filters") + void shouldSeeATimestampRangeOnlyWhereTheProbeCanUseIt( + String shape, QueryBuilder filter, boolean expected) { + assertEquals(expected, IndexPruner.containsTimeRange(filter)); + } + + private static Stream filters() { + return Stream.of( + arguments("bare range on the timestamp", timeRange(), true), + arguments("range under must", boolQuery().must(timeRange()), true), + arguments("range under filter", boolQuery().filter(timeRange()), true), + arguments("range under should", boolQuery().should(timeRange()), true), + arguments("range under constant_score", constantScoreQuery(timeRange()), true), + arguments("no filter at all", null, false), + arguments("filter carrying no range", queryStringQuery("error"), false), + arguments("range on another field", rangeQuery("status").gte(200), false), + arguments( + "range under should beside another clause", + boolQuery().should(timeRange()).should(rangeQuery("status").gte(200)), + true), + // A negated clause cannot prove a shard disjoint, so it must not count. + arguments("range under must_not", boolQuery().mustNot(timeRange()), false)); + } + } + + @Nested + class GateRejections { + + @Test + void shouldNotPruneWhenExpressionHasNoWildcard() { + givenIndexExpression("logs-2024", timeRange()).shouldNotPrune().shouldNotProbe(); + } + + @Test + void shouldNotPruneWhenExpressionIsCrossCluster() { + givenIndexExpression("remote:logs-*", timeRange()).shouldNotPrune().shouldNotProbe(); + } + + @Test + void shouldNotPruneWhenFilterHasNoTimestampRange() { + givenIndexExpression("logs-*", queryStringQuery("error")).shouldNotPrune().shouldNotProbe(); + } + } + + @Nested + class PruningDecisions { + + @Test + void shouldPruneToTheMatchingIndices() { + givenIndexExpression(indices("logs-*", 3), timeRange()) + .whenMatching("logs-e", "logs-f") + .shouldPruneTo("logs-e,logs-f"); + } + + @Test + void shouldNotPruneWhenNoIndexMatches() { + givenIndexExpression(indices("logs-*", 3), timeRange()).whenMatching().shouldNotPrune(); + } + + @Test + void shouldNotPruneWhenEveryIndexMatches() { + givenIndexExpression(indices("logs-*", 2), timeRange()) + .whenMatching("logs-a", "logs-b") + .shouldNotPrune(); + } + + @Test + void shouldNotPruneWhenMoreIndicesMatchThanResolved() { + givenIndexExpression(indices("logs-*", 3), timeRange()) + .whenMatching("logs-a", "logs-b", "logs-c", "logs-d") + .shouldNotPrune(); + } + + @Test + void shouldPruneWhenMatchCountIsAtTheCap() { + givenIndexExpression(indices("logs-*", 100), timeRange()) + .whenMatching(logs(50)) + .shouldPruneTo(String.join(",", logs(50))); + } + + @Test + void shouldNotPruneWhenMatchCountExceedsTheCap() { + givenIndexExpression(indices("logs-*", 100), timeRange()) + .whenMatching(logs(51)) + .shouldNotPrune(); + } + } + + @Nested + class IndirectResolution { + + @Test + void shouldNotPruneWhenExpressionResolvesToAnAlias() { + givenIndexExpression(alias("logs-*"), timeRange()) + .shouldNotPrune() + .shouldNotProbeForMatches(); + } + + @Test + void shouldNotPruneWhenExpressionResolvesToADataStream() { + givenIndexExpression(ds("logs-*"), timeRange()).shouldNotPrune().shouldNotProbeForMatches(); + } + } + + @Nested + class ProbeFailures { + + @Test + void shouldNotPruneWhenTheResolveProbeFails() { + givenIndexExpression(unresolvable("logs-*"), timeRange()) + .shouldNotPrune() + .shouldNotProbeForMatches(); + } + + @Test + void shouldNotPruneWhenTheMatchProbeFails() { + givenIndexExpression(indices("logs-*", 3), timeRange()) + .whenMatchProbeFails() + .shouldNotPrune(); + } + } + + @Nested + class ProbeRequest { + + @Test + void shouldProbeWithTheFilterExpressionAndTimestampField() { + QueryBuilder filter = timeRange(); + ArgumentCaptor captor = + ArgumentCaptor.forClass(FieldCapabilitiesRequest.class); + givenIndexExpression(indices("logs-*", 3), filter) + .whenMatching("logs-a") + .shouldPruneTo("logs-a"); + verify(node).fieldCaps(captor.capture()); + + FieldCapabilitiesRequest probe = captor.getValue(); + assertSame(filter, probe.indexFilter()); + assertArrayEquals(new String[] {"logs-*"}, probe.indices()); + assertEquals( + "[@timestamp]|" + SearchRequest.DEFAULT_INDICES_OPTIONS, + Arrays.toString(probe.fields()) + "|" + probe.indicesOptions()); + } + + @Test + void shouldProbeEveryNameOfACommaSeparatedExpression() { + ArgumentCaptor captor = + ArgumentCaptor.forClass(FieldCapabilitiesRequest.class); + givenIndexExpression(indices("logs-a-*,logs-b-*", 3), timeRange()) + .whenMatching("logs-a-1") + .shouldPruneTo("logs-a-1"); + verify(node).fieldCaps(captor.capture()); + + assertArrayEquals(new String[] {"logs-a-*", "logs-b-*"}, captor.getValue().indices()); + } + } + + private static QueryBuilder timeRange() { + return rangeQuery("@timestamp").gte("now-1d"); + } + + private static String[] logs(int count) { + return IntStream.rangeClosed(1, count).mapToObj(i -> "logs-" + i).toArray(String[]::new); + } + + /** What the resolve probe reports for an expression. */ + private record Resolution(String expression, int indexCount, Shape shape) { + private enum Shape { + INDICES, + ALIAS, + DATA_STREAM, + FAILURE + } + } + + private static Resolution indices(String expression, int indexCount) { + return new Resolution(expression, indexCount, Resolution.Shape.INDICES); + } + + private static Resolution alias(String expression) { + return new Resolution(expression, 0, Resolution.Shape.ALIAS); + } + + private static Resolution ds(String expression) { + return new Resolution(expression, 0, Resolution.Shape.DATA_STREAM); + } + + private static Resolution unresolvable(String expression) { + return new Resolution(expression, 0, Resolution.Shape.FAILURE); + } + + /** An expression the gates reject, so nothing is ever resolved. */ + private Fixture givenIndexExpression(String expression, QueryBuilder filter) { + return new Fixture(expression, filter); + } + + /** + * Each shape stubs only what {@code prune} reads for it, because short-circuiting leaves the rest + * unread and Mockito rejects a stub nobody uses. + */ + private Fixture givenIndexExpression(Resolution resolution, QueryBuilder filter) { + when(node.execute(eq(ResolveIndexAction.INSTANCE), any())).thenReturn(resolveFuture); + switch (resolution.shape()) { + case FAILURE -> + when(resolveFuture.actionGet(any(TimeValue.class))) + .thenThrow(new RuntimeException("boom")); + case ALIAS -> { + whenResolved(); + when(resolveResponse.getAliases()) + .thenReturn(List.of(mock(ResolveIndexAction.ResolvedAlias.class))); + } + case DATA_STREAM -> { + whenResolved(); + when(resolveResponse.getAliases()).thenReturn(List.of()); + when(resolveResponse.getDataStreams()) + .thenReturn(List.of(mock(ResolveIndexAction.ResolvedDataStream.class))); + } + case INDICES -> { + whenResolved(); + when(resolveResponse.getAliases()).thenReturn(List.of()); + when(resolveResponse.getDataStreams()).thenReturn(List.of()); + // Lenient because the count is read only once a match list exists, so a test whose probe + // throws or matches nothing never consumes it. + lenient() + .when(resolveResponse.getIndices()) + .thenReturn( + Collections.nCopies( + resolution.indexCount(), mock(ResolveIndexAction.ResolvedIndex.class))); + } + } + return new Fixture(resolution.expression(), filter); + } + + private void whenResolved() { + when(resolveFuture.actionGet(any(TimeValue.class))).thenReturn(resolveResponse); + } + + private final class Fixture { + + private final IndexName original; + private final QueryBuilder filter; + private IndexName result; + + Fixture(String expression, QueryBuilder filter) { + this.original = new IndexName(expression); + this.filter = filter; + } + + Fixture whenMatching(String... matching) { + when(node.fieldCaps(any())).thenReturn(matchFuture); + when(matchFuture.actionGet(any(TimeValue.class))) + .thenReturn(new FieldCapabilitiesResponse(matching, Collections.emptyMap())); + return this; + } + + Fixture whenMatchProbeFails() { + when(node.fieldCaps(any())).thenReturn(matchFuture); + when(matchFuture.actionGet(any(TimeValue.class))).thenThrow(new RuntimeException("boom")); + return this; + } + + Fixture shouldPruneTo(String expected) { + assertEquals(new IndexName(expected), pruned()); + return this; + } + + /** Equality rather than identity: a declined probe rebuilds the expression it hands back. */ + Fixture shouldNotPrune() { + assertEquals(original, pruned()); + return this; + } + + Fixture shouldNotProbe() { + pruned(); + verify(node, never()).execute(eq(ResolveIndexAction.INSTANCE), any()); + verify(node, never()).fieldCaps(any()); + return this; + } + + Fixture shouldNotProbeForMatches() { + pruned(); + verify(node, never()).fieldCaps(any()); + return this; + } + + /** Runs the pruner once, on the first assertion, so stubbing reads before acting. */ + private IndexName pruned() { + if (result == null) { + result = new IndexPruner(node).prune(original, filter); + } + return result; + } + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java index 1f21f1e769e..dd1d4c725e7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -17,6 +17,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import org.apache.commons.lang3.tuple.Pair; @@ -137,6 +138,57 @@ void build_PIT_request_with_correct_size() { requestBuilder.build(indexName, DEFAULT_QUERY_TIMEOUT, client)); } + @Test + void build_PIT_request_does_not_prune_when_pruning_disabled() { + when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); + when(settings.getSettingValue(Settings.Key.QUERY_PRUNING_ENABLED)).thenReturn(false); + OpenSearchRequest.IndexName wildcard = new OpenSearchRequest.IndexName("test-*"); + requestBuilder.pushDownFilter(rangeQuery("@timestamp").gte("now-1d")); + requestBuilder.pushDownLimit(1, 0); + requestBuilder.pushDownPageSize(2); + + assertEquals( + OpenSearchQueryRequest.pitOf( + new OpenSearchRequest.IndexName("test-*"), + new SearchSourceBuilder() + .from(0) + .size(2) + .timeout(DEFAULT_QUERY_TIMEOUT) + .query(rangeQuery("@timestamp").gte("now-1d")), + exprValueFactory, + List.of(), + TimeValue.timeValueMinutes(1), + "samplePITId"), + requestBuilder.build(wildcard, DEFAULT_QUERY_TIMEOUT, client)); + verify(client, never()).getNodeClient(); + } + + @Test + void build_PIT_request_does_not_prune_without_a_node_client() { + when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); + when(settings.getSettingValue(Settings.Key.QUERY_PRUNING_ENABLED)).thenReturn(true); + // Only the node client can issue the pruning probes, so a REST client leaves the wildcard be. + when(client.getNodeClient()).thenReturn(Optional.empty()); + OpenSearchRequest.IndexName wildcard = new OpenSearchRequest.IndexName("test-*"); + requestBuilder.pushDownFilter(rangeQuery("@timestamp").gte("now-1d")); + requestBuilder.pushDownLimit(1, 0); + requestBuilder.pushDownPageSize(2); + + assertEquals( + OpenSearchQueryRequest.pitOf( + new OpenSearchRequest.IndexName("test-*"), + new SearchSourceBuilder() + .from(0) + .size(2) + .timeout(DEFAULT_QUERY_TIMEOUT) + .query(rangeQuery("@timestamp").gte("now-1d")), + exprValueFactory, + List.of(), + TimeValue.timeValueMinutes(1), + "samplePITId"), + requestBuilder.build(wildcard, DEFAULT_QUERY_TIMEOUT, client)); + } + @Test void buildRequestWithPit_pageSizeNull_sizeGreaterThanMaxResultWindow() { when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId");