Expand SQL plan coverage

plan-dumper/filter-timing-diff.py

@@ -33,9 +33,95 @@
     re.IGNORECASE,
 )
 
-# DuckDB also reports allocator-dependent peak/total memory counters.
+# DuckDB's `optimizer` object consists entirely of per-pass timing counters.
+# These keys do not contain a timing suffix, so the common matcher cannot
+# identify them by name. DuckDB also reports allocator-dependent peak/total
+# memory counters.
+DUCKDB_OPTIMIZER_TIMING_FIELDS = (
+    "aggregate_function_rewriter",
+    "aggregate_reuse",
+    "build_side_probe_side",
+    "column_lifetime",
+    "common_aggregate",
+    "common_subexpressions",
+    "common_subplan",
+    "cte_filter_pusher",
+    "cte_inlining",
+    "deliminator",
+    "duplicate_groups",
+    "empty_result_pullup",
+    "expression_rewriter",
+    "filter_pullup",
+    "filter_pushdown",
+    "grouping_sets",
+    "in_clause",
+    "join_elimination",
+    "join_filter_pushdown",
+    "join_order",
+    "late_materialization",
+    "limit_pushdown",
+    "outer_join_simplification",
+    "partial_aggregate_pushdown",
+    "partitioned_execution",
+    "projection_placement",
+    "projection_pullup",
+    "redundant_order_keys",
+    "regex_range",
+    "reorder_filter",
+    "row_group_pruner",
+    "row_number_rewriter",
+    "sampling_pushdown",
+    "scalar_fn_pushdown",
+    "statistics_propagation",
+    "top_n",
+    "top_n_window_elimination",
+    "type_pushdown",
+    "unnest_rewriter",
+    "unused_columns",
+    "window_self_join",
+)
+
+DUCKDB_PLANNER_TIMING_FIELDS = (
+    "column_binding",
+    "create_plan",
+    "resolve_types",
+)
+
+DUCKDB_OPTIMIZER_TIMING_FIELD_RE = re.compile(
+    r'"(?:'
+    + "|".join(DUCKDB_OPTIMIZER_TIMING_FIELDS + DUCKDB_PLANNER_TIMING_FIELDS)
+    + r')"'
+    + JSON_SCALAR_RE
+)
+
 DUCKDB_VOLATILE_FIELD_RE = re.compile(
-    r'"(?:system_peak_buffer_memory|total_memory_allocated)"' + JSON_SCALAR_RE
+    r'"(?:intermediate_size_bytes|peak_buffer_memory|peak_temp_dir_size|'
+    r'system_peak_buffer_memory|total_intermediate_size_bytes|total_memory_allocated)"'
+    + JSON_SCALAR_RE
+)
+
+# Hyper nests additional execution counters in pipeline statistics. Several of
+# these keys describe a metric rather than a duration, so they need an
+# engine-specific matcher as well.
+HYPER_VOLATILE_FIELD_RE = re.compile(
+    r'"(?:actual|elapsed|expected|instance-seconds|memory-bytes|memory-mb-seconds|'
+    r'peak-transaction-memory-mb|rows-matching-restrictions)"'
+    + JSON_SCALAR_RE
+)
+
+HYPER_WALL_CLOCK_RE = re.compile(r'"wall-clock"' + JSON_SCALAR_RE)
+
+# Hyper's processed-row counters are nested objects within the per-pipeline
+# query metrics. The row counts vary with runtime scheduling and execution
+# details, but are not part of the plan structure.
+HYPER_PROCESSED_ROWS_RE = re.compile(r'"processed-rows"\s*:\s*\{[^{}]*\}')
+
+# CedarDB does not guarantee the order in which analyzePlanPipelines entries
+# are reported. Their operator-ID lists and parallelism labels describe the
+# runtime pipeline association, so mask them together with the timing fields
+# when comparing CedarDB plans.
+CEDARDB_PIPELINE_OPERATORS_RE = re.compile(
+    r'(?:"operators"\s*:\s*\[[^\]]*\]|"parallelism"\s*:\s*"[^"]*")'
 )
 
 def mask_fields(line, pattern):
@@ -47,7 +133,18 @@ def normalize(line, path=None):
     in those values compare equal; a line with any other change still differs."""
     line = mask_fields(line, COMMON_VOLATILE_FIELD_RE)
     if path and "/duckdb/" in f"/{path}":
+        line = mask_fields(line, DUCKDB_OPTIMIZER_TIMING_FIELD_RE)
         line = mask_fields(line, DUCKDB_VOLATILE_FIELD_RE)
+    if path and "/hyper/" in f"/{path}":
+        line = mask_fields(line, HYPER_VOLATILE_FIELD_RE)
+        line = mask_fields(line, HYPER_WALL_CLOCK_RE)
+        line = HYPER_PROCESSED_ROWS_RE.sub(
+            lambda match: match.group(0).split(":")[0] + ": ~", line
+        )
+    if path and "/cedardb/" in f"/{path}":
+        line = CEDARDB_PIPELINE_OPERATORS_RE.sub(
+            lambda match: match.group(0).split(":")[0] + ": ~", line
+        )
     return line
 
 

plan-dumper/queries/cte.sql

@@ -1,5 +1,27 @@
-WITH cte AS (SELECT l_orderkey, SUM(l_quantity), AVG(l_extendedprice)
-   FROM lineitem
-   GROUP BY l_orderkey
+WITH base AS (
+    SELECT l_orderkey, l_partkey, l_quantity, l_extendedprice
+    FROM lineitem
+), filtered AS (
+    SELECT *
+    FROM base
+    WHERE l_quantity > 1
+), grouped AS (
+    SELECT l_orderkey, SUM(l_quantity) AS quantity, AVG(l_extendedprice) AS price
+    FROM filtered
+    GROUP BY l_orderkey
+), nested AS (
+    WITH nested_filtered AS (
+        SELECT l_orderkey, quantity
+        FROM grouped
+        WHERE quantity > 10
+    )
+    SELECT l_orderkey, quantity
+    FROM nested_filtered
+), reused AS (
+    SELECT l_orderkey, quantity FROM nested
+    UNION ALL
+    SELECT l_orderkey, quantity FROM nested
 )
-SELECT * FROM cte UNION ALL SELECT * FROM cte
+SELECT r.l_orderkey, r.quantity, g.price
+FROM reused r
+JOIN grouped g ON g.l_orderkey = r.l_orderkey

plan-dumper/queries/dml/delete-using.sql

@@ -0,0 +1,10 @@
+--- DML queries are intentionally limited to simple mode because ANALYZE
+--- executes the mutation and would change the shared setup state.
+--- UNSUPPORTED: mariadb
+--- MariaDB supports DELETE ... USING, but does not allow RETURNING on
+--- multi-table DELETE statements.
+--- MODES: simple
+DELETE FROM t2
+USING t1
+WHERE t2.a2 = t1.a1
+RETURNING t2.a2, t2.b2

plan-dumper/queries/dml/delete.sql

@@ -0,0 +1,6 @@
+--- DML queries are intentionally limited to simple mode because ANALYZE
+--- executes the mutation and would change the shared setup state.
+--- MODES: simple
+DELETE FROM t2
+WHERE a2 < 10
+RETURNING a2, b2

plan-dumper/queries/dml/insert.sql

@@ -0,0 +1,7 @@
+--- DML queries are intentionally limited to simple mode because ANALYZE
+--- executes the mutation and would change the shared setup state.
+--- MODES: simple
+INSERT INTO t2 (a2, b2, c2)
+SELECT a1, b1, c1
+FROM t1
+RETURNING a2, b2, c2

plan-dumper/queries/dml/update-from.sql

@@ -0,0 +1,11 @@
+--- DML queries are intentionally limited to simple mode because ANALYZE
+--- executes the mutation and would change the shared setup state.
+--- UNSUPPORTED: mariadb
+--- MariaDB does not support PostgreSQL's UPDATE ... FROM syntax.
+--- MODES: simple
+UPDATE t2
+SET a2 = t1.a1 + 1,
+    b2 = t1.b1
+FROM t1
+WHERE t2.a2 = t1.a1
+RETURNING t2.a2, t2.b2

plan-dumper/queries/dml/update.sql

@@ -0,0 +1,7 @@
+--- DML queries are intentionally limited to simple mode because ANALYZE
+--- executes the mutation and would change the shared setup state.
+--- MODES: simple
+UPDATE t2
+SET a2 = a2 + 1,
+    b2 = b2 + 1
+WHERE a2 < 10

plan-dumper/queries/pgq-path.sql

@@ -0,0 +1,36 @@
+--- UNSUPPORTED: postgres, duckdb, umbra, cedardb, mariadb
+--- SQL/PGQ is currently only available in Hyper.
+--- Simple mode is sufficient here because the fixture targets PGQ's
+--- relational lowering and does not need runtime measurements.
+--- MODES: simple, steps
+
+-- Exercise a questioned edge and ONE ROW PER STEP on the TPC-H part/supplier
+-- relationship. The optional edge also gives the sqlpos implementation a
+-- concrete absent-edge branch to annotate.
+WITH pg AS PROPERTY GRAPH (
+    VERTEX TABLES (
+        part KEY (p_partkey)
+            LABEL Product
+            PROPERTIES (part.p_name AS name),
+        supplier KEY (s_suppkey)
+            LABEL Supplier
+            PROPERTIES (supplier.s_name AS name)
+    )
+    EDGE TABLES (
+        partsupp KEY (ps_partkey, ps_suppkey)
+            SOURCE KEY (ps_partkey) REFERENCES part
+            DESTINATION KEY (ps_suppkey) REFERENCES supplier
+            LABEL Supply
+            PROPERTIES (partsupp.ps_supplycost AS supply_cost)
+    )
+)
+SELECT *
+FROM GRAPH_TABLE(pg
+    MATCH (product:Product)-[supply:Supply]->?(vendor:Supplier)
+    ONE ROW PER STEP (source_step, edge_step, destination_step)
+    COLUMNS (
+        source_step.name AS product_name,
+        edge_step.supply_cost AS supply_cost,
+        destination_step.name AS supplier_name
+    )
+) AS matched

plan-dumper/queries/pgq.sql

@@ -0,0 +1,41 @@
+--- UNSUPPORTED: postgres, duckdb, umbra, cedardb, mariadb
+--- SQL/PGQ is currently only available in Hyper.
+--- Simple mode is sufficient here because the fixture targets PGQ's
+--- relational lowering and does not need runtime measurements.
+--- MODES: simple, steps
+
+-- Match parts to their suppliers, using labels and projected properties from
+-- the shared TPC-H schema.
+WITH pg AS PROPERTY GRAPH (
+    VERTEX TABLES (
+        part KEY (p_partkey)
+            LABEL Product
+            PROPERTIES (part.p_name AS name, part.p_brand AS brand),
+        supplier KEY (s_suppkey)
+            LABEL Supplier
+            PROPERTIES (supplier.s_name AS name, supplier.s_acctbal AS account_balance)
+    )
+    EDGE TABLES (
+        partsupp KEY (ps_partkey, ps_suppkey)
+            SOURCE KEY (ps_partkey) REFERENCES part
+            DESTINATION KEY (ps_suppkey) REFERENCES supplier
+            LABEL Supply
+            PROPERTIES (partsupp.ps_supplycost AS supply_cost)
+    )
+)
+SELECT *
+FROM GRAPH_TABLE(pg
+    MATCH (
+        product:Product WHERE product.brand = 'Brand#11'
+    )-[
+        supply:Supply WHERE supply.supply_cost > 100
+    ]->(
+        vendor:Supplier WHERE vendor.account_balance > 0
+    )
+    WHERE product.name LIKE '%blue%'
+    COLUMNS (
+        product.name AS product_name,
+        supply.supply_cost AS supply_cost,
+        vendor.name AS supplier_name
+    )
+) AS matched

plan-dumper/queries/setoperation.sql

@@ -1,14 +1,56 @@
-SELECT l_orderkey, l_partkey FROM lineitem
-UNION ALL
-SELECT l_orderkey, l_suppkey FROM lineitem
-UNION ALL
-(
-   SELECT l_suppkey, l_partkey FROM lineitem
-   WHERE l_quantity < 25
-   INTERSECT ALL
-   SELECT l_suppkey, l_partkey FROM lineitem
-   WHERE l_quantity >= 25
-   EXCEPT ALL
-   SELECT l_suppkey, l_partkey FROM lineitem
-   WHERE l_quantity >= 25
-)
+-- Exercise every set-operation variant, including n-ary and nested trees.
+-- Keeping the arms independent gives optimizers an opportunity to flatten the
+-- UNION/INTERSECT/EXCEPT chains into n-ary operators.
+SELECT 'union' AS operation, value
+FROM (
+    SELECT a1 AS value FROM t1 WHERE a1 < 10
+    UNION
+    SELECT b1 FROM t1 WHERE b1 < 10
+    UNION
+    SELECT c1 FROM t1 WHERE c1 < 10
+) u
+UNION ALL
+SELECT 'union-all', value
+FROM (
+    SELECT a1 AS value FROM t1 WHERE a1 >= 10
+    UNION ALL
+    SELECT b1 FROM t1 WHERE b1 >= 10
+    UNION ALL
+    SELECT c1 FROM t1 WHERE c1 >= 10
+) ua
+UNION ALL
+SELECT 'intersect', value
+FROM (
+    SELECT a1 AS value FROM t1
+    INTERSECT
+    SELECT b1 FROM t1
+    INTERSECT
+    SELECT c1 FROM t1
+) i
+UNION ALL
+SELECT 'intersect-all', value
+FROM (
+    SELECT a1 AS value FROM t1
+    INTERSECT ALL
+    SELECT b1 FROM t1
+    INTERSECT ALL
+    SELECT c1 FROM t1
+) ia
+UNION ALL
+SELECT 'except', value
+FROM (
+    SELECT a1 AS value FROM t1
+    EXCEPT
+    SELECT b1 FROM t1
+    EXCEPT
+    SELECT c1 FROM t1
+) e
+UNION ALL
+SELECT 'except-all', value
+FROM (
+    SELECT a1 AS value FROM t1
+    EXCEPT ALL
+    SELECT b1 FROM t1
+    EXCEPT ALL
+    SELECT c1 FROM t1
+) ea

plan-dumper/queries/subquery-correlated.sql

@@ -0,0 +1,5 @@
+-- A correlated scalar subquery is lowered to a single join on Hyper. The
+-- correlation makes the inner query depend on the current outer row.
+SELECT t1.a1,
+       (SELECT t2.b2 FROM t2 WHERE t2.a2 = t1.a1) AS matching_b
+FROM t1

plan-dumper/queries/subquery.sql

@@ -0,0 +1,6 @@
+-- For this query, the engine must ensure that the subquery returns at most
+-- one row. The optimizer cannot prove this, so that assertion must be
+-- preserved in the query plan somehow.
+SELECT t1.a1,
+       (SELECT t2.b2 FROM t2) AS matching_b
+FROM t1

plan-dumper/queries/tablesample.sql

@@ -0,0 +1,5 @@
+--- UNSUPPORTED: umbra, cedardb, mariadb
+--- Umbra, CedarDB, and MariaDB do not implement TABLESAMPLE SYSTEM or BERNOULLI.
+SELECT a1, b1, c1
+FROM t1 TABLESAMPLE SYSTEM (50)
+REPEATABLE (1)