diff --git a/Cargo.lock b/Cargo.lock index 69d011fa9..d2dbebede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4607,6 +4607,7 @@ dependencies = [ "arrow-schema", "arrow-select", "async-trait", + "axum", "bytes", "chrono", "constant_time_eq", diff --git a/crates/integrations/datafusion/Cargo.toml b/crates/integrations/datafusion/Cargo.toml index 17acaea02..27e344027 100644 --- a/crates/integrations/datafusion/Cargo.toml +++ b/crates/integrations/datafusion/Cargo.toml @@ -50,6 +50,8 @@ uuid = { version = "1", features = ["v4"] } [dev-dependencies] arrow-array = { workspace = true } arrow-schema = { workspace = true } +# for the shared REST catalog mock (crates/paimon/tests/mock_server.rs) +axum = { version = "0.7", features = ["macros", "tokio", "http1", "http2"] } bytes = "1.7.1" flate2 = "1" paimon-ftindex-core = "0.1.0" diff --git a/crates/integrations/datafusion/src/merge_into.rs b/crates/integrations/datafusion/src/merge_into.rs index 97c360795..36bbd838e 100644 --- a/crates/integrations/datafusion/src/merge_into.rs +++ b/crates/integrations/datafusion/src/merge_into.rs @@ -1233,12 +1233,13 @@ pub(crate) async fn register_cow_target_table( return Ok((false, table_name)); } - // Read all files in parallel - let read_futures: Vec<_> = file_index - .iter() - .enumerate() - .map(|(file_idx, file_info)| async move { - let single_split = DataSplitBuilder::new() + // This read rewrites the target files, so it must see raw rows. The scan + // plan's grant would shift the positional row offsets the writer replays, + // rewriting the wrong rows. + let mut splits = Vec::with_capacity(file_index.len()); + for file_info in file_index.iter() { + splits.push( + DataSplitBuilder::new() .with_snapshot(file_info.snapshot_id) .with_partition( paimon::spec::BinaryRow::from_serialized_bytes(&file_info.partition) @@ -1249,8 +1250,19 @@ pub(crate) async fn register_cow_target_table( .with_total_buckets(file_info.total_buckets) .with_data_files(vec![file_info.file_meta.clone()]) .build() - .map_err(to_datafusion_error)?; + .map_err(to_datafusion_error)?, + ); + } + let splits = table + .authorize_rewrite_splits(splits) + .await + .map_err(to_datafusion_error)?; + // Read all files in parallel + let read_futures: Vec<_> = splits + .into_iter() + .enumerate() + .map(|(file_idx, single_split)| async move { let read = table .new_read_builder() .new_read() diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index bf8373ddc..eec468bdf 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -768,6 +768,10 @@ pub struct PaimonTableScan { /// Column-name case sensitivity carried from planning to execution so the /// read path resolves names the same way the scan was planned. case_sensitive: bool, + /// Set from [`paimon::table::Plan::planned_under_restricted_grant`]. Kept + /// on the plan rather than inferred from the splits, so a fully pruned scan + /// still suppresses its metadata. + query_auth_restricted: bool, /// Physical filters retained from DataFusion's runtime filter-pushdown pass. /// They are evaluated exactly by this scan. runtime_filters: Vec>, @@ -778,6 +782,22 @@ pub struct PaimonTableScan { } impl PaimonTableScan { + /// Record that this scan was planned under a row filter or column masking. + pub(crate) fn with_query_auth_restricted(mut self, restricted: bool) -> Self { + self.query_auth_restricted = restricted; + self + } + + /// Whether this scan runs under a row filter or column masking. + fn is_query_auth_restricted(&self) -> bool { + self.query_auth_restricted + || self + .planned_partitions + .iter() + .flat_map(|splits| splits.iter()) + .any(paimon::DataSplit::has_restricted_query_auth_grant) + } + #[allow(clippy::too_many_arguments)] pub(crate) fn new( schema: ArrowSchemaRef, @@ -808,6 +828,7 @@ impl PaimonTableScan { scan_trace, pushed_variants, case_sensitive, + query_auth_restricted: false, runtime_filters: Vec::new(), decoder_filters: Vec::new(), } @@ -851,6 +872,16 @@ impl PaimonTableScan { return Statistics::unknown_column(&self.schema()); } + // Manifest stats precede enforcement: a masked column's bounds are the + // raw values it hides, and null counts cover filtered-out rows. + if partitions + .iter() + .flat_map(|splits| splits.iter()) + .any(DataSplit::has_restricted_query_auth_grant) + { + return Statistics::unknown_column(&self.schema()); + } + let Ok(merge_engine) = self.table.schema().core_options().merge_engine() else { return Statistics::unknown_column(&self.schema()); }; @@ -1066,6 +1097,16 @@ impl ExecutionPlan for PaimonTableScan { None => &self.planned_partitions, }; + // The manifest row count precedes the filter, so it would disclose how + // much data is hidden. Report the whole thing as unknown. + if partitions + .iter() + .flat_map(|splits| splits.iter()) + .any(DataSplit::has_restricted_query_auth_grant) + { + return Ok(Arc::new(Statistics::new_unknown(&self.schema()))); + } + let mut total_rows: usize = 0; let mut all_row_counts_known = true; for splits in partitions { @@ -1109,6 +1150,19 @@ impl DisplayAs for PaimonTableScan { ) -> std::fmt::Result { write!(f, "PaimonTableScan: table={}", self.table.identifier())?; + // These counts and the trace precede the filter/masking that run in + // `TableRead`, so EXPLAIN would disclose what enforcement hides — the + // same reason the statistics are suppressed. + if self.is_query_auth_restricted() { + write!(f, ", query-auth=restricted")?; + let columns = self + .read_type + .iter() + .map(|f| f.name().to_string()) + .collect::>(); + return write!(f, ", projection=[{}]", columns.join(", ")); + } + let total_splits: usize = self.planned_partitions.iter().map(|p| p.len()).sum(); let total_files: usize = self .planned_partitions diff --git a/crates/integrations/datafusion/src/table/mod.rs b/crates/integrations/datafusion/src/table/mod.rs index 1bfc47fc1..4f2acc6ca 100644 --- a/crates/integrations/datafusion/src/table/mod.rs +++ b/crates/integrations/datafusion/src/table/mod.rs @@ -372,18 +372,24 @@ impl PaimonScanBuilder<'_> { .collect() }; - Ok(Arc::new(PaimonTableScan::new( - projected_schema, - self.table.clone(), - read_type, - self.pushed_predicate, - planned_partitions, - self.limit, - self.filter_exact, - self.scan_trace, - None, - self.case_sensitive, - ))) + // From the plan, not its splits: a fully pruned plan carries no split + // to stamp, but its scan metadata is just as pre-enforcement. + let restricted = self.plan.planned_under_restricted_grant(); + Ok(Arc::new( + PaimonTableScan::new( + projected_schema, + self.table.clone(), + read_type, + self.pushed_predicate, + planned_partitions, + self.limit, + self.filter_exact, + self.scan_trace, + None, + self.case_sensitive, + ) + .with_query_auth_restricted(restricted), + )) } } @@ -450,7 +456,11 @@ impl TableProvider for PaimonTableProvider { .map_err(to_datafusion_error)?; let target = state.config_options().execution.target_partitions; + // Inexact plan row counts (a query-auth row filter drops rows inside + // `TableRead`) would let DataFusion's aggregate-statistics rule answer + // COUNT(*) with the unfiltered count without ever invoking the read. let filter_exact = !filter_analysis.requires_residual + && plan.row_counts_exact() && filter_analysis .pushed_predicate .as_ref() diff --git a/crates/integrations/datafusion/src/variant_pushdown.rs b/crates/integrations/datafusion/src/variant_pushdown.rs index a5bb3482b..ef293852b 100644 --- a/crates/integrations/datafusion/src/variant_pushdown.rs +++ b/crates/integrations/datafusion/src/variant_pushdown.rs @@ -243,23 +243,30 @@ impl ExtensionPlanner for VariantExtractionExtensionPlanner { .collect() }; let filter_exact = !filter_analysis.requires_residual + && plan.row_counts_exact() && filter_analysis .pushed_predicate .as_ref() .is_none_or(|p| read_builder.is_exact_filter_pushdown(p)); - Ok(Some(Arc::new(PaimonTableScan::new( - Arc::clone(&node.arrow_schema), - node.table.clone(), - node.read_type.clone(), - filter_analysis.pushed_predicate, - planned_partitions, - pushed_limit, - filter_exact, - Some(scan_trace), - Some(node.pushed_variants.clone()), - case_sensitive, - )))) + // From the plan, not its splits: a fully pruned plan carries no split to + // stamp, but its scan metadata is just as pre-enforcement. + let restricted = plan.planned_under_restricted_grant(); + Ok(Some(Arc::new( + PaimonTableScan::new( + Arc::clone(&node.arrow_schema), + node.table.clone(), + node.read_type.clone(), + filter_analysis.pushed_predicate, + planned_partitions, + pushed_limit, + filter_exact, + Some(scan_trace), + Some(node.pushed_variants.clone()), + case_sensitive, + ) + .with_query_auth_restricted(restricted), + ))) } } diff --git a/crates/integrations/datafusion/tests/query_auth.rs b/crates/integrations/datafusion/tests/query_auth.rs new file mode 100644 index 000000000..46250d69a --- /dev/null +++ b/crates/integrations/datafusion/tests/query_auth.rs @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Query-auth enforcement through the DataFusion provider: SQL over a REST +//! catalog table with `query-auth.enabled` must apply the per-user grant +//! (row filter + column masking) fetched at scan-plan time, and COUNT(*) +//! must not shortcut to unfiltered statistics. +//! +//! This is the same provider path the Python binding +//! (`pypaimon_rust.datafusion`) drives via FFI. + +use std::collections::HashMap; +use std::sync::Arc; + +use datafusion::arrow::array::{Array, Int32Array, Int64Array, StringArray}; +use datafusion::arrow::compute::cast; +use datafusion::arrow::datatypes::{ + DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema, +}; +use datafusion::arrow::record_batch::RecordBatch; +use paimon::api::{AuthTableQueryResponse, ConfigResponse}; +use paimon::catalog::{Identifier, RESTCatalog}; +use paimon::spec::{BigIntType, DataType, IntType, Schema, VarCharType}; +use paimon::{Catalog, CatalogOptions, FileSystemCatalog, Options, Table}; +use paimon_datafusion::SQLContext; + +// Shared REST catalog mock (same source the paimon crate's integration tests +// use); only a subset of its helpers is exercised here. +#[allow(dead_code)] +#[path = "../../../paimon/tests/mock_server.rs"] +mod mock_server; +use mock_server::start_mock_server; + +async fn write_batch(table: &Table, batch: RecordBatch, commit_user: &str) { + let write_builder = table + .new_write_builder() + .with_commit_user(commit_user) + .expect("valid commit user"); + let mut write = write_builder.new_write().expect("create writer"); + write.write_arrow_batch(&batch).await.expect("write batch"); + let messages = write.prepare_commit().await.expect("prepare commit"); + write_builder + .new_commit() + .commit(messages) + .await + .expect("commit batch"); +} + +// Multi-threaded: the provider's `block_on_with_runtime` bridges park the +// current thread, which must not be the only thread serving the mock server. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_query_auth_grant_enforced_via_sql() { + let tmp = tempfile::tempdir().unwrap(); + let warehouse = format!("file://{}", tmp.path().display()); + + // Write demo_employees (id, name, salary) through a plain FileSystemCatalog. + let mut fs_options = Options::new(); + fs_options.set(CatalogOptions::WAREHOUSE, &warehouse); + let fs_catalog = FileSystemCatalog::new(fs_options).expect("create filesystem catalog"); + fs_catalog + .create_database("default", true, HashMap::new()) + .await + .unwrap(); + let columns = || { + Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("name", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("salary", DataType::BigInt(BigIntType::new())) + .option("bucket", "1") + .option("bucket-key", "id") + }; + let identifier = Identifier::new("default", "qa_emp"); + fs_catalog + .create_table(&identifier, columns().build().unwrap(), false) + .await + .unwrap(); + let writer = fs_catalog.get_table(&identifier).await.unwrap(); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, true), + ArrowField::new("name", ArrowDataType::Utf8, true), + ArrowField::new("salary", ArrowDataType::Int64, true), + ])); + let batch = RecordBatch::try_new( + arrow_schema, + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec![ + "alice", "bob", "charlie", "diana", "eve", + ])), + Arc::new(Int64Array::from(vec![120000, 85000, 95000, 70000, 99000])), + ], + ) + .unwrap(); + write_batch(&writer, batch, "u1").await; + + // Serve the same files through a mock REST catalog whose grant applies a + // row filter (salary >= 90000) and masks name -> UPPER(name). + let mut defaults = HashMap::new(); + defaults.insert("prefix".to_string(), "mock-test".to_string()); + let server = start_mock_server( + "test_warehouse".to_string(), + "/tmp/test_warehouse".to_string(), + ConfigResponse::new(defaults), + vec!["default".to_string()], + ) + .await; + server.add_table_with_schema( + "default", + "qa_emp", + columns() + .option("query-auth.enabled", "true") + .build() + .unwrap(), + &format!("{warehouse}/default.db/qa_emp"), + ); + server.set_auth_response( + "default", + "qa_emp", + AuthTableQueryResponse { + filter: Some(vec![ + r#"{"kind":"LEAF","transform":{"name":"FIELD_REF","fieldRef":{"index":2,"name":"salary","type":"BIGINT"}},"function":"GREATER_OR_EQUAL","literals":[90000]}"# + .to_string(), + ]), + column_masking: Some(HashMap::from([( + "name".to_string(), + r#"{"name":"UPPER","inputs":[{"index":1,"name":"name","type":"STRING"}]}"# + .to_string(), + )])), + }, + ); + + let mut rest_options = Options::new(); + rest_options.set("uri", server.url().expect("server url")); + rest_options.set("warehouse", "test_warehouse"); + rest_options.set("token.provider", "bear"); + rest_options.set("token", "test_token"); + let rest_catalog = RESTCatalog::new(rest_options, true) + .await + .expect("create REST catalog"); + + // Sanity: the mock-served table must resolve through the Catalog trait. + rest_catalog + .get_table(&identifier) + .await + .expect("REST get_table(default.qa_emp)"); + + let mut ctx = SQLContext::new(); + ctx.register_catalog("paimon", Arc::new(rest_catalog)) + .await + .expect("register catalog"); + + // Row filter + masking must both be applied on the SQL result. + let batches = ctx + .sql("SELECT id, name, salary FROM paimon.default.qa_emp ORDER BY id") + .await + .expect("plan select") + .collect() + .await + .expect("execute select"); + let mut rows = Vec::new(); + for b in &batches { + let ids = b.column(0).as_any().downcast_ref::().unwrap(); + // DataFusion may hand back Utf8View for string columns; normalize. + let names = cast(b.column(1), &ArrowDataType::Utf8).expect("cast name to Utf8"); + let names = names.as_any().downcast_ref::().unwrap(); + let sal = b.column(2).as_any().downcast_ref::().unwrap(); + for r in 0..b.num_rows() { + rows.push((ids.value(r), names.value(r).to_string(), sal.value(r))); + } + } + assert_eq!( + rows, + vec![ + (1, "ALICE".to_string(), 120000), + (3, "CHARLIE".to_string(), 95000), + (5, "EVE".to_string(), 99000), + ], + "grant must drop salary<90000 rows and uppercase name" + ); + + // COUNT(*) must reflect the filtered row count, not unfiltered statistics + // (the aggregate-statistics optimization must be disabled by the inexact + // plan row counts). + let batches = ctx + .sql("SELECT COUNT(*) FROM paimon.default.qa_emp") + .await + .expect("plan count") + .collect() + .await + .expect("execute count"); + let count = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + assert_eq!(count, 3, "COUNT(*) must not use unfiltered statistics"); +} diff --git a/crates/paimon/src/arrow/residual.rs b/crates/paimon/src/arrow/residual.rs index 56b77050e..7c4419220 100644 --- a/crates/paimon/src/arrow/residual.rs +++ b/crates/paimon/src/arrow/residual.rs @@ -368,7 +368,7 @@ pub(crate) fn evaluate_exact_leaf_predicate( /// operator, including finer-scale literals that cannot be represented at the /// column scale (e.g. `d > 1.05` on a DECIMAL(_,1) column is exactly `d >= 1.1`). /// NULL rows stay NULL in the mask (collapsed to `false` by the caller). -fn evaluate_decimal_leaf( +pub(crate) fn evaluate_decimal_leaf( array: &ArrayRef, op: PredicateOperator, literals: &[Datum], @@ -696,7 +696,7 @@ fn set_membership_hash_mask( } } -fn evaluate_column_predicate( +pub(crate) fn evaluate_column_predicate( column: &ArrayRef, scalar: &Scalar, op: PredicateOperator, @@ -855,7 +855,7 @@ fn combine_filter_masks(left: &BooleanArray, right: &BooleanArray, use_or: bool) BooleanArray::new(values, None) } -fn boolean_mask_from_predicate( +pub(crate) fn boolean_mask_from_predicate( len: usize, mut predicate: impl FnMut(usize) -> bool, ) -> BooleanArray { diff --git a/crates/paimon/src/catalog/partition_listing.rs b/crates/paimon/src/catalog/partition_listing.rs index 3cb75041e..7cdcc44a0 100644 --- a/crates/paimon/src/catalog/partition_listing.rs +++ b/crates/paimon/src/catalog/partition_listing.rs @@ -32,6 +32,9 @@ use crate::Result; /// Scan a table's manifest entries and aggregate them into [`Partition`] rows, /// matching the shape catalogs would otherwise return from a metastore. pub async fn list_partitions_from_file_system(table: &Table) -> Result> { + // Like `Table::partition_stats`: record counts and key values come from raw + // manifests, which a row filter would not touch. + table.authorize_unrestricted_read().await?; let file_io = table.file_io(); let snapshot_sm = table.snapshot_manager(); let manifest_sm = SnapshotManager::new(file_io.clone(), table.location().to_string()); diff --git a/crates/paimon/src/spec/core_options.rs b/crates/paimon/src/spec/core_options.rs index 507471c88..48f341453 100644 --- a/crates/paimon/src/spec/core_options.rs +++ b/crates/paimon/src/spec/core_options.rs @@ -416,8 +416,9 @@ impl<'a> CoreOptions<'a> { /// Whether `query-auth.enabled` is set. /// - /// When set, the server enforces a per-user row filter / column masking that this client - /// can't yet apply, so read paths fail closed (see `ensure_read_authorized`). + /// When set, every read must first be authorized against the REST server, + /// which returns a per-user row filter / column masking to apply. Paths that + /// cannot do so fail closed (see `ensure_read_authorized`). pub fn query_auth_enabled(&self) -> bool { self.options .get(QUERY_AUTH_ENABLED_OPTION) @@ -431,9 +432,9 @@ impl<'a> CoreOptions<'a> { pub fn ensure_read_authorized(&self) -> crate::Result<()> { if self.query_auth_enabled() { return Err(crate::Error::Unsupported { - message: "reading a table with 'query-auth.enabled' = true is not supported: \ - the Rust client cannot yet enforce its row-level auth filter / column \ - masking, so it refuses to read to avoid returning unfiltered data" + message: "reading a table with 'query-auth.enabled' = true is not supported on \ + this path: it cannot obtain or apply the server's per-user row filter / \ + column masking, so it refuses to read to avoid returning raw data" .to_string(), }); } diff --git a/crates/paimon/src/spec/predicate.rs b/crates/paimon/src/spec/predicate.rs index f91573b7f..82b2aa2ed 100644 --- a/crates/paimon/src/spec/predicate.rs +++ b/crates/paimon/src/spec/predicate.rs @@ -631,6 +631,45 @@ impl Predicate { Predicate::AlwaysTrue | Predicate::AlwaysFalse => {} } } + + /// Like [`Self::collect_leaf_field_indices`], but skipping leaves that name + /// a reserved system column. Their index is a placeholder (`_ROW_ID` uses + /// 0), so treating it as a table-schema index would point at an unrelated + /// column; callers track those leaves by name instead. + pub fn collect_user_leaf_field_indices(&self, out: &mut std::collections::HashSet) { + match self { + Predicate::Leaf { index, column, .. } => { + if !crate::table::query_auth::is_reserved_system_field_name(column) { + out.insert(*index); + } + } + Predicate::And(children) | Predicate::Or(children) => { + children + .iter() + .for_each(|c| c.collect_user_leaf_field_indices(out)); + } + Predicate::Not(inner) => inner.collect_user_leaf_field_indices(out), + Predicate::AlwaysTrue | Predicate::AlwaysFalse => {} + } + } + + /// Column names referenced by every leaf. Unlike + /// [`Self::collect_leaf_field_indices`] this surfaces system columns too, + /// whose positional index is not a table-schema index. + pub fn collect_leaf_column_names(&self, out: &mut std::collections::HashSet) { + match self { + Predicate::Leaf { column, .. } => { + out.insert(column.clone()); + } + Predicate::And(children) | Predicate::Or(children) => { + children + .iter() + .for_each(|c| c.collect_leaf_column_names(out)); + } + Predicate::Not(inner) => inner.collect_leaf_column_names(out), + Predicate::AlwaysTrue | Predicate::AlwaysFalse => {} + } + } } fn rest_json_err(detail: impl fmt::Display) -> Error { @@ -2191,6 +2230,28 @@ mod tests { // ======================== Decimal equivalence ======================== + #[test] + fn test_user_leaf_indices_skip_system_columns() { + use std::collections::HashSet; + // A `_ROW_ID` leaf carries a placeholder index (0). Treating it as a + // table-schema index would drag an unrelated column into the auth + // request and the scope checks. + let row_id = Predicate::Leaf { + index: 0, + column: crate::spec::ROW_ID_FIELD_NAME.to_string(), + data_type: DataType::BigInt(BigIntType::new()), + op: PredicateOperator::GtEq, + literals: vec![Datum::Long(5)], + }; + let mut all = HashSet::new(); + row_id.collect_leaf_field_indices(&mut all); + assert_eq!(all, HashSet::from([0]), "placeholder index is present"); + + let mut user = HashSet::new(); + row_id.collect_user_leaf_field_indices(&mut user); + assert!(user.is_empty(), "system leaf must not contribute an index"); + } + #[test] fn test_decimal_eq_same_scale() { let a = Datum::Decimal { diff --git a/crates/paimon/src/spec/schema.rs b/crates/paimon/src/spec/schema.rs index b3c48d13f..f516af16d 100644 --- a/crates/paimon/src/spec/schema.rs +++ b/crates/paimon/src/spec/schema.rs @@ -255,6 +255,17 @@ impl TableSchema { Ok(()) } + /// Force `query-auth.enabled = true`. The REST catalog delivers the flag on + /// the table response, so a copy built from an on-disk schema (a branch's) + /// must re-assert it. + pub(crate) fn copy_with_query_auth_enabled(&self) -> Self { + let mut new_schema = self.clone(); + new_schema + .options + .insert(QUERY_AUTH_ENABLED_OPTION.to_string(), "true".to_string()); + new_schema + } + /// Apply a list of schema changes and return a new schema with incremented ID. /// /// Column-level changes operate on **top-level** columns only: a diff --git a/crates/paimon/src/table/audit_log_table.rs b/crates/paimon/src/table/audit_log_table.rs index a6b6e5fe0..e4b0e2039 100644 --- a/crates/paimon/src/table/audit_log_table.rs +++ b/crates/paimon/src/table/audit_log_table.rs @@ -77,7 +77,11 @@ impl AuditLogTable { start_exclusive: i64, end_inclusive: i64, ) -> IncrementalScan<'_> { + // The audit read prepends `rowkind` (and `_SEQUENCE_NUMBER` when + // enabled) to its output; declare them at planning time so the server + // sees them in `select` and the grant's system scope covers them. IncrementalScan::for_table(&self.wrapped, mode, start_exclusive, end_inclusive) + .with_audit_system_fields() } pub fn to_arrow(&self, plan: &IncrementalPlan) -> crate::Result { diff --git a/crates/paimon/src/table/btree_global_index_build_builder.rs b/crates/paimon/src/table/btree_global_index_build_builder.rs index 693a15dc5..76619f145 100644 --- a/crates/paimon/src/table/btree_global_index_build_builder.rs +++ b/crates/paimon/src/table/btree_global_index_build_builder.rs @@ -84,6 +84,11 @@ impl<'a> BTreeGlobalIndexBuildBuilder<'a> { pub async fn execute(&self) -> Result { self.table.ensure_not_branch_reference_for_write()?; + // Authorize before reading any manifest or index metadata, and once for + // the whole build: doing it per shard both skipped the early-return + // paths and issued one REST round-trip per shard. + let build_grant = self.table.authorize_unrestricted_read().await?; + let grant = build_grant.as_ref(); let index_type = normalize_sorted_global_index_type(&self.index_type).ok_or_else(|| { Error::Unsupported { @@ -167,7 +172,7 @@ impl<'a> BTreeGlobalIndexBuildBuilder<'a> { let mut messages = Vec::with_capacity(shard_count); for shard in shards { let index_file = self - .build_index_file(&shard, index_field, index_column) + .build_index_file(&shard, index_field, index_column, grant) .await?; let mut message = CommitMessage::new(shard.partition_bytes.clone(), shard.source_bucket, vec![]); @@ -194,6 +199,7 @@ impl<'a> BTreeGlobalIndexBuildBuilder<'a> { shard: &BTreeGlobalIndexShard, index_field: &DataField, index_column: &str, + grant: Option<&std::sync::Arc>, ) -> Result { let index_type = normalize_sorted_global_index_type(&self.index_type).ok_or_else(|| { Error::Unsupported { @@ -205,8 +211,15 @@ impl<'a> BTreeGlobalIndexBuildBuilder<'a> { })?; let row_count = checked_row_count(shard.row_range_start, shard.row_range_end)?; let (cmp, serialize_key) = make_index_key_codec(index_type, index_field.data_type()); - let mut rows = - extract_index_rows(self.table, shard, index_column, index_field, serialize_key).await?; + let mut rows = extract_index_rows( + self.table, + shard, + index_column, + index_field, + serialize_key, + grant, + ) + .await?; sort_index_rows(&mut rows, &cmp); self.table @@ -613,8 +626,12 @@ async fn extract_index_rows( index_column: &str, index_field: &DataField, serialize_key: SerializeKeyFn, + grant: Option<&std::sync::Arc>, ) -> Result> { - let splits = build_read_splits_for_shard(shard)?; + let splits: Vec = build_read_splits_for_shard(shard)? + .into_iter() + .map(|s| s.with_query_auth_grant(grant.cloned())) + .collect(); let mut read_builder = table.new_read_builder(); read_builder.with_projection(&[index_column, ROW_ID_FIELD_NAME])?; diff --git a/crates/paimon/src/table/bucket_assigner_cross.rs b/crates/paimon/src/table/bucket_assigner_cross.rs index f0e711b2a..6deb7e00e 100644 --- a/crates/paimon/src/table/bucket_assigner_cross.rs +++ b/crates/paimon/src/table/bucket_assigner_cross.rs @@ -85,6 +85,11 @@ impl GlobalPartitionIndex { .collect(); let projected_pk_indices: Vec = (0..pk_fields.len()).collect(); + // The cross-partition PK index reads every primary key; under a + // restricted grant it would miss hidden keys and produce duplicate PKs + // or lost updates on upsert. + table.authorize_unrestricted_read().await?; + let mut rb = table.new_read_builder(); rb.with_projection(&pk_field_names)?; let scan = rb.new_scan().with_scan_all_files(); diff --git a/crates/paimon/src/table/cow_writer.rs b/crates/paimon/src/table/cow_writer.rs index 6c348d15c..06e119f44 100644 --- a/crates/paimon/src/table/cow_writer.rs +++ b/crates/paimon/src/table/cow_writer.rs @@ -210,6 +210,12 @@ impl CopyOnWriteMergeWriter { return Ok(Vec::new()); } + // A copy-on-write rewrite reads each affected file and rewrites it in + // place, so under a restricted grant it would silently delete hidden + // rows and persist masked values. Require an unrestricted grant and + // stamp it on each split, so `to_arrow` reads raw. + let write_grant = self.table.authorize_unrestricted_read().await?; + let schema = self.table.schema(); let core_options = CoreOptions::new(schema.options()); let partition_keys: Vec = schema.partition_keys().to_vec(); @@ -232,6 +238,7 @@ impl CopyOnWriteMergeWriter { let update_batches = &self.update_batches; let file_index = &self.file_index; let table = &self.table; + let write_grant = &write_grant; let partition_keys = &partition_keys; let partition_computer = &partition_computer; let write_fields = &write_fields; @@ -253,7 +260,8 @@ impl CopyOnWriteMergeWriter { .with_bucket_path(file_info.bucket_path.clone()) .with_total_buckets(file_info.total_buckets) .with_data_files(vec![file_info.file_meta.clone()]) - .build()?; + .build()? + .with_query_auth_grant(write_grant.clone()); let read = table.new_read_builder().new_read()?; let original_batches: Vec = diff --git a/crates/paimon/src/table/data_evolution_writer.rs b/crates/paimon/src/table/data_evolution_writer.rs index 47752e3f7..0f86beb8b 100644 --- a/crates/paimon/src/table/data_evolution_writer.rs +++ b/crates/paimon/src/table/data_evolution_writer.rs @@ -164,6 +164,10 @@ impl DataEvolutionWriter { return Ok(Vec::new()); } + // This rewrite reads each affected file's original columns, so a + // restricted grant would filter/mask rows into the committed result. + let write_grant = self.table.authorize_unrestricted_read().await?; + // 1. Scan file metadata and build row_id -> file group index. // In data-evolution tables, multiple files can share the same first_row_id // (base file + partial-column files). We must group them so the reader @@ -248,7 +252,8 @@ impl DataEvolutionWriter { .with_total_buckets(file_range.total_buckets) .with_data_files(file_range.files.clone()) .with_raw_convertible(file_range.files.len() == 1) - .build()?; + .build()? + .with_query_auth_grant(write_grant.clone()); let stream = read.to_arrow(&[split])?; let original_batches: Vec = stream.try_collect().await?; @@ -460,6 +465,10 @@ impl DataEvolutionDeleteWriter { return Ok(Vec::new()); } + // The row ids come from a read the caller performed; under a restricted + // grant the deletion vectors would encode a filtered view. + self.table.authorize_unrestricted_read().await?; + let scan = self .table .new_read_builder() diff --git a/crates/paimon/src/table/format_read_builder.rs b/crates/paimon/src/table/format_read_builder.rs index f97805c0a..b12b0ddee 100644 --- a/crates/paimon/src/table/format_read_builder.rs +++ b/crates/paimon/src/table/format_read_builder.rs @@ -24,6 +24,7 @@ use super::{Table, TableRead, TableScan}; use crate::spec::{DataField, Predicate}; use crate::table::source::RowRange; use crate::Result; +use std::collections::HashSet; #[derive(Debug, Clone)] pub(crate) struct FormatReadBuilder<'a> { @@ -37,6 +38,7 @@ pub(crate) struct FormatReadBuilder<'a> { data_predicates: Vec, limit: Option, case_sensitive: bool, + filter_columns: HashSet, } impl<'a> FormatReadBuilder<'a> { @@ -49,6 +51,7 @@ impl<'a> FormatReadBuilder<'a> { data_predicates: Vec::new(), limit: None, case_sensitive: true, + filter_columns: HashSet::new(), } } @@ -81,6 +84,10 @@ impl<'a> FormatReadBuilder<'a> { } pub(crate) fn with_filter(&mut self, filter: Predicate) -> &mut Self { + // Capture the full predicate's columns before it is split, so masked and + // out-of-scope partition keys can't prune on their raw value. + self.filter_columns.clear(); + filter.collect_leaf_field_indices(&mut self.filter_columns); let (partition_predicate, data_predicates) = split_scan_predicates(self.table, filter); self.partition_filter = partition_predicate.map(|pred| { PartitionFilter::from_predicate(pred, &self.table.schema().partition_fields()) @@ -111,11 +118,46 @@ impl<'a> FormatReadBuilder<'a> { self.limit, None, ) + .with_query_auth_scope( + self.filter_columns.clone(), + self.projected_schema_indices(), + self.projected_system_field_names(), + ) + } + + /// Table-schema indices of the projected columns (`None` = all). + /// Projected system fields (`_ROW_ID`, …). `projected_schema_indices` drops + /// them for lack of an index, but Java's `select` includes them. + fn projected_system_field_names(&self) -> Vec { + // Format tables have no row-id extraction (`with_row_ranges` is inert), + // so the read type is the only source. + crate::table::query_auth::projected_system_field_names( + self.resolve_read_type().ok().flatten().as_deref(), + &std::collections::HashSet::new(), + false, + ) + } + + fn projected_schema_indices(&self) -> Option> { + // Resolve names too (see `PaimonReadBuilder::projected_schema_indices`). + self.resolve_read_type().ok().flatten().map(|fields| { + fields + .iter() + .filter_map(|f| { + self.table + .schema() + .fields() + .iter() + .position(|s| s.id() == f.id()) + }) + .collect() + }) } pub(crate) fn new_read(&self) -> Result> { - let core_options = self.table.schema().core_options(); - core_options.ensure_read_authorized()?; + // Query-auth is enforced in `TableRead::to_arrow` off the grant stamped + // on the splits by planning; no gate needed here (see the Paimon + // `PaimonReadBuilder::new_read`). let read_type = match self.resolve_read_type()? { None => self.table.schema().fields().to_vec(), Some(fields) => fields, diff --git a/crates/paimon/src/table/format_table_read.rs b/crates/paimon/src/table/format_table_read.rs index b9ea8e5c2..17ac09d9e 100644 --- a/crates/paimon/src/table/format_table_read.rs +++ b/crates/paimon/src/table/format_table_read.rs @@ -70,6 +70,10 @@ impl<'a> FormatTableRead<'a> { self.table } + pub(crate) fn limit(&self) -> Option { + self.limit + } + pub(crate) fn with_filter(mut self, filter: Predicate) -> Self { self.data_predicates = split_scan_predicates(self.table, filter).1; self @@ -87,8 +91,9 @@ impl<'a> FormatTableRead<'a> { &self, data_splits: &[DataSplit], ) -> crate::Result { - let core_options = self.table.schema().core_options(); - core_options.ensure_read_authorized()?; + // Query-auth (fail-closed + row filter + masking) is enforced by the + // outer `TableRead::to_arrow` off the grant stamped on the splits. + let core_options = self.table.schema.core_options(); let read_type = self.read_type.clone(); let output_schema = build_target_arrow_schema(&read_type)?; let partition_keys = self.table.schema().partition_keys().to_vec(); diff --git a/crates/paimon/src/table/format_table_scan.rs b/crates/paimon/src/table/format_table_scan.rs index df2a48e02..96d3a2ae4 100644 --- a/crates/paimon/src/table/format_table_scan.rs +++ b/crates/paimon/src/table/format_table_scan.rs @@ -32,6 +32,10 @@ pub(crate) struct FormatTableScan<'a> { table: &'a Table, partition_filter: Option, limit: Option, + query_auth_filter_columns: std::collections::HashSet, + query_auth_projected: Option>, + /// Sent in `select`, but has no index to scope. + query_auth_system_select: Vec, } impl<'a> FormatTableScan<'a> { @@ -44,26 +48,104 @@ impl<'a> FormatTableScan<'a> { table, partition_filter, limit, + query_auth_filter_columns: std::collections::HashSet::new(), + query_auth_projected: None, + query_auth_system_select: Vec::new(), } } + pub(super) fn with_query_auth_scope( + mut self, + filter_columns: std::collections::HashSet, + projected: Option>, + system_select: Vec, + ) -> Self { + self.query_auth_filter_columns = filter_columns; + self.query_auth_projected = projected; + self.query_auth_system_select = system_select; + self + } + pub(crate) async fn plan(&self) -> crate::Result { - self.ensure_query_auth_allowed()?; - self.plan_inner(None).await + let grant = self.ensure_query_auth_allowed().await?; + let has_row_filter = grant.as_deref().is_some_and(|g| g.has_row_filter()); + self.plan_inner(None, has_row_filter) + .await + .map(|plan| self.finalize_plan(plan, grant.as_ref())) } pub(crate) async fn plan_with_trace(&self) -> crate::Result<(Plan, ScanTrace)> { - self.ensure_query_auth_allowed()?; + let grant = self.ensure_query_auth_allowed().await?; + let has_row_filter = grant.as_deref().is_some_and(|g| g.has_row_filter()); let mut trace = ScanTrace::default(); - let plan = self.plan_inner(Some(&mut trace)).await?; - Ok((plan, trace)) + let plan = self.plan_inner(Some(&mut trace), has_row_filter).await?; + Ok((self.finalize_plan(plan, grant.as_ref()), trace)) + } + + /// Stamp the grant onto every split (so `TableRead::to_arrow` enforces it) + /// and mark row counts inexact when it carries a row filter. + fn finalize_plan( + &self, + plan: Plan, + grant: Option<&std::sync::Arc>, + ) -> Plan { + // Any restricted grant invalidates the plan's statistics (see the + // Paimon `TableScan::finalize_plan`). + let restricted = grant.is_some_and(|g| g.has_server_restrictions()); + let plan = plan.stamp_query_auth_grant(grant.cloned()); + if restricted { + plan.with_inexact_row_counts() + } else { + plan + } } - fn ensure_query_auth_allowed(&self) -> crate::Result<()> { - CoreOptions::new(self.table.schema().options()).ensure_read_authorized() + async fn ensure_query_auth_allowed( + &self, + ) -> crate::Result>> { + // Fetch/refresh the grant at plan time (Java parity), then guard + // against pruning on masked or out-of-scope columns. + let select = self.query_auth_projected.as_ref().map(|projected| { + projected + .iter() + .copied() + .chain(self.query_auth_filter_columns.iter().copied()) + .collect::>() + }); + let grant = self + .table + .verify_query_auth_for_read(select.as_ref(), Some(&self.query_auth_system_select)) + .await?; + if let Some(grant) = &grant { + // The grant authorizes the catalog table, but `plan_inner` reads + // from `CoreOptions::path`. An override could point the scan at + // another directory and have those splits stamped with this grant. + let core = CoreOptions::new(self.table.schema().options()); + if core.path().is_some_and(|p| { + p.trim_end_matches('/') != self.table.location().trim_end_matches('/') + }) { + return Err(crate::Error::Unsupported { + message: "a 'path' override cannot be used on a \ + 'query-auth.enabled' table: the grant authorizes the \ + catalog table's location, not the overridden one" + .to_string(), + }); + } + crate::table::query_auth::scope_check( + grant, + self.table.schema().fields(), + &self.query_auth_filter_columns, + self.query_auth_projected.clone(), + )?; + } + Ok(grant) } - async fn plan_inner(&self, trace: Option<&mut ScanTrace>) -> crate::Result { + async fn plan_inner( + &self, + trace: Option<&mut ScanTrace>, + query_auth_row_filter: bool, + ) -> crate::Result { let core_options = CoreOptions::new(self.table.schema().options()); let format_extension = supported_format_table_extension(core_options.file_format())?; let schema_id = self.table.schema().id(); @@ -103,7 +185,7 @@ impl<'a> FormatTableScan<'a> { .cmp(&right.data_files()[0].file_name) }) }); - splits = self.apply_limit_pushdown(splits); + splits = self.apply_limit_pushdown(splits, query_auth_row_filter); if let Some(trace) = trace { trace.record_final_plan(splits.len(), splits.len(), splits.len()); @@ -275,7 +357,13 @@ impl<'a> FormatTableScan<'a> { pub(crate) fn apply_limit_pushdown( &self, splits: Vec, + query_auth_row_filter: bool, ) -> Vec { + // A query-auth row filter runs as a residual pass at read time, so the + // scan must not cap files by an unfiltered limit before that. + if query_auth_row_filter { + return splits; + } match self.limit { Some(0) => Vec::new(), Some(limit) if splits.len() > limit => splits.into_iter().take(limit).collect(), diff --git a/crates/paimon/src/table/full_text_search_builder.rs b/crates/paimon/src/table/full_text_search_builder.rs index 702c32959..4f8d78ba2 100644 --- a/crates/paimon/src/table/full_text_search_builder.rs +++ b/crates/paimon/src/table/full_text_search_builder.rs @@ -116,8 +116,10 @@ impl<'a> FullTextSearchBuilder<'a> { pub async fn execute_scored(&self) -> crate::Result { // Fail closed: returns data-derived row ranges outside `TableScan`/`TableRead`. + // Strict: search results bypass the query-auth row filter, so only a + // fully unrestricted grant may search. + self.table.authorize_unrestricted_read().await?; let core = CoreOptions::new(self.table.schema().options()); - core.ensure_read_authorized()?; let text_column = self.text_column .as_deref() @@ -202,9 +204,11 @@ impl<'a> FullTextSearchBuilder<'a> { /// returning nothing, since the append/data-evolution materialized read is not /// supported here. pub async fn execute_read(&self) -> crate::Result { - // Fail closed: returns data outside `TableScan`/`TableRead`. + // Fail closed: materializes rows outside `TableScan`/`TableRead`, so it + // cannot apply the row filter / masking — only a fully unrestricted + // grant may run it (same gate as the scored entry points). + self.table.authorize_unrestricted_read().await?; let core = CoreOptions::new(self.table.schema().options()); - core.ensure_read_authorized()?; let text_column = self.text_column .as_deref() diff --git a/crates/paimon/src/table/hybrid_search_builder.rs b/crates/paimon/src/table/hybrid_search_builder.rs index ab2908206..f80e5beba 100644 --- a/crates/paimon/src/table/hybrid_search_builder.rs +++ b/crates/paimon/src/table/hybrid_search_builder.rs @@ -286,8 +286,10 @@ impl<'a> HybridSearchBuilder<'a> { } pub async fn execute_scored(&self) -> crate::Result { + // Strict: search results bypass the query-auth row filter, so only a + // fully unrestricted grant may search. + self.table.authorize_unrestricted_read().await?; let core = CoreOptions::new(self.table.schema().options()); - core.ensure_read_authorized()?; let limit = self.limit.ok_or_else(|| crate::Error::ConfigInvalid { message: "Limit must be set via with_limit()".to_string(), })?; @@ -350,8 +352,11 @@ impl<'a> HybridSearchBuilder<'a> { /// (append/data-evolution) hybrid is unsupported here — those use /// `execute`/`execute_scored`. Mirrors Java `HybridSearchBuilderImpl` PK path. pub async fn execute_read(&self) -> crate::Result { + // Materializes rows outside `TableScan`/`TableRead`, so it cannot apply + // the row filter / masking — only a fully unrestricted grant may run it + // (same gate as the vector and full-text builders). + self.table.authorize_unrestricted_read().await?; let core = CoreOptions::new(self.table.schema().options()); - core.ensure_read_authorized()?; let limit = self.limit.ok_or_else(|| crate::Error::ConfigInvalid { message: "Limit must be set via with_limit()".to_string(), })?; diff --git a/crates/paimon/src/table/incremental_scan.rs b/crates/paimon/src/table/incremental_scan.rs index cdfc15fa1..d568680d8 100644 --- a/crates/paimon/src/table/incremental_scan.rs +++ b/crates/paimon/src/table/incremental_scan.rs @@ -158,6 +158,24 @@ impl IncrementalPlan { }) .collect() } + + /// Every data split this plan reads, both sides of a + /// [`IncrementalSplit::DiffPair`] included. Authorization must use this: + /// [`Self::data_splits`] drops the pairs, so a Diff plan would present no + /// splits and skip the grant check. + pub(crate) fn all_data_splits(&self) -> Vec<&DataSplit> { + let mut out = Vec::new(); + for split in &self.splits { + match split { + IncrementalSplit::Data(data) => out.push(data), + IncrementalSplit::DiffPair { before, after } => { + out.extend(before.iter()); + out.extend(after.iter()); + } + } + } + out + } } pub(crate) fn validate_diff_pair(before: &[DataSplit], after: &[DataSplit]) -> crate::Result<()> { @@ -224,6 +242,19 @@ impl<'a> IncrementalScan<'a> { Self::new(table, scan, mode, start_exclusive, end_inclusive) } + /// Declare the system fields an audit-log read emits on top of the read + /// type, so planning asks the server about them. + pub(crate) fn with_audit_system_fields(mut self) -> Self { + let mut names = vec![crate::spec::ROW_KIND_FIELD_NAME.to_string()]; + if crate::table::table_read::audit_sequence_number_enabled(self.table) { + names.push(crate::spec::SEQUENCE_NUMBER_FIELD_NAME.to_string()); + } + self.scan = self + .scan + .with_query_auth_scope(std::collections::HashSet::new(), None, names); + self + } + pub(crate) fn new( table: &'a Table, scan: TableScan<'a>, diff --git a/crates/paimon/src/table/lumina_index_build_builder.rs b/crates/paimon/src/table/lumina_index_build_builder.rs index 5e45abf8e..ba8aa5240 100644 --- a/crates/paimon/src/table/lumina_index_build_builder.rs +++ b/crates/paimon/src/table/lumina_index_build_builder.rs @@ -72,6 +72,11 @@ impl<'a> LuminaIndexBuildBuilder<'a> { pub async fn execute(&self) -> Result { self.table.ensure_not_branch_reference_for_write()?; + // Authorize before reading any manifest or index metadata, and once for + // the whole build: doing it per shard both skipped the early-return + // paths and issued one REST round-trip per shard. + let build_grant = self.table.authorize_unrestricted_read().await?; + let grant = build_grant.as_ref(); if !is_lumina_index_type(&self.index_type) { return Err(Error::DataInvalid { @@ -159,7 +164,8 @@ impl<'a> LuminaIndexBuildBuilder<'a> { let shard_count = shards.len(); let mut messages = Vec::with_capacity(shard_count); for shard in shards { - let vectors = extract_vectors(self.table, &shard, index_column, dimension).await?; + let vectors = + extract_vectors(self.table, &shard, index_column, dimension, grant).await?; let index_file = self .build_index_file( &shard, @@ -570,7 +576,10 @@ async fn extract_vectors( shard: &LuminaIndexShard, index_column: &str, dimension: i32, + grant: Option<&std::sync::Arc>, ) -> Result> { + // Index building reads raw values, so `grant` must be the unrestricted one + // the caller obtained; stamp it so the read is authorized. let split = DataSplitBuilder::new() .with_snapshot(shard.snapshot_id) .with_partition(shard.partition.clone()) @@ -582,7 +591,8 @@ async fn extract_vectors( shard.row_range_start, shard.row_range_end, )]) - .build()?; + .build()? + .with_query_auth_grant(grant.cloned()); let mut read_builder = table.new_read_builder(); read_builder.with_projection(&[index_column, ROW_ID_FIELD_NAME])?; diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 4078e3a23..cacd6febe 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -74,6 +74,7 @@ mod pk_vector_position_read; mod pk_vector_scan; mod postpone_file_writer; mod prepared_files; +pub(crate) mod query_auth; mod read_builder; pub mod referenced_files; pub(crate) mod rest_env; @@ -142,7 +143,9 @@ pub use write_builder::WriteBuilder; use crate::catalog::{validate_branch_name, Identifier, DEFAULT_MAIN_BRANCH}; use crate::io::FileIO; use crate::spec::{CoreOptions, DataField, Snapshot, TableSchema}; -use std::collections::HashMap; +use query_auth::QueryAuthGrant; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; /// Table represents a table in the catalog. #[derive(Debug, Clone)] @@ -229,6 +232,244 @@ impl Table { }) } + /// Fetch the per-user row filter / column masking as the grant the read + /// pipeline enforces; `None` when the table is not `query-auth.enabled`. + /// + /// `select` are the queried columns (table-schema indices, `None` = all), + /// like Java's `readType.getFieldNames()`: a column-restricted user can + /// read an authorized subset, and a wider read fails closed until it + /// re-plans. Called per plan, as Java's `CatalogEnvironment.tableQueryAuth` + /// is, so a revoked grant takes effect on the next one. + /// `system_select` is `None` for internal raw reads, which run only under a + /// fully unrestricted grant and legitimately touch `_ROW_ID`; scan planning + /// always passes `Some(names)`, and an empty slice then approves none. + pub(crate) async fn verify_query_auth_for_read( + &self, + select: Option<&HashSet>, + system_select: Option<&[String]>, + ) -> Result>> { + if !CoreOptions::new(self.schema.options()).query_auth_enabled() { + return Ok(None); + } + let Some(rest_env) = &self.rest_env else { + return Err(crate::Error::Unsupported { + message: "reading a table with 'query-auth.enabled' = true requires a REST \ + catalog to authorize the query" + .to_string(), + }); + }; + let fields = self.schema.fields(); + // Java's `select` is `readType.getFieldNames()`, so projected system + // fields go too. They have no index to scope, but the server may still + // refuse them — which it can only do if they are actually sent, so a + // full read that wants one must spell out every column rather than rely + // on a null select. + let wanted_system: &[String] = system_select.unwrap_or(&[]); + let select_names = match select { + Some(indices) => Some( + fields + .iter() + .enumerate() + .filter(|(i, _)| indices.contains(i)) + .map(|(_, f)| f.name().to_string()) + .chain(wanted_system.iter().cloned()) + .collect::>(), + ), + None if wanted_system.is_empty() => None, + None => Some( + fields + .iter() + .map(|f| f.name().to_string()) + .chain(wanted_system.iter().cloned()) + .collect::>(), + ), + }; + let auth = rest_env.table_query_auth(select_names).await?; + let filters = query_auth::parse_auth_filters(&auth.filter.unwrap_or_default(), fields)?; + let masks = + query_auth::parse_column_masking(&auth.column_masking.unwrap_or_default(), fields)?; + + // Rules are expressed against the LATEST schema, but this `Table` may + // predate a concurrent evolution: a re-added column keeps its name and + // gets a fresh id. Java re-validates on every plan. + // Also with no rules: `select` names columns of THIS copy's schema, so a + // column dropped since would be authorized against a latest schema that + // no longer has it, while the read still decodes it by field id. + // Always, not just when rules exist: `select` (and `None`, meaning every + // column of THIS copy) names columns of this schema, so one dropped + // since would be authorized against a latest schema that no longer has + // it while the read still decodes it by field id. + self.ensure_rules_bind_to_latest_schema(&filters, &masks, select) + .await?; + let grant = QueryAuthGrant::new( + filters, + masks, + select.cloned(), + system_select.map(|names| names.iter().cloned().collect()), + query_auth::GrantBinding::of(self), + ); + + // Rules are expressed against the CURRENT schema; a time-travelled or + // branch copy reads a different one, where the same name may be an + // unrelated field id. Keyed on server rules, not `is_unrestricted()`: + // the client's own projection scope binds nothing to the schema. + if grant.has_server_restrictions() && (self.time_traveled || self.branch_reference) { + return Err(crate::Error::Unsupported { + message: "a query-auth row filter / column masking grant cannot be applied to a \ + time-travelled or branch read: the grant is bound to the table's \ + current schema" + .to_string(), + }); + } + + Ok(Some(Arc::new(grant))) + } + + /// Fail closed when a rule column binds to a different field in the latest + /// schema than in this copy — a rename or drop-and-re-add would apply the + /// rule to unrelated data. Mirrors Java `validateReadableWithoutRename`. + async fn ensure_rules_bind_to_latest_schema( + &self, + filters: &[crate::spec::Predicate], + masks: &[query_auth::ColumnMask], + select: Option<&HashSet>, + ) -> Result<()> { + // Compare ids before loading: the directory listing is the freshness + // check and cannot be skipped, but the schema itself need not be read + // when this copy is already current (the common case). + // A table with no on-disk schema directory has no drift to detect — the + // REST-provided schema is all there is. Listing it is also the only file + // access on this path, so a table whose location is not readable (mock + // catalogs, tables served entirely over REST) must not fail the read. + let Ok(ids) = self.schema_manager.list_all_ids().await else { + return Ok(()); + }; + let Some(&latest_id) = ids.last() else { + return Ok(()); + }; + if latest_id == self.schema.id() { + return Ok(()); + } + let latest = self.schema_manager.schema(latest_id).await?; + let mut referenced = HashSet::new(); + for filter in filters { + filter.collect_leaf_field_indices(&mut referenced); + } + for mask in masks { + referenced.insert(mask.column); + mask.transform.collect_field_indices(&mut referenced); + } + let fields = self.schema.fields(); + match select { + Some(select) => referenced.extend(select.iter().copied()), + // `None` = every column of this copy, which is exactly what a full + // read decodes. + None => referenced.extend(0..fields.len()), + } + for index in referenced { + let Some(field) = fields.get(index) else { + continue; + }; + // Type too, not just name and id: `disable-explicit-type-casting` + // defaults to false, so a narrowing evolution (DOUBLE -> FLOAT) is + // allowed. The auth JSON is parsed and evaluated against THIS + // copy's type, so `x > 0.1` on a stale DOUBLE would admit values the + // current FLOAT semantics reject. + let matched = latest + .fields() + .iter() + .find(|f| f.name() == field.name()) + .is_some_and(|f| f.id() == field.id() && f.data_type() == field.data_type()); + if !matched { + return Err(crate::Error::Unsupported { + message: format!( + "query-auth read references column `{}`, which the table's latest \ + schema exposes as a different column (renamed, or dropped and \ + re-added); refusing to read rather than apply it to unrelated data", + field.name() + ), + }); + } + } + Ok(()) + } + + /// Authorize a read that cannot enforce filtering / masking on its output + /// (search, system tables, write-path rewrites). Returns the grant to stamp + /// on its splits; fails closed on a restricted one, which would either leak + /// (search/metadata) or commit a filtered view (rewrites). + pub(crate) async fn authorize_unrestricted_read(&self) -> Result>> { + match self.verify_query_auth_for_read(None, None).await? { + Some(grant) if grant.is_unrestricted() => Ok(Some(grant)), + Some(_) => Err(crate::Error::Unsupported { + message: "this read on a 'query-auth.enabled' table must see raw rows, so it \ + cannot apply the server's row filter / column masking and refuses to \ + run under a restricted grant" + .to_string(), + }), + None => Ok(None), + } + } + + /// Authorize an internal read that rewrites data (copy-on-write DML, index + /// builds) and stamp the grant on `splits`. + /// + /// Rewriting from a filtered or masked view would destroy hidden rows and + /// persist masked values, so this requires a fully unrestricted grant. Scan + /// planning's grant must NOT be used: a row filter shifts the positional + /// row offsets rewrites replay. + pub async fn authorize_rewrite_splits(&self, splits: Vec) -> Result> { + // This is the only public API that stamps a grant, so it must not be + // usable to launder splits: refuse ones that already carry a restricted + // grant rather than overwriting it. + if splits.iter().any(DataSplit::carries_query_auth_restriction) { + return Err(crate::Error::Unsupported { + message: "cannot re-authorize a split already planned under a query-auth row \ + filter / column masking grant" + .to_string(), + }); + } + // Catch another table's splits being handed to a permissive one. + // Bucket paths only: `external_path` legitimately points anywhere. A + // sanity guard, not a boundary against in-process code. + let root = self.location().trim_end_matches('/'); + if let Some(split) = splits.iter().find(|s| { + // On a component boundary: `.../t` must reject `.../t2`. + s.bucket_path() + .trim_end_matches('/') + .strip_prefix(root) + .is_none_or(|rest| !rest.is_empty() && !rest.starts_with('/')) + }) { + return Err(crate::Error::Unsupported { + message: format!( + "cannot authorize a split at `{}`: it does not belong to table `{}`", + split.bucket_path(), + self.identifier().full_name() + ), + }); + } + let grant = self.authorize_unrestricted_read().await?; + Ok(splits + .into_iter() + .map(|split| split.with_query_auth_grant(grant.clone())) + .collect()) + } + + /// Authorize a commit. The data may have come from an enforced read (e.g. + /// `INSERT OVERWRITE t SELECT * FROM t`), which would destroy hidden rows + /// and persist masked values; committing cannot tell, so it fails closed + /// for any restricted grant. + pub(crate) async fn authorize_unrestricted_write(&self) -> Result<()> { + self.authorize_unrestricted_read().await.map(|_| ()) + } + + /// Fail closed when a read reaches `TableRead::to_arrow` without a grant + /// stamped on its splits (an unauthorized path) and the table is + /// `query-auth.enabled`; a no-op otherwise. + pub(crate) fn ensure_read_without_grant(&self) -> Result<()> { + CoreOptions::new(self.schema.options()).ensure_read_authorized() + } + /// Get the table's identifier. pub fn identifier(&self) -> &Identifier { &self.identifier @@ -456,11 +697,21 @@ impl Table { })?; let mut options = schema.options().clone(); options.insert("branch".to_string(), branch.clone()); + // The flag comes from the REST table response, not the branch's on-disk + // schema. A branch references the same files, so dropping it would make + // `t$branch_x` read them unauthorized. + let branch_schema = if CoreOptions::new(self.schema.options()).query_auth_enabled() { + schema + .copy_with_replaced_options(options) + .copy_with_query_auth_enabled() + } else { + schema.copy_with_replaced_options(options) + }; Ok(Self { file_io: self.file_io.clone(), identifier: self.identifier.clone(), location: self.location.clone(), - schema: schema.copy_with_replaced_options(options), + schema: branch_schema, schema_manager, branch, branch_reference: true, @@ -522,3 +773,171 @@ pub(crate) fn query_auth_table() -> Table { None, ) } + +#[cfg(test)] +mod tests { + #[tokio::test] + async fn test_authorize_unrestricted_read_fails_closed() { + // Every write/build path authorizes through this gate before its + // internal read, so a table it cannot authorize as unrestricted must + // fail closed rather than commit a filtered/masked view. + let table = super::query_auth_table(); + let err = table.authorize_unrestricted_read().await.unwrap_err(); + assert!( + matches!(err, crate::Error::Unsupported { ref message } if message.contains("query-auth.enabled")), + "write-path authorization must fail closed, got: {err}" + ); + } + + #[tokio::test] + async fn test_latest_schema_check_tolerates_an_unlistable_location() { + use crate::catalog::Identifier; + use crate::io::FileIOBuilder; + use crate::spec::{DataType, IntType, Schema, TableSchema}; + + // The check is the only file access on the authorization path. A table + // served entirely over REST — or a mock catalog — may point at a + // location with no schema directory, and that must not fail the read. + let table = super::Table::new( + FileIOBuilder::new("file").build().unwrap(), + Identifier::new("default", "t"), + "/nonexistent-a1b2c3/does/not/exist".to_string(), + TableSchema::new( + 0, + &Schema::builder() + .column("id", DataType::Int(IntType::new())) + .build() + .unwrap(), + ), + None, + ); + assert!( + table + .ensure_rules_bind_to_latest_schema(&[], &[], None) + .await + .is_ok(), + "an unlistable location must not fail authorization" + ); + } + + #[tokio::test] + async fn test_authorize_rewrite_splits_rejects_foreign_splits() { + use crate::spec::BinaryRow; + use crate::table::DataSplitBuilder; + + let table = super::query_auth_table(); + let root = table.location().to_string(); + let split_at = |path: &str| { + DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(path.to_string()) + .with_total_buckets(1) + .with_data_files(vec![]) + .build() + .unwrap() + }; + + // A sibling sharing this location's prefix must not pass. + let sibling = format!("{root}2/bucket-0"); + let err = table + .authorize_rewrite_splits(vec![split_at(&sibling)]) + .await + .unwrap_err(); + assert!( + matches!(err, crate::Error::Unsupported { ref message } if message.contains("does not belong to table")), + "prefix-sharing sibling must be rejected, got: {err}" + ); + + // Its own split passes provenance and fails later, at authorization. + let own = format!("{root}/bucket-0"); + let err = table + .authorize_rewrite_splits(vec![split_at(&own)]) + .await + .unwrap_err(); + assert!( + !err.to_string().contains("does not belong to table"), + "own split must pass the provenance check, got: {err}" + ); + } + + // `copy_with_branch` lists the schema directory, and opendal's fs lister + // panics on Windows when stripping the prefix of a `file:/C:/…` path. + #[cfg(not(windows))] + #[tokio::test] + async fn test_branch_copy_inherits_query_auth_flag() { + use crate::catalog::Identifier; + use crate::io::FileIOBuilder; + use crate::spec::{CoreOptions, DataType, IntType, Schema, TableSchema}; + + // The flag arrives on the REST table response, not the branch's + // on-disk schema; if `copy_with_branch` dropped it, `t$branch_x` would + // read the same files unauthorized. Drive the real method. + let tmp = tempfile::tempdir().unwrap(); + // `file:` URL with forward slashes; a bare path breaks on Windows. + let location = { + let p = tmp.path().to_string_lossy().replace('\\', "/"); + if p.starts_with('/') { + format!("file:{p}") + } else { + format!("file:/{p}") + } + }; + let file_io = FileIOBuilder::new("file").build().unwrap(); + + // The on-disk schema deliberately has NO query-auth option. + let on_disk = TableSchema::new( + 0, + &Schema::builder() + .column("id", DataType::Int(IntType::new())) + .build() + .unwrap(), + ); + let sm = super::SchemaManager::new(file_io.clone(), location.clone()); + file_io + .new_output(&sm.schema_path(0)) + .unwrap() + .write(serde_json::to_vec(&on_disk).unwrap().into()) + .await + .unwrap(); + + let build = |schema: TableSchema| { + super::Table::new( + file_io.clone(), + Identifier::new("default", "auth_t"), + location.clone(), + schema, + None, + ) + }; + + let plain = build(on_disk.clone()); + assert!( + !CoreOptions::new( + plain + .copy_with_branch(super::DEFAULT_MAIN_BRANCH) + .await + .unwrap() + .schema() + .options() + ) + .query_auth_enabled(), + "a branch of a non-query-auth table must not gain the flag" + ); + + let guarded = build(on_disk.copy_with_query_auth_enabled()); + assert!( + CoreOptions::new( + guarded + .copy_with_branch(super::DEFAULT_MAIN_BRANCH) + .await + .unwrap() + .schema() + .options() + ) + .query_auth_enabled(), + "the branch copy must inherit query-auth.enabled from the REST table" + ); + } +} diff --git a/crates/paimon/src/table/partition_stat.rs b/crates/paimon/src/table/partition_stat.rs index 9d78b58ad..da785d4cd 100644 --- a/crates/paimon/src/table/partition_stat.rs +++ b/crates/paimon/src/table/partition_stat.rs @@ -64,6 +64,9 @@ impl Table { /// /// Returns an empty Vec when the table has no snapshots yet. pub async fn partition_stats(&self) -> crate::Result> { + // Record counts and key values come from raw manifests, which a row + // filter would not touch. Also covers `list_partitions`. + self.authorize_unrestricted_read().await?; let sm = SnapshotManager::new(self.file_io().clone(), self.location().to_string()); let snapshot = match sm.get_latest_snapshot().await? { Some(s) => s, diff --git a/crates/paimon/src/table/query_auth.rs b/crates/paimon/src/table/query_auth.rs new file mode 100644 index 000000000..f059a0484 --- /dev/null +++ b/crates/paimon/src/table/query_auth.rs @@ -0,0 +1,1454 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Query-auth enforcement: apply the REST server's per-user row filter and +//! column masking exactly to read output. Parsing of the Java `Predicate` / +//! `Transform` JSON lives in [`crate::spec`]; anything unrecognised is an +//! error, so callers keep the table fail-closed. + +use crate::arrow::residual::{ + boolean_mask_from_predicate, evaluate_column_predicate, evaluate_decimal_leaf, + literal_scalar_for_arrow_filter, sanitize_filter_mask, +}; +use crate::spec::{ + DataField, DataType, Datum, Predicate, PredicateOperator, Transform, TransformInput, +}; +use crate::{Error, Result}; +use arrow_arith::boolean::{and_kleene, not, or_kleene}; +use arrow_array::{ArrayRef, BooleanArray, Float32Array, Float64Array, RecordBatch}; +use std::collections::HashSet; +use std::sync::Arc; + +/// Row filters and column masks the REST server granted this user. +/// `authorized = None` means all columns; `Some(set)` scopes the grant to those +/// table-schema indices. Only the REST catalog constructs grants. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct QueryAuthGrant { + filters: Vec, + masks: Vec, + authorized: Option>, + /// System fields the request asked about (`_ROW_ID`, …). They have no table + /// index, so `authorized` cannot hold them. + /// + /// `Some(set)` approves exactly `set` — an empty set approves none, so a + /// full projection is not blanket approval for `_ROW_ID`. `None` means + /// system scoping does not apply: internal raw reads (index builds, write + /// rewrites) authorize through `authorize_unrestricted_read`, run only when + /// the server imposed no rules at all, and legitimately read `_ROW_ID`. + authorized_system: Option>, + /// Where this grant's positional indices are meaningful. The schema id + /// alone would not do: it is a per-table counter, so two fresh tables + /// both sit at 0. + binding: GrantBinding, +} + +/// Identity a grant is bound to. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct GrantBinding { + /// REST table UUID when available: an identifier or schema id can be reused + /// across catalogs or a drop/recreate, a UUID cannot. + uuid: Option, + /// Catalog identity: two REST aliases can share a location. + identifier: String, + location: String, + branch: String, + schema_id: i64, +} + +impl GrantBinding { + pub(crate) fn of(table: &super::Table) -> Self { + Self { + uuid: table.rest_env().map(|e| e.uuid().to_string()), + identifier: table.identifier().full_name(), + location: table.location().to_string(), + branch: table.branch().to_string(), + schema_id: table.schema().id(), + } + } +} + +impl QueryAuthGrant { + /// `binding` is required so it cannot be forgotten. + pub(crate) fn new( + filters: Vec, + masks: Vec, + authorized: Option>, + authorized_system: Option>, + binding: GrantBinding, + ) -> Self { + Self { + filters, + masks, + authorized, + authorized_system, + binding, + } + } + + /// Like [`Self::check_system_scope`], for system fields a read path adds on + /// top of its read type (the audit schema's `rowkind`). + pub(crate) fn check_system_scope_by_name(&self, names: &[&str]) -> Result<()> { + let Some(approved) = &self.authorized_system else { + return Ok(()); + }; + for name in names { + if !approved.contains(*name) { + return Err(unsupported(format!( + "query-auth read emits system column `{name}` outside the authorized set" + ))); + } + } + Ok(()) + } + + /// Fail closed when `read_type` projects a system field outside this grant's + /// scope. Scoped by name: system fields have no table index. + pub(crate) fn check_system_scope(&self, read_type: &[DataField]) -> Result<()> { + let Some(approved) = &self.authorized_system else { + return Ok(()); + }; + for field in read_type.iter().filter(|f| is_reserved_system_field(f)) { + if !approved.contains(field.name()) { + return Err(unsupported(format!( + "query-auth read projects system column `{}` outside the authorized set", + field.name() + ))); + } + } + Ok(()) + } + + /// Whether this grant may be enforced on `table`. + pub(crate) fn matches_table(&self, table: &super::Table) -> bool { + self.binding == GrantBinding::of(table) + } + + /// Fully unrestricted: every column approved, no filter, no masking. + pub(crate) fn is_unrestricted(&self) -> bool { + self.authorized.is_none() && self.filters.is_empty() && self.masks.is_empty() + } + + /// Whether the SERVER restricted this user, as opposed to the client merely + /// scoping its own projection. Statistics suppression and the historical / + /// scan-all gates key off this: a column scope distorts nothing. + pub(crate) fn has_server_restrictions(&self) -> bool { + !self.filters.is_empty() || !self.masks.is_empty() + } + + pub(crate) fn filters(&self) -> &[Predicate] { + &self.filters + } + + pub(crate) fn masks(&self) -> &[ColumnMask] { + &self.masks + } + + /// Whether every table-schema index in `columns` was authorized. A grant + /// scoped to a subset does not authorize columns outside it, so a wider + /// projection or a predicate on an un-approved column fails closed. + pub(crate) fn authorizes_columns(&self, columns: impl IntoIterator) -> bool { + match &self.authorized { + None => true, + Some(set) => columns.into_iter().all(|c| set.contains(&c)), + } + } + + /// Whether this grant carries a row filter. Such a filter is applied as a + /// residual pass in `TableRead::to_arrow`, so split row counts, count + /// statistics, and count-based limit pushdown are no longer exact. + pub(crate) fn has_row_filter(&self) -> bool { + !self.filters.is_empty() + } + + /// Table-schema indices of columns this grant masks (empty when no masking). + /// Callers must not push predicates on these columns to scan pruning, which + /// would leak the raw value via row presence. + pub(crate) fn masked_columns(&self) -> Vec { + self.masks.iter().map(|m| m.column).collect() + } + + /// The first of `columns` (table-schema indices) this grant does not + /// authorize, if any. + pub(crate) fn first_unauthorized( + &self, + columns: impl IntoIterator, + ) -> Option { + columns.into_iter().find(|c| !self.authorizes_columns([*c])) + } + + /// Field IDs the grant must physically read (filter columns, mask targets + /// and inputs). Projection planning must include them, or column-slice + /// pruning drops the file and the column reads as null. + pub(crate) fn read_field_ids(&self, fields: &[DataField]) -> Vec { + let mut indices = HashSet::new(); + for filter in &self.filters { + filter.collect_leaf_field_indices(&mut indices); + } + for mask in &self.masks { + indices.insert(mask.column); + mask.transform.collect_field_indices(&mut indices); + } + indices + .into_iter() + .filter_map(|i| fields.get(i).map(|f| f.id())) + .collect() + } +} + +/// Mask one column (by table-schema index) with a transform. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ColumnMask { + pub(crate) column: usize, + pub(crate) transform: Transform, +} + +fn unsupported(message: String) -> Error { + Error::Unsupported { message } +} + +fn field_name(fields: &[DataField], column: usize) -> &str { + fields.get(column).map(|f| f.name()).unwrap_or("?") +} + +/// Fail-closed error for a caller predicate on a masked column (would leak the +/// raw value via row selection). Shared by every builder/scan choke point. +pub(crate) fn masked_filter_error(fields: &[DataField], column: usize) -> Error { + unsupported(format!( + "cannot filter on masked column `{}`", + field_name(fields, column) + )) +} + +/// Fail-closed error for a read that touches a column outside the grant's scope. +pub(crate) fn unauthorized_column_error(fields: &[DataField], column: usize) -> Error { + unsupported(format!( + "query-auth read touches column `{}` outside the authorized set", + field_name(fields, column) + )) +} + +/// Reserved system fields a read type may project. The reader produces them, so +/// they carry no grant scope. Both id and name must match, so a forged field +/// cannot borrow a system id. Mirrors Java `SpecialFields.SYSTEM_FIELD_NAMES`. +const RESERVED_SYSTEM_FIELDS: [(i32, &str); 4] = [ + (crate::spec::ROW_ID_FIELD_ID, crate::spec::ROW_ID_FIELD_NAME), + ( + crate::spec::SEQUENCE_NUMBER_FIELD_ID, + crate::spec::SEQUENCE_NUMBER_FIELD_NAME, + ), + ( + crate::spec::VALUE_KIND_FIELD_ID, + crate::spec::VALUE_KIND_FIELD_NAME, + ), + ( + crate::spec::ROW_KIND_FIELD_ID, + crate::spec::ROW_KIND_FIELD_NAME, + ), +]; + +pub(crate) fn is_reserved_system_field_name(name: &str) -> bool { + RESERVED_SYSTEM_FIELDS.iter().any(|(_, n)| *n == name) +} + +pub(crate) fn is_reserved_system_field(field: &DataField) -> bool { + RESERVED_SYSTEM_FIELDS + .iter() + .any(|(id, name)| *id == field.id() && *name == field.name()) +} + +/// Table-schema indices a read type touches, rejecting any field that is not a +/// canonical `(id, name)` pair of `fields`. +/// +/// Scoping resolves by id while the physical read resolves by name, so an +/// authorized id under another column's name would be scoped by one and read as +/// the other. `TableRead::new` is public, so this is reachable. +pub(crate) fn canonical_projection( + fields: &[DataField], + read_type: &[DataField], +) -> crate::Result> { + let mut indices = Vec::with_capacity(read_type.len()); + for field in read_type { + let by_id = fields.iter().position(|s| s.id() == field.id()); + let by_name = fields.iter().position(|s| s.name() == field.name()); + match (by_id, by_name) { + (Some(i), Some(n)) if i == n => indices.push(i), + // Absent from the schema. Anything but a system field is a dropped + // column, which the reader still resolves by id in older files. + // Java rejects it in `TableQueryAuthResult.checkFieldExists`. + (None, None) if is_reserved_system_field(field) => {} + _ => { + return Err(unsupported(format!( + "query-auth read type field #{} `{}` is not a column of this table", + field.id(), + field.name() + ))) + } + } + } + Ok(indices) +} + +/// Names of the reserved system fields a read projects, plus any the caller's +/// filter references. Shared by the Paimon and Format read builders, which +/// otherwise kept two copies of this and drifted apart. +/// +/// `filter_system_names` must be captured before row-id extraction rewrites the +/// predicates, and `slices_by_row_id` covers an explicit row-range slice, which +/// selects by `_ROW_ID` with no predicate at all. +pub(crate) fn projected_system_field_names( + read_type: Option<&[DataField]>, + filter_system_names: &HashSet, + slices_by_row_id: bool, +) -> Vec { + let mut names: HashSet = read_type + .map(|fields| { + fields + .iter() + .filter(|f| is_reserved_system_field(f)) + .map(|f| f.name().to_string()) + .collect() + }) + .unwrap_or_default(); + names.extend(filter_system_names.iter().cloned()); + if slices_by_row_id { + names.insert(crate::spec::ROW_ID_FIELD_NAME.to_string()); + } + let mut names: Vec = names.into_iter().collect(); + names.sort(); + names +} + +/// Everything a read must satisfy to run under `grant`, in one place. +/// +/// Every read-side gate calls exactly this, so a check cannot be added to one +/// gate and forgotten in another — the failure mode this module kept hitting. +/// Returns the read type's table-schema indices, which callers need anyway. +/// +/// `implicit_system_fields` are system columns the path emits on top of +/// `read_type` (the audit schema prepends `rowkind`), which therefore never +/// reached the projection or the auth request. +pub(crate) fn authorize_read( + grant: &QueryAuthGrant, + table: &super::Table, + read_type: &[DataField], + predicates: &[Predicate], + implicit_system_fields: &[&str], +) -> crate::Result> { + // The grant's positional indices only mean anything on the table it was + // issued for; `Table::authorize_rewrite_splits` is public, so a grant can + // arrive on another table's splits. + if !grant.matches_table(table) { + return Err(unsupported( + "a query-auth grant issued for a different table or schema cannot be \ + enforced here; re-plan the scan" + .to_string(), + )); + } + grant.check_system_scope(read_type)?; + grant.check_system_scope_by_name(implicit_system_fields)?; + + let fields = table.schema().fields(); + let projected = canonical_projection(fields, read_type)?; + let mut filter_columns = HashSet::new(); + for predicate in predicates { + predicate.collect_leaf_field_indices(&mut filter_columns); + } + scope_check(grant, fields, &filter_columns, Some(projected.clone()))?; + Ok(projected) +} + +/// Scope check shared by the read/scan gates: fail closed when the caller filter +/// references a masked column (pruning on its raw value would leak it) or +/// touches one outside the grant. `projected = None` means all columns. +pub(crate) fn scope_check( + grant: &QueryAuthGrant, + fields: &[DataField], + filter_columns: &HashSet, + projected: Option>, +) -> crate::Result<()> { + if let Some(column) = grant + .masked_columns() + .into_iter() + .find(|c| filter_columns.contains(c)) + { + return Err(masked_filter_error(fields, column)); + } + let projected = projected.unwrap_or_else(|| (0..fields.len()).collect()); + if let Some(column) = + grant.first_unauthorized(projected.into_iter().chain(filter_columns.iter().copied())) + { + return Err(unauthorized_column_error(fields, column)); + } + Ok(()) +} + +/// Parse the auth response's JSON filter strings into predicates whose leaf +/// indices refer to `fields` (table-schema order). Empty strings are skipped +/// (Java parity); anything unparseable is an error. +pub(crate) fn parse_auth_filters( + filters: &[String], + fields: &[DataField], +) -> Result> { + filters + .iter() + // Java skips only length-0 entries (`StringUtils.isEmpty`). Whitespace + // is invalid JSON: dropping it could leave a grant with no filter. + .filter(|f| !f.is_empty()) + .map(|f| Predicate::from_rest_json(f, fields)) + .collect() +} + +// ==================== Exact evaluation ==================== + +/// Evaluate the ANDed `predicates` against `batch` (columns 1:1 with +/// `batch_fields`; leaf indices into `schema_fields`) and drop non-matching +/// rows. Unlike the pruning evaluators, anything unevaluable is an error — a +/// security filter must not fall open. +pub(crate) fn strict_filter_batch( + batch: &RecordBatch, + predicates: &[Predicate], + schema_fields: &[DataField], + batch_fields: &[DataField], +) -> Result { + let mut combined: Option = None; + for predicate in predicates { + let mask = strict_mask(batch, predicate, schema_fields, batch_fields)?; + combined = Some(match combined { + Some(existing) => kleene(and_kleene(&existing, &mask))?, + None => mask, + }); + } + let Some(mask) = combined else { + return Ok(batch.clone()); + }; + let mask = sanitize_filter_mask(mask); + arrow_select::filter::filter_record_batch(batch, &mask).map_err(|e| Error::DataInvalid { + message: format!("failed to apply query-auth row filter: {e}"), + source: Some(Box::new(e)), + }) +} + +fn strict_mask( + batch: &RecordBatch, + predicate: &Predicate, + schema_fields: &[DataField], + batch_fields: &[DataField], +) -> Result { + match predicate { + Predicate::AlwaysTrue => Ok(BooleanArray::from(vec![true; batch.num_rows()])), + Predicate::AlwaysFalse => Ok(BooleanArray::from(vec![false; batch.num_rows()])), + Predicate::And(children) => fold_masks(batch, children, schema_fields, batch_fields, true), + Predicate::Or(children) => fold_masks(batch, children, schema_fields, batch_fields, false), + Predicate::Not(inner) => { + let mask = strict_mask(batch, inner, schema_fields, batch_fields)?; + kleene(not(&mask)) + } + Predicate::Leaf { + index, + op, + literals, + .. + } => { + let field = schema_fields.get(*index).ok_or_else(|| { + unsupported(format!( + "query-auth filter references unknown field #{index}" + )) + })?; + let position = batch_fields + .iter() + .position(|f| f.id() == field.id() && f.name() == field.name()) + .ok_or_else(|| { + unsupported(format!( + "query-auth filter field `{}` missing from read", + field.name() + )) + })?; + let column = canonicalize_nan(batch.column(position)); + strict_leaf_mask(&column, field.data_type(), *op, literals) + } + } +} + +/// Replace every NaN in a float column with the canonical (positive) NaN. +/// +/// Arrow's total ordering sorts a NEGATIVE NaN below every finite value, so +/// `f < 0` would admit it; Java's `Float`/`Double.compare` makes every NaN the +/// greatest. Authorization filters only — ordinary pushdown keeps Arrow +/// semantics. Signed zero already agrees. +fn canonicalize_nan(column: &ArrayRef) -> ArrayRef { + match column.data_type() { + arrow_schema::DataType::Float32 => { + let values = column.as_any().downcast_ref::(); + match values { + Some(values) if values.iter().any(|v| v.is_some_and(f32::is_nan)) => { + Arc::new(values.unary::<_, arrow_array::types::Float32Type>(|v| { + if v.is_nan() { + f32::NAN + } else { + v + } + })) as ArrayRef + } + _ => Arc::clone(column), + } + } + arrow_schema::DataType::Float64 => { + let values = column.as_any().downcast_ref::(); + match values { + Some(values) if values.iter().any(|v| v.is_some_and(f64::is_nan)) => { + Arc::new(values.unary::<_, arrow_array::types::Float64Type>(|v| { + if v.is_nan() { + f64::NAN + } else { + v + } + })) as ArrayRef + } + _ => Arc::clone(column), + } + } + _ => Arc::clone(column), + } +} + +fn fold_masks( + batch: &RecordBatch, + children: &[Predicate], + schema_fields: &[DataField], + batch_fields: &[DataField], + use_and: bool, +) -> Result { + let mut combined: Option = None; + for child in children { + let mask = strict_mask(batch, child, schema_fields, batch_fields)?; + combined = Some(match combined { + Some(existing) if use_and => kleene(and_kleene(&existing, &mask))?, + Some(existing) => kleene(or_kleene(&existing, &mask))?, + None => mask, + }); + } + combined.ok_or_else(|| unsupported("query-auth filter has an empty compound".to_string())) +} + +fn strict_leaf_mask( + column: &ArrayRef, + data_type: &DataType, + op: PredicateOperator, + literals: &[Datum], +) -> Result { + // Decimals compare by value across scales (`datum_cmp`), which no Arrow + // scalar expresses — `literal_scalar_for_arrow_filter` returns `None` for + // them by design, so without this every decimal policy would error. The + // exact evaluator keeps nulls, so Kleene combination is unchanged. + if matches!(column.data_type(), arrow_schema::DataType::Decimal128(_, _)) + && !matches!(op, PredicateOperator::IsNull | PredicateOperator::IsNotNull) + { + return kleene(evaluate_decimal_leaf(column, op, literals)); + } + let scalar = |literal: &Datum| -> Result> { + literal_scalar_for_arrow_filter(literal, data_type)?.ok_or_else(|| { + unsupported(format!( + "query-auth filter literal is not comparable to type {data_type:?}" + )) + }) + }; + match op { + PredicateOperator::IsNull => Ok(boolean_mask_from_predicate(column.len(), |row| { + column.is_null(row) + })), + PredicateOperator::IsNotNull => Ok(boolean_mask_from_predicate(column.len(), |row| { + column.is_valid(row) + })), + PredicateOperator::In | PredicateOperator::NotIn => { + // Kleene IN: OR of equalities; x NOT IN (..) = NOT(IN), nulls stay null. + let mut combined = BooleanArray::from(vec![false; column.len()]); + for literal in literals { + let eq = kleene(evaluate_column_predicate( + column, + &scalar(literal)?, + PredicateOperator::Eq, + ))?; + combined = kleene(or_kleene(&combined, &eq))?; + } + if matches!(op, PredicateOperator::NotIn) { + combined = kleene(not(&combined))?; + if literals.is_empty() { + // `x NOT IN ()` is true only for non-null rows. + combined = + boolean_mask_from_predicate(column.len(), |row| column.is_valid(row)); + } + } + Ok(combined) + } + PredicateOperator::Eq + | PredicateOperator::NotEq + | PredicateOperator::Lt + | PredicateOperator::LtEq + | PredicateOperator::Gt + | PredicateOperator::GtEq + | PredicateOperator::StartsWith + | PredicateOperator::EndsWith + | PredicateOperator::Contains + | PredicateOperator::Like => { + let literal = literals.first().ok_or_else(|| { + unsupported("query-auth filter comparison without literal".to_string()) + })?; + kleene(evaluate_column_predicate(column, &scalar(literal)?, op)) + } + PredicateOperator::Between | PredicateOperator::NotBetween => { + let (Some(low), Some(high)) = (literals.first(), literals.get(1)) else { + return Err(unsupported( + "query-auth BETWEEN filter without bounds".to_string(), + )); + }; + let lo = kleene(evaluate_column_predicate( + column, + &scalar(low)?, + PredicateOperator::GtEq, + ))?; + let hi = kleene(evaluate_column_predicate( + column, + &scalar(high)?, + PredicateOperator::LtEq, + ))?; + let between = kleene(and_kleene(&lo, &hi))?; + if matches!(op, PredicateOperator::NotBetween) { + kleene(not(&between)) + } else { + Ok(between) + } + } + } +} + +fn kleene( + result: std::result::Result, +) -> Result { + result.map_err(|e| Error::DataInvalid { + message: format!("failed to evaluate query-auth row filter: {e}"), + source: Some(Box::new(e)), + }) +} + +// ==================== Column masking ==================== + +fn mask_err(detail: impl std::fmt::Display) -> Error { + unsupported(format!("cannot parse query-auth column masking: {detail}")) +} + +/// Parse the auth response's `columnMasking` map (column name -> Java +/// `Transform` JSON) against `fields` (table-schema order). +pub(crate) fn parse_column_masking( + masking: &std::collections::HashMap, + fields: &[DataField], +) -> Result> { + let mut masks = Vec::with_capacity(masking.len()); + for (column, json) in masking { + let target = fields + .iter() + .position(|f| f.name() == column) + .ok_or_else(|| mask_err(format!("unknown field `{column}`")))?; + let transform = Transform::from_rest_json(json, fields)?; + // The masked value replaces the column in place, so a type-changing + // transform (`CAST(id AS STRING)` on an INT column) cannot be + // represented. Compared via the arrow type (nullability-agnostic). + if let Some(out) = mask_output_type(&transform, fields) { + let target_type = crate::arrow::paimon_type_to_arrow(fields[target].data_type())?; + if out != target_type { + return Err(mask_err(format!( + "masking `{column}` produces {out:?} but the column is {target_type:?}" + ))); + } + } + // A mask that can yield null on a NOT NULL column would leave the output + // batch's schema claiming non-nullable, letting engines fold + // `col IS [NOT] NULL` before the masked-predicate guard. Fail closed. + if !fields[target].data_type().is_nullable() && mask_can_be_null(&transform, fields) { + return Err(mask_err(format!( + "masking `{column}` can produce null but the column is NOT NULL" + ))); + } + masks.push(ColumnMask { + column: target, + transform, + }); + } + // Deterministic order regardless of map iteration. + masks.sort_by_key(|m| m.column); + + // Masks read their inputs from the RAW batch (like Java), so a mask + // referencing ANOTHER mask's target would copy its unmasked value out. + // Referencing your own target (`name := UPPER(name)`) stays valid. + let targets: HashSet = masks.iter().map(|m| m.column).collect(); + for mask in &masks { + let mut inputs = HashSet::new(); + mask.transform.collect_field_indices(&mut inputs); + if let Some(other) = inputs + .into_iter() + .find(|i| *i != mask.column && targets.contains(i)) + { + return Err(mask_err(format!( + "masking `{}` reads masked column `{}`, which would expose its raw value", + field_name(fields, mask.column), + field_name(fields, other) + ))); + } + } + Ok(masks) +} + +/// Whether a mask transform can produce a null value. +fn mask_can_be_null(transform: &Transform, fields: &[DataField]) -> bool { + let field_nullable = |index: &usize| fields[*index].data_type().is_nullable(); + let input_nullable = |inputs: &[TransformInput]| { + inputs.iter().any(|i| match i { + TransformInput::Literal(literal) => literal.is_none(), + TransformInput::Field(index) => field_nullable(index), + }) + }; + match transform { + Transform::Null => true, + Transform::FieldRef(index) | Transform::Cast(index, _) => field_nullable(index), + Transform::Upper(inputs) | Transform::Lower(inputs) | Transform::Concat(inputs) => { + input_nullable(inputs) + } + // CONCAT_WS skips null payloads, so only a null separator (the first + // input) can make the result null. + Transform::ConcatWs(inputs) => inputs.first().is_some_and(|sep| match sep { + TransformInput::Literal(literal) => literal.is_none(), + TransformInput::Field(index) => field_nullable(index), + }), + } +} + +/// Arrow output type of a mask transform, or `None` when it always matches the +/// target column (the `NULL` transform builds a null of the column's own type). +fn mask_output_type(transform: &Transform, fields: &[DataField]) -> Option { + let of = |index: &usize| crate::arrow::paimon_type_to_arrow(fields[*index].data_type()).ok(); + match transform { + Transform::Null => None, + Transform::FieldRef(index) => of(index), + Transform::Cast(_, to) => crate::arrow::paimon_type_to_arrow(to).ok(), + Transform::Upper(_) + | Transform::Lower(_) + | Transform::Concat(_) + | Transform::ConcatWs(_) => Some(arrow_schema::DataType::Utf8), + } +} + +/// Overwrite masked columns of `batch` (whose columns correspond 1:1 to +/// `batch_fields`). Masks whose target column is not in the batch are skipped +/// (Java parity); anything that cannot be evaluated is an error. +pub(crate) fn mask_batch( + batch: &RecordBatch, + masks: &[ColumnMask], + schema_fields: &[DataField], + batch_fields: &[DataField], +) -> Result { + use arrow_array::new_null_array; + + // Nothing to mask (e.g. a `COUNT(*)` read projects no columns); return the + // batch unchanged, preserving its row count even with zero columns. + if masks.is_empty() { + return Ok(batch.clone()); + } + + // All batch positions holding a given table-schema field (a projection may + // repeat a column, so every copy must be masked, not just the first). + let positions_of = |schema_index: usize| -> Result> { + let field = schema_fields.get(schema_index).ok_or_else(|| { + unsupported(format!( + "query-auth mask references unknown field #{schema_index}" + )) + })?; + // By field id alone: the read type may carry the column under another + // name (aliasing, or a rename seen from an older file schema), and + // matching the name too would skip the mask and emit the raw value. + Ok(batch_fields + .iter() + .enumerate() + .filter(|(_, f)| f.id() == field.id()) + .map(|(pos, _)| pos) + .collect()) + }; + let input_column = |schema_index: usize| -> Result { + positions_of(schema_index)? + .first() + .map(|pos| batch.column(*pos).clone()) + .ok_or_else(|| unsupported("query-auth mask input missing from read".to_string())) + }; + + let mut columns = batch.columns().to_vec(); + for mask in masks { + let targets = positions_of(mask.column)?; + // `masks` is already filtered to targets the caller projects, so a mask + // that resolves to no batch column means the read cannot enforce it — + // emitting the column unmasked would leak the raw value. + let Some(&first) = targets.first() else { + return Err(unsupported(format!( + "query-auth mask for column `{}` cannot be applied: the column is missing \ + from the read", + field_name(schema_fields, mask.column) + ))); + }; + let target_type = batch.schema().field(first).data_type().clone(); + let masked: ArrayRef = match &mask.transform { + Transform::Null => new_null_array(&target_type, batch.num_rows()), + Transform::FieldRef(index) => input_column(*index)?, + Transform::Cast(index, to) => { + let to_arrow = crate::arrow::paimon_type_to_arrow(to)?; + cast_masked(&input_column(*index)?, &to_arrow)? + } + Transform::Upper(inputs) => string_mask(batch, inputs, &input_column, |v| { + Some(v.first()?.as_ref().map(|s| s.to_uppercase())) + })?, + Transform::Lower(inputs) => string_mask(batch, inputs, &input_column, |v| { + Some(v.first()?.as_ref().map(|s| s.to_lowercase())) + })?, + // SQL semantics: CONCAT is null if any input is null; CONCAT_WS + // uses the first input as separator and skips null values. + Transform::Concat(inputs) => string_mask(batch, inputs, &input_column, |v| { + Some( + v.iter() + .cloned() + .collect::>>() + .map(|p| p.concat()), + ) + })?, + Transform::ConcatWs(inputs) => string_mask(batch, inputs, &input_column, |v| { + let (sep, rest) = v.split_first()?; + Some( + sep.as_ref() + .map(|sep| rest.iter().flatten().cloned().collect::>().join(sep)), + ) + })?, + }; + // Parse-time type checks guarantee a compatible type, so this only + // aligns arrow representations. Mask every copy of the target column. + let masked = cast_masked(&masked, &target_type)?; + for pos in targets { + columns[pos] = masked.clone(); + } + } + RecordBatch::try_new_with_options( + batch.schema(), + columns, + &arrow_array::RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(|e| Error::DataInvalid { + message: format!("failed to apply query-auth column masking: {e}"), + source: Some(Box::new(e)), + }) +} + +fn cast_masked(array: &ArrayRef, to: &arrow_schema::DataType) -> Result { + if array.data_type() == to { + return Ok(array.clone()); + } + arrow_cast::cast(array, to).map_err(|e| { + unsupported(format!( + "query-auth mask value of type {:?} cannot be cast to column type {to:?}: {e}", + array.data_type() + )) + }) +} + +/// Evaluate a string transform row by row. `combine` receives the resolved +/// inputs (None = SQL NULL) and returns the masked value; a `None` from +/// `combine` means the transform is malformed for this input arity. +fn string_mask( + batch: &RecordBatch, + inputs: &[TransformInput], + input_column: &dyn Fn(usize) -> Result, + combine: impl Fn(&[Option]) -> Option>, +) -> Result { + use arrow_array::{Array, StringArray}; + use std::sync::Arc; + + // Resolve field inputs once, as string arrays (None = literal slot). + let resolved: Vec> = inputs + .iter() + .map(|input| match input { + TransformInput::Literal(_) => Ok(None), + TransformInput::Field(index) => { + cast_masked(&input_column(*index)?, &arrow_schema::DataType::Utf8).map(Some) + } + }) + .collect::>>()?; + + let mut values: Vec> = Vec::with_capacity(batch.num_rows()); + for row in 0..batch.num_rows() { + let row_inputs = inputs + .iter() + .zip(&resolved) + .map(|(input, array)| match (input, array) { + (TransformInput::Literal(s), _) => Ok(s.clone()), + (TransformInput::Field(_), Some(array)) => { + let strings = array + .as_any() + .downcast_ref::() + .ok_or_else(|| unsupported("mask input is not a string".to_string()))?; + Ok((!strings.is_null(row)).then(|| strings.value(row).to_string())) + } + (TransformInput::Field(_), None) => unreachable!("field inputs are resolved above"), + }) + .collect::>>()?; + let value = combine(&row_inputs) + .ok_or_else(|| unsupported("query-auth string mask is malformed".to_string()))?; + values.push(value); + } + Ok(Arc::new(StringArray::from(values)) as ArrayRef) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::spec::{IntType, VarCharType}; + use arrow_array::{Int32Array, StringArray}; + use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; + use std::sync::Arc; + + fn fields() -> Vec { + vec![ + DataField::new(0, "id".to_string(), DataType::Int(IntType::new())), + DataField::new( + 1, + "name".to_string(), + DataType::VarChar(VarCharType::new(255).unwrap()), + ), + ] + } + + fn leaf_json(function: &str, field: &str, literals: &str) -> String { + format!( + r#"{{"kind":"LEAF","transform":{{"name":"FIELD_REF","fieldRef":{{"index":0,"name":"{field}","type":"INT"}}}},"function":"{function}","literals":{literals}}}"# + ) + } + + fn batch() -> RecordBatch { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, true), + ArrowField::new("name", ArrowDataType::Utf8, true), + ])); + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![Some(1), Some(2), None, Some(4)])), + Arc::new(StringArray::from(vec![ + Some("a"), + Some("b"), + Some("c"), + Some("d"), + ])), + ], + ) + .unwrap() + } + + #[test] + fn test_strict_filter_batch_filters_rows() { + let fields = fields(); + let filters = + parse_auth_filters(&[leaf_json("GREATER_THAN", "id", "[1]")], &fields).unwrap(); + let filtered = strict_filter_batch(&batch(), &filters, &fields, &fields).unwrap(); + // id > 1 keeps rows 2 and 4; the NULL row is excluded. + assert_eq!(filtered.num_rows(), 2); + } + + #[test] + fn test_strict_filter_matches_java_nan_ordering() { + use crate::spec::{DoubleType, PredicateBuilder}; + use arrow_array::Float64Array; + use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; + + // Java `Double.compare` canonicalizes NaN above every finite value, so + // `f < 0` must reject NaN — including a NEGATIVE NaN, which Arrow's + // total ordering would otherwise sort below finite values and admit. + let fields = vec![DataField::new( + 0, + "f".to_string(), + DataType::Double(DoubleType::new()), + )]; + let neg_nan = f64::from_bits(0xFFF8_0000_0000_0000); + let batch = RecordBatch::try_new( + std::sync::Arc::new(ArrowSchema::new(vec![ArrowField::new( + "f", + ArrowDataType::Float64, + true, + )])), + vec![std::sync::Arc::new(Float64Array::from(vec![ + neg_nan, + f64::NAN, + -1.0, + 1.0, + ]))], + ) + .unwrap(); + + let less = PredicateBuilder::new(&fields) + .less_than("f", crate::spec::Datum::Double(0.0)) + .unwrap(); + let filtered = strict_filter_batch(&batch, &[less], &fields, &fields).unwrap(); + assert_eq!( + filtered.num_rows(), + 1, + "only -1.0 may pass `f < 0`; neither NaN sign may" + ); + + let greater = PredicateBuilder::new(&fields) + .greater_than("f", crate::spec::Datum::Double(0.0)) + .unwrap(); + let filtered = strict_filter_batch(&batch, &[greater], &fields, &fields).unwrap(); + assert_eq!( + filtered.num_rows(), + 3, + "both NaNs and 1.0 pass `f > 0` (NaN is greatest, like Java)" + ); + } + + #[test] + fn test_strict_filter_not_excludes_nulls() { + let fields = fields(); + // NOT (id = 2): NULL rows must stay excluded (SQL three-valued logic). + let json = format!( + r#"{{"kind":"COMPOUND","function":"AND","children":[{}]}}"#, + leaf_json("NOT_EQUAL", "id", "[2]") + ); + let filters = parse_auth_filters(&[json], &fields).unwrap(); + let filtered = strict_filter_batch(&batch(), &filters, &fields, &fields).unwrap(); + assert_eq!( + filtered.num_rows(), + 2, + "rows 1 and 4 only, not the NULL row" + ); + } + + /// Java #7034 baseline: filter each supported literal type exactly. + #[test] + fn test_strict_filter_batch_typed_matrix() { + use crate::spec::{BigIntType, BooleanType, DoubleType, FloatType}; + use arrow_array::{BooleanArray, Float32Array, Float64Array, Int64Array}; + + let fields = vec![ + DataField::new(0, "id".to_string(), DataType::Int(IntType::new())), + DataField::new(1, "age".to_string(), DataType::BigInt(BigIntType::new())), + DataField::new(2, "salary".to_string(), DataType::Double(DoubleType::new())), + DataField::new( + 3, + "is_active".to_string(), + DataType::Boolean(BooleanType::new()), + ), + DataField::new(4, "score".to_string(), DataType::Float(FloatType::new())), + DataField::new( + 5, + "name".to_string(), + DataType::VarChar(VarCharType::new(255).unwrap()), + ), + ]; + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, true), + ArrowField::new("age", ArrowDataType::Int64, true), + ArrowField::new("salary", ArrowDataType::Float64, true), + ArrowField::new("is_active", ArrowDataType::Boolean, true), + ArrowField::new("score", ArrowDataType::Float32, true), + ArrowField::new("name", ArrowDataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4])), + Arc::new(Int64Array::from(vec![25, 30, 35, 28])), + Arc::new(Float64Array::from(vec![50000.0, 60000.0, 70000.0, 55000.0])), + Arc::new(BooleanArray::from(vec![true, false, true, true])), + Arc::new(Float32Array::from(vec![85.5, 90.0, 95.5, 88.0])), + Arc::new(StringArray::from(vec!["Alice", "Bob", "Charlie", "David"])), + ], + ) + .unwrap(); + fn typed_leaf(function: &str, field: &str, ftype: &str, literals: &str) -> String { + format!( + r#"{{"kind":"LEAF","transform":{{"name":"FIELD_REF","fieldRef":{{"index":0,"name":"{field}","type":"{ftype}"}}}},"function":"{function}","literals":{literals}}}"# + ) + } + // (filter, expected surviving ids) — mirrors Java MockRESTCatalogTest. + let cases: Vec<(String, Vec)> = vec![ + (typed_leaf("GREATER_THAN", "id", "INT", "[2]"), vec![3, 4]), + ( + typed_leaf("GREATER_OR_EQUAL", "age", "BIGINT", "[30]"), + vec![2, 3], + ), + ( + typed_leaf("GREATER_THAN", "salary", "DOUBLE", "[55000.0]"), + vec![2, 3], + ), + ( + typed_leaf("EQUAL", "is_active", "BOOLEAN", "[true]"), + vec![1, 3, 4], + ), + ( + typed_leaf("GREATER_OR_EQUAL", "score", "FLOAT", "[90.0]"), + vec![2, 3], + ), + ( + typed_leaf("EQUAL", "name", "STRING", "[\"Alice\"]"), + vec![1], + ), + ( + // Two predicates ANDed by the grant list semantics. + format!( + r#"{{"kind":"COMPOUND","function":"AND","children":[{},{}]}}"#, + typed_leaf("GREATER_OR_EQUAL", "age", "BIGINT", "[30]"), + typed_leaf("EQUAL", "is_active", "BOOLEAN", "[true]") + ), + vec![3], + ), + ]; + for (json, expected) in cases { + let filters = parse_auth_filters(std::slice::from_ref(&json), &fields).unwrap(); + let filtered = strict_filter_batch(&batch, &filters, &fields, &fields).unwrap(); + let ids = filtered + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + assert_eq!(ids, expected, "for {json}"); + } + } + + fn masking(column: &str, json: &str) -> std::collections::HashMap { + std::collections::HashMap::from([(column.to_string(), json.to_string())]) + } + + #[test] + fn test_reject_mask_reading_another_masked_column() { + // `name := UPPER(name)` is the normal self-reference and stays valid. + let fields = fields(); + assert!(parse_column_masking( + &masking( + "name", + r#"{"name":"UPPER","inputs":[{"index":1,"name":"name","type":"STRING"}]}"# + ), + &fields + ) + .is_ok()); + + // But a mask that reads ANOTHER masked column would copy that column's + // raw value out (masks read the unmasked batch), defeating its mask. + let both = std::collections::HashMap::from([ + ("id".to_string(), r#"{"name":"NULL"}"#.to_string()), + ( + "name".to_string(), + r#"{"name":"UPPER","inputs":[{"index":1,"name":"name","type":"STRING"}]}"# + .to_string(), + ), + ]); + assert!( + parse_column_masking(&both, &fields).is_ok(), + "unrelated masks are fine" + ); + + let cross = std::collections::HashMap::from([ + ("name".to_string(), r#"{"name":"NULL"}"#.to_string()), + ( + "alias".to_string(), + r#"{"name":"UPPER","inputs":[{"index":1,"name":"name","type":"STRING"}]}"# + .to_string(), + ), + ]); + let mut fields_with_alias = fields.clone(); + fields_with_alias.push(DataField::new( + 2, + "alias".to_string(), + DataType::VarChar(VarCharType::new(255).unwrap()), + )); + let Err(err) = parse_column_masking(&cross, &fields_with_alias) else { + panic!("a mask reading another masked column must fail closed"); + }; + assert!(err.to_string().contains("raw value"), "got: {err}"); + } + + #[test] + fn test_parse_column_masking() { + let fields = fields(); + let masks = parse_column_masking(&masking("name", r#"{"name":"NULL"}"#), &fields).unwrap(); + assert!(matches!(masks[0].transform, Transform::Null)); + assert_eq!(masks[0].column, 1); + + let upper = r#"{"name":"UPPER","inputs":[{"index":1,"name":"name","type":"STRING"}]}"#; + let masks = parse_column_masking(&masking("name", upper), &fields).unwrap(); + assert!(matches!(&masks[0].transform, Transform::Upper(inputs) if inputs.len() == 1)); + + // Unknown transform / unknown column / bad JSON: all fail closed. + for (column, json) in [ + ("name", r#"{"name":"ROT13"}"#), + ("missing", r#"{"name":"NULL"}"#), + ("name", "not json"), + ] { + assert!(parse_column_masking(&masking(column, json), &fields).is_err()); + } + } + + #[test] + fn test_mask_batch_null_and_string_transforms() { + use arrow_array::Array; + let fields = fields(); + + // NULL mask: the whole column becomes null. + let masks = parse_column_masking(&masking("name", r#"{"name":"NULL"}"#), &fields).unwrap(); + let masked = mask_batch(&batch(), &masks, &fields, &fields).unwrap(); + assert_eq!(masked.column(1).null_count(), 4); + assert_eq!(masked.column(0).null_count(), 1, "other columns untouched"); + + // CONCAT_WS("-", literal, field): "x-a", "x-b", ... + let concat = + r#"{"name":"CONCAT_WS","inputs":["-","x",{"index":1,"name":"name","type":"STRING"}]}"#; + let masks = parse_column_masking(&masking("name", concat), &fields).unwrap(); + let masked = mask_batch(&batch(), &masks, &fields, &fields).unwrap(); + let names = masked + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(names.value(0), "x-a"); + + // Masked column absent from the batch: fail closed. The caller filters + // masks to projected targets, so this means the read cannot enforce the + // mask — emitting the column unmasked would leak the raw value. + let one_col = batch().project(&[0]).unwrap(); + let one_field = vec![fields[0].clone()]; + let masks = parse_column_masking(&masking("name", r#"{"name":"NULL"}"#), &fields).unwrap(); + let Err(err) = mask_batch(&one_col, &masks, &fields, &one_field) else { + panic!("a mask whose target is missing from the read must fail closed"); + }; + assert!(err.to_string().contains("cannot be applied"), "got: {err}"); + } + + #[test] + fn test_reject_type_changing_cast_mask() { + let fields = fields(); + // CAST(id AS STRING) on an INT column changes type -> fail closed. + let cast = + r#"{"name":"CAST","fieldRef":{"index":0,"name":"id","type":"INT"},"type":"STRING"}"#; + assert!(parse_column_masking(&masking("id", cast), &fields).is_err()); + // A string transform on a non-string column also fails closed. + let upper = r#"{"name":"UPPER","inputs":[{"index":0,"name":"id","type":"INT"}]}"#; + assert!(parse_column_masking(&masking("id", upper), &fields).is_err()); + } + + #[test] + fn test_mask_batch_masks_every_duplicate_target() { + use arrow_array::Array; + let fields = fields(); + let masks = parse_column_masking(&masking("name", r#"{"name":"NULL"}"#), &fields).unwrap(); + // A projection that repeats the masked column: both copies must be masked. + let base = batch(); + let dup = base.project(&[1, 1]).unwrap(); + let dup_fields = vec![fields[1].clone(), fields[1].clone()]; + let masked = mask_batch(&dup, &masks, &fields, &dup_fields).unwrap(); + assert_eq!(masked.column(0).null_count(), 4); + assert_eq!(masked.column(1).null_count(), 4); + } + + #[test] + fn test_implicit_system_fields_are_scope_checked() { + // The audit schema prepends `rowkind` (and may prepend + // `_SEQUENCE_NUMBER`) on top of the read type, so neither reaches the + // auth request via the projection and both need a by-name check. + let grant = QueryAuthGrant::new( + Vec::new(), + Vec::new(), + Some(HashSet::from([0])), + Some(HashSet::from( + [crate::spec::ROW_KIND_FIELD_NAME.to_string()], + )), + GrantBinding::default(), + ); + assert!(grant + .check_system_scope_by_name(&[crate::spec::ROW_KIND_FIELD_NAME]) + .is_ok()); + assert!(grant + .check_system_scope_by_name(&[ + crate::spec::ROW_KIND_FIELD_NAME, + crate::spec::SEQUENCE_NUMBER_FIELD_NAME, + ]) + .is_err()); + // An unscoped grant covers everything. + let all = QueryAuthGrant::new(Vec::new(), Vec::new(), None, None, GrantBinding::default()); + assert!(all + .check_system_scope_by_name(&[crate::spec::SEQUENCE_NUMBER_FIELD_NAME]) + .is_ok()); + } + + #[test] + fn test_whitespace_only_filter_fails_closed() { + use crate::spec::{DataType, IntType}; + let fields = vec![DataField::new( + 0, + "id".to_string(), + DataType::Int(IntType::new()), + )]; + // Java skips only length-0 entries, so " " reaches the parser and throws. + // Dropping it here would leave a grant with no filter at all. + assert!(parse_auth_filters(&[" ".to_string()], &fields).is_err()); + assert!(parse_auth_filters(&["\n".to_string()], &fields).is_err()); + // A genuinely empty entry is still skipped (Java parity). + assert_eq!( + parse_auth_filters(&[String::new()], &fields).unwrap().len(), + 0 + ); + } + + #[test] + fn test_system_scope_does_not_make_a_grant_restricted() { + // `is_unrestricted` answers "did the server restrict, or did the client + // scope columns". The system scope answers a different question, so + // letting it gate this made every write/build path on a query-auth table + // fail: `authorize_unrestricted_read` would never see an unrestricted + // grant. + let scoped_system = QueryAuthGrant::new( + Vec::new(), + Vec::new(), + None, + Some(HashSet::new()), + GrantBinding::default(), + ); + assert!( + scoped_system.is_unrestricted(), + "an empty system scope with no rules is still unrestricted" + ); + assert!(!scoped_system.has_server_restrictions()); + + // Internal raw reads carry no system scoping at all. + let internal = + QueryAuthGrant::new(Vec::new(), Vec::new(), None, None, GrantBinding::default()); + assert!(internal.is_unrestricted()); + } + + #[test] + fn test_full_projection_is_not_blanket_system_approval() { + use crate::spec::{DataType, IntType}; + // `select = None` means "every table column", never "every reserved + // system field": the request only carries the system names the read + // needs, so anything else was never shown to the server. + let grant = QueryAuthGrant::new( + Vec::new(), + Vec::new(), + None, + Some(HashSet::new()), + GrantBinding::default(), + ); + let row_id = DataField::new( + crate::spec::ROW_ID_FIELD_ID, + crate::spec::ROW_ID_FIELD_NAME.to_string(), + DataType::Int(IntType::new()), + ); + assert!(grant.check_system_scope(&[row_id]).is_err()); + assert!(grant + .check_system_scope_by_name(&[crate::spec::ROW_KIND_FIELD_NAME]) + .is_err()); + } + + #[test] + fn test_system_field_scope_is_enforced_by_name() { + use crate::spec::{DataType, IntType}; + let row_id = DataField::new( + crate::spec::ROW_ID_FIELD_ID, + crate::spec::ROW_ID_FIELD_NAME.to_string(), + DataType::Int(IntType::new()), + ); + let seq = DataField::new( + crate::spec::SEQUENCE_NUMBER_FIELD_ID, + crate::spec::SEQUENCE_NUMBER_FIELD_NAME.to_string(), + DataType::Int(IntType::new()), + ); + // Authorized for _ROW_ID only: system fields have no table index, so the + // positional scope cannot cover them and they need their own check. + let grant = QueryAuthGrant::new( + Vec::new(), + Vec::new(), + Some(HashSet::from([0])), + Some(HashSet::from([crate::spec::ROW_ID_FIELD_NAME.to_string()])), + GrantBinding::default(), + ); + assert!(grant + .check_system_scope(std::slice::from_ref(&row_id)) + .is_ok()); + assert!(grant.check_system_scope(&[seq]).is_err()); + // An unscoped grant covers everything. + let all = QueryAuthGrant::new(Vec::new(), Vec::new(), None, None, GrantBinding::default()); + assert!(all.check_system_scope(&[row_id]).is_ok()); + } + + #[test] + fn test_canonical_projection_system_and_dropped_fields() { + use crate::spec::{DataType, IntType}; + let fields = vec![DataField::new( + 0, + "id".to_string(), + DataType::Int(IntType::new()), + )]; + + // Produced by the reader, so it maps to no index and is allowed. + let row_id = DataField::new( + crate::spec::ROW_ID_FIELD_ID, + crate::spec::ROW_ID_FIELD_NAME.to_string(), + DataType::Int(IntType::new()), + ); + assert_eq!( + canonical_projection(&fields, std::slice::from_ref(&row_id)).unwrap(), + Vec::::new() + ); + + // Matches neither an id nor a name, but is still readable by field id + // from older files — must not pass as a system field. + let dropped = DataField::new(7, "dropped".to_string(), DataType::Int(IntType::new())); + assert!( + canonical_projection(&fields, &[dropped]).is_err(), + "a dropped column must not be waved through as a system field" + ); + + // A system id under another name is not a system field. + let forged = DataField::new( + crate::spec::ROW_ID_FIELD_ID, + "not_row_id".to_string(), + DataType::Int(IntType::new()), + ); + assert!(canonical_projection(&fields, &[forged]).is_err()); + } + + #[test] + fn test_grant_authorized_column_scope() { + // A grant scoped to a subset is not globally unrestricted and rejects + // columns outside the approved set. + let grant = QueryAuthGrant::new( + Vec::new(), + Vec::new(), + Some(HashSet::from([0])), + None, + GrantBinding::default(), + ); + assert!(!grant.is_unrestricted()); + assert!(grant.authorizes_columns([0])); + assert!(!grant.authorizes_columns([1])); + // `None` (all columns) with no filter/mask is fully unrestricted. + let all = QueryAuthGrant::new(Vec::new(), Vec::new(), None, None, GrantBinding::default()); + assert!(all.is_unrestricted()); + assert!(all.authorizes_columns([0, 1, 99])); + } +} diff --git a/crates/paimon/src/table/read_builder.rs b/crates/paimon/src/table/read_builder.rs index de432a5cd..5322e4ace 100644 --- a/crates/paimon/src/table/read_builder.rs +++ b/crates/paimon/src/table/read_builder.rs @@ -210,6 +210,8 @@ impl<'a> ReadBuilder<'a> { } /// Set row ID ranges `[from, to]` (inclusive) for filtering in data evolution mode. + /// Slicing by physical row id selects rows by `_ROW_ID` without any + /// predicate, so it must reach the auth request like a filter on it would. pub fn with_row_ranges(&mut self, ranges: Vec) -> &mut Self { match &mut self.0 { ReadBuilderKind::Paimon(builder) => { @@ -291,6 +293,13 @@ struct PaimonReadBuilder<'a> { limit: Option, row_ranges: Option>, case_sensitive: bool, + /// Table-schema indices referenced by the full caller filter (before it is + /// split into partition/data conjuncts). The query-auth gates check these + /// against the grant fetched at plan time. + filter_columns: HashSet, + /// System columns (`_ROW_ID`, …) the caller filter referenced. They have no + /// table index, and row-id extraction strips the leaf before planning. + filter_system_names: HashSet, } impl<'a> PaimonReadBuilder<'a> { @@ -303,6 +312,8 @@ impl<'a> PaimonReadBuilder<'a> { limit: None, row_ranges: None, case_sensitive: true, + filter_columns: HashSet::new(), + filter_system_names: HashSet::new(), } } @@ -362,6 +373,20 @@ impl<'a> PaimonReadBuilder<'a> { /// primary-key merge reads push key conjuncts below the merge and enforce /// the full predicate with an exact post-merge residual filter. pub fn with_filter(&mut self, filter: Predicate) -> &mut Self { + // Capture the FULL predicate's columns before it is split into + // partition/data conjuncts, so the guards see a masked partition key + // that would otherwise be pruned on its raw value. + self.filter_columns.clear(); + filter.collect_user_leaf_field_indices(&mut self.filter_columns); + // System columns too, and BEFORE `try_extract_row_id_ranges` strips the + // `_ROW_ID` leaf: it selects rows by that column without ever appearing + // in the read type, so the server must still get to refuse it. + let mut names = std::collections::HashSet::new(); + filter.collect_leaf_column_names(&mut names); + self.filter_system_names = names + .into_iter() + .filter(|n| crate::table::query_auth::is_reserved_system_field_name(n.as_str())) + .collect(); self.filter = normalize_filter(self.table, filter); self.try_extract_row_id_ranges(); self @@ -380,6 +405,8 @@ impl<'a> PaimonReadBuilder<'a> { } /// Set row ID ranges `[from, to]` (inclusive) for filtering in data evolution mode. + /// Slicing by physical row id selects rows by `_ROW_ID` without any + /// predicate, so it must reach the auth request like a filter on it would. pub fn with_row_ranges(&mut self, ranges: Vec) -> &mut Self { self.row_ranges = if ranges.is_empty() { None @@ -434,6 +461,9 @@ impl<'a> PaimonReadBuilder<'a> { let partition_filter = self.filter.partition_predicate.clone().map(|pred| { PartitionFilter::from_predicate(pred, &self.table.schema().partition_fields()) }); + // The grant's field ids are folded into the projection in + // `TableScan::plan`, where it has been fetched; reading an empty grant + // here was a fail-open row-filter bypass. let read_type = self.resolve_read_type().unwrap_or(None); TableScan::new( self.table, @@ -448,14 +478,47 @@ impl<'a> PaimonReadBuilder<'a> { &self.filter.data_predicates, self.table.schema().fields(), )) + .with_query_auth_scope( + self.filter_columns.clone(), + self.projected_schema_indices(), + self.projected_system_field_names(), + ) + } + + /// Table-schema indices of the projected columns (`None` = all). + /// Projected system fields (`_ROW_ID`, …). `projected_schema_indices` drops + /// them for lack of an index, but Java's `select` includes them. + fn projected_system_field_names(&self) -> Vec { + crate::table::query_auth::projected_system_field_names( + self.resolve_read_type().ok().flatten().as_deref(), + &self.filter_system_names, + self.row_ranges.is_some(), + ) + } + + fn projected_schema_indices(&self) -> Option> { + // Resolve names too: a `with_projection` selection lives in + // `projection_names`, and scoping to all columns would deny a user + // authorized for exactly the subset. Unresolvable falls back to full + // scope; `new_read` reports the error. + self.resolve_read_type().ok().flatten().map(|fields| { + fields + .iter() + .filter_map(|f| { + self.table + .schema() + .fields() + .iter() + .position(|s| s.id() == f.id()) + }) + .collect() + }) } /// Create a table read for consuming splits (e.g. from a scan plan). pub fn new_read(&self) -> Result> { - // Fail closed at read construction so bindings that short-circuit before - // `to_arrow` (e.g. an empty-splits fast path) can't bypass the guard. - let core_options = self.table.schema.core_options(); - core_options.ensure_read_authorized()?; + // Enforced in `TableRead::to_arrow` off the grant planning stamped on + // the splits, so read construction needs no gate. let read_type = match self.resolve_read_type()? { None => self.table.schema.fields().to_vec(), Some(fields) => fields, @@ -813,31 +876,275 @@ mod tests { .unwrap() } - #[test] - fn test_read_fails_closed_when_query_auth_enabled() { + /// A real split carrying no query-auth grant (an unauthorized read path). + fn ungranted_split() -> crate::table::DataSplit { + DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path("file:/tmp/bucket-0".to_string()) + .with_total_buckets(1) + .with_data_files(vec![test_data_file("data.parquet", 4, 1)]) + .build() + .unwrap() + } + + #[tokio::test] + async fn test_read_fails_closed_when_query_auth_enabled() { let table = query_auth_table(); - // `new_read` fails closed, so bindings that short-circuit before `to_arrow` can't bypass. - let err = table.new_read_builder().new_read().unwrap_err(); + // Enforcement is at `to_arrow` off the split grant: a read whose splits + // carry no grant (never authorized by planning) must fail closed, so + // bindings that short-circuit can't bypass. + let read = table.new_read_builder().new_read().unwrap(); + let Err(err) = read.to_arrow(&[ungranted_split()]) else { + panic!("a query-auth read without a stamped grant must fail closed"); + }; assert!( matches!(err, crate::Error::Unsupported { ref message } if message.contains("query-auth.enabled")), "building a read for a query-auth.enabled table must fail closed" ); } - #[test] - fn test_dynamic_option_cannot_disable_query_auth() { + #[tokio::test] + async fn test_dynamic_option_cannot_disable_query_auth() { // Copying the table with the option off must not weaken a stored `true`. let table = query_auth_table().copy_with_options(HashMap::from([( "query-auth.enabled".to_string(), "false".to_string(), )])); - let err = table.new_read_builder().new_read().unwrap_err(); + let read = table.new_read_builder().new_read().unwrap(); + let Err(err) = read.to_arrow(&[ungranted_split()]) else { + panic!("a dynamic override must not disable query-auth"); + }; assert!( matches!(err, crate::Error::Unsupported { ref message } if message.contains("query-auth.enabled")), "a dynamic override must not disable query-auth" ); } + #[tokio::test] + async fn test_query_auth_filtered_grant_filters_rows_exactly() { + let tempdir = tempdir().unwrap(); + let table_path = local_file_path(tempdir.path()); + let bucket_dir = tempdir.path().join("bucket-0"); + fs::create_dir_all(&bucket_dir).unwrap(); + + let parquet_path = bucket_dir.join("data.parquet"); + write_int_parquet_file( + &parquet_path, + vec![("id", vec![1, 2, 3, 4]), ("value", vec![1, 2, 20, 30])], + None, + ); + let file_size = fs::metadata(&parquet_path).unwrap().len() as i64; + + let file_io = FileIOBuilder::new("file").build().unwrap(); + let table_schema = TableSchema::new( + 0, + &Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option("query-auth.enabled", "true") + .build() + .unwrap(), + ); + let table = Table::new( + file_io, + Identifier::new("default", "t"), + table_path, + table_schema, + None, + ); + // Grant: the user may only see rows with value >= 10. The filter column + // is NOT in the projection, so the read must fetch it and project it away. + // The grant is threaded on the split (as scan planning would stamp it). + let auth_filter = PredicateBuilder::new(table.schema().fields()) + .greater_or_equal("value", crate::spec::Datum::Int(10)) + .unwrap(); + let grant = std::sync::Arc::new(crate::table::query_auth::QueryAuthGrant::new( + vec![auth_filter], + Vec::new(), + None, + None, + crate::table::query_auth::GrantBinding::of(&table), + )); + + let split = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(local_file_path(&bucket_dir)) + .with_total_buckets(1) + .with_data_files(vec![test_data_file("data.parquet", 4, file_size)]) + .build() + .unwrap() + .with_query_auth_grant(Some(grant)); + + let read = TableRead::new(&table, vec![table.schema().fields()[0].clone()], Vec::new()); + let batches = read + .to_arrow(&[split]) + .unwrap() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(collect_int_column(&batches, "id"), vec![3, 4]); + // The filter column must not leak into the output schema. + assert_eq!(batches[0].num_columns(), 1); + } + + #[tokio::test] + async fn test_query_auth_masked_grant_masks_and_guards_predicates() { + use crate::table::query_auth::{parse_column_masking, QueryAuthGrant}; + use arrow_array::Array; + + let tempdir = tempdir().unwrap(); + let table_path = local_file_path(tempdir.path()); + let bucket_dir = tempdir.path().join("bucket-0"); + fs::create_dir_all(&bucket_dir).unwrap(); + let parquet_path = bucket_dir.join("data.parquet"); + write_int_parquet_file( + &parquet_path, + vec![("id", vec![1, 2, 3, 4]), ("value", vec![1, 2, 20, 30])], + None, + ); + let file_size = fs::metadata(&parquet_path).unwrap().len() as i64; + + let file_io = FileIOBuilder::new("file").build().unwrap(); + let table_schema = TableSchema::new( + 0, + &Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("value", DataType::Int(IntType::new())) + .option("query-auth.enabled", "true") + .build() + .unwrap(), + ); + let table = Table::new( + file_io, + Identifier::new("default", "t"), + table_path, + table_schema, + None, + ); + // Grant: filter on raw `value` >= 10, then mask `value` with NULL. + let auth_filter = PredicateBuilder::new(table.schema().fields()) + .greater_or_equal("value", crate::spec::Datum::Int(10)) + .unwrap(); + let masks = parse_column_masking( + &std::collections::HashMap::from([( + "value".to_string(), + r#"{"name":"NULL"}"#.to_string(), + )]), + table.schema().fields(), + ) + .unwrap(); + let grant = std::sync::Arc::new(QueryAuthGrant::new( + vec![auth_filter], + masks, + None, + None, + crate::table::query_auth::GrantBinding::of(&table), + )); + + let split = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path(local_file_path(&bucket_dir)) + .with_total_buckets(1) + .with_data_files(vec![test_data_file("data.parquet", 4, file_size)]) + .build() + .unwrap() + .with_query_auth_grant(Some(grant)); + + // Filter runs on raw values, then the surviving rows are masked. + let read = TableRead::new(&table, table.schema().fields().to_vec(), Vec::new()); + let batches = read + .to_arrow(std::slice::from_ref(&split)) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(collect_int_column(&batches, "id"), vec![3, 4]); + assert_eq!(batches[0].column(1).null_count(), 2, "value masked to NULL"); + + // A caller predicate on the masked column must fail closed (oracle guard). + let caller_filter = PredicateBuilder::new(table.schema().fields()) + .equal("value", crate::spec::Datum::Int(20)) + .unwrap(); + let read = TableRead::new( + &table, + table.schema().fields().to_vec(), + vec![caller_filter], + ); + let Err(err) = read.to_arrow(&[split]) else { + panic!("filtering on a masked column must fail closed"); + }; + assert!(err.to_string().contains("masked column"), "got: {err}"); + } + + #[test] + fn test_row_id_filter_reaches_the_auth_select() { + // `try_extract_row_id_ranges` strips the `_ROW_ID` leaf from the + // predicates during `with_filter`, so collecting system names from the + // surviving predicates afterwards would never see it. + let table = crate::table::query_auth_table(); + let mut rb = table.new_read_builder(); + let filter = crate::spec::Predicate::Leaf { + index: 0, + column: crate::spec::ROW_ID_FIELD_NAME.to_string(), + data_type: crate::spec::DataType::BigInt(crate::spec::BigIntType::new()), + op: crate::spec::PredicateOperator::GtEq, + literals: vec![crate::spec::Datum::Long(5)], + }; + rb.with_filter(filter); + let names = paimon_builder(&rb).projected_system_field_names(); + assert!( + names.iter().any(|n| n == crate::spec::ROW_ID_FIELD_NAME), + "a _ROW_ID filter must reach the auth select, got {names:?}" + ); + + // An explicit row-range slice selects by `_ROW_ID` with no predicate. + let mut rb2 = table.new_read_builder(); + rb2.with_row_ranges(vec![crate::table::RowRange::new(0, 10)]); + let names2 = paimon_builder(&rb2).projected_system_field_names(); + assert!(names2.iter().any(|n| n == crate::spec::ROW_ID_FIELD_NAME)); + } + + #[tokio::test] + async fn test_query_auth_scope_rejects_unauthorized_column() { + use crate::table::query_auth::QueryAuthGrant; + // A grant scoped to no columns must fail closed when the read projects + // `id` — the scope check runs in `to_arrow` before any data is read + // (and also at plan time; see the rest_catalog integration test). + let table = query_auth_table(); + let grant = std::sync::Arc::new(QueryAuthGrant::new( + Vec::new(), + Vec::new(), + Some(HashSet::new()), + None, + crate::table::query_auth::GrantBinding::of(&table), + )); + let split = DataSplitBuilder::new() + .with_snapshot(1) + .with_partition(BinaryRow::new(0)) + .with_bucket(0) + .with_bucket_path("/tmp/does-not-matter".to_string()) + .with_total_buckets(1) + .with_data_files(vec![test_data_file("data.parquet", 4, 1)]) + .build() + .unwrap() + .with_query_auth_grant(Some(grant)); + let read = TableRead::new(&table, vec![table.schema().fields()[0].clone()], Vec::new()); + let Err(err) = read.to_arrow(&[split]) else { + panic!("reading an unauthorized column must fail closed"); + }; + assert!( + err.to_string().contains("outside the authorized set"), + "got: {err}" + ); + } + #[test] fn test_projected_read_field_ids_uses_projection_ids() { let read_type = vec![DataField::new( diff --git a/crates/paimon/src/table/rest_env.rs b/crates/paimon/src/table/rest_env.rs index 4f2a9df76..5dd7e797c 100644 --- a/crates/paimon/src/table/rest_env.rs +++ b/crates/paimon/src/table/rest_env.rs @@ -71,6 +71,12 @@ impl RESTEnv { } } + /// The REST catalog's table UUID: stable across renames and unique across + /// catalogs, unlike an identifier or the per-table schema counter. + pub(crate) fn uuid(&self) -> &str { + &self.uuid + } + #[cfg(test)] fn has_local_cache(&self) -> bool { self.local_cache.is_some() @@ -188,6 +194,15 @@ impl RESTEnv { )) } + /// Fetch the per-user row filter and column masking for this table. + /// Mirrors Java `CatalogEnvironment.tableQueryAuth()`. + pub(crate) async fn table_query_auth( + &self, + select: Option>, + ) -> Result { + self.api.auth_table_query(&self.identifier, select).await + } + /// Create a `RESTSnapshotCommit` from this environment. pub fn snapshot_commit(&self) -> Arc { Arc::new(RESTSnapshotCommit::new( diff --git a/crates/paimon/src/table/source.rs b/crates/paimon/src/table/source.rs index 933cb4415..6743c74a1 100644 --- a/crates/paimon/src/table/source.rs +++ b/crates/paimon/src/table/source.rs @@ -20,8 +20,10 @@ //! Reference: [org.apache.paimon.table.source](https://github.com/apache/paimon/blob/master/paimon-core/src/main/java/org/apache/paimon/table/source/). use crate::spec::{BinaryRow, DataFileMeta}; +use crate::table::query_auth::QueryAuthGrant; use crate::table::stats_filter::group_by_overlapping_row_id; use serde::{Deserialize, Serialize}; +use std::sync::Arc; fn is_vector_store_file_name(file_name: &str) -> bool { file_name.to_ascii_lowercase().contains(".vector.") @@ -471,7 +473,7 @@ impl PartitionBucket { /// Input split for reading: partition + bucket + list of data files and optional deletion files. /// /// Reference: [org.apache.paimon.table.source.DataSplit](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java) -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize)] pub struct DataSplit { snapshot_id: i64, partition: BinaryRow, @@ -487,6 +489,51 @@ pub struct DataSplit { /// physical rows are exactly its logical rows (modulo deletion files). /// Mirrors Java `DataSplit#rawConvertible`. raw_convertible: bool, + /// The grant this split must be read under, stamped by scan planning. + /// Runtime-only, mirroring Java `QueryAuthSplit(split, authResult)`: on the + /// split so `to_arrow` enforces exactly the grant its plan fetched. + #[serde(skip)] + query_auth_grant: Option>, +} + +/// Hand-written so serializing a split planned under a grant FAILS instead of +/// silently dropping it — no serde format carries it. The emitted shape is +/// identical to the derived one. +impl Serialize for DataSplit { + fn serialize( + &self, + serializer: S, + ) -> std::result::Result { + use serde::ser::Error as _; + self.ensure_no_restricted_grant("serialize") + .map_err(S::Error::custom)?; + + #[derive(Serialize)] + struct Wire<'a> { + snapshot_id: &'a i64, + partition: &'a BinaryRow, + bucket: &'a i32, + bucket_path: &'a String, + total_buckets: &'a i32, + data_files: &'a Vec, + data_deletion_files: &'a Option>>, + row_ranges: &'a Option>, + raw_convertible: &'a bool, + } + + Wire { + snapshot_id: &self.snapshot_id, + partition: &self.partition, + bucket: &self.bucket, + bucket_path: &self.bucket_path, + total_buckets: &self.total_buckets, + data_files: &self.data_files, + data_deletion_files: &self.data_deletion_files, + row_ranges: &self.row_ranges, + raw_convertible: &self.raw_convertible, + } + .serialize(serializer) + } } impl DataSplit { @@ -539,6 +586,37 @@ impl DataSplit { .all(|file| file.level != 0 && file.delete_row_count == Some(0)) } + /// The query-auth grant this split is read under, if any (see the field doc). + pub(crate) fn query_auth_grant(&self) -> Option<&Arc> { + self.query_auth_grant.as_ref() + } + + /// Whether this split is planned under a grant that filters rows or masks + /// columns. Engines must not publish its raw manifest statistics, which + /// describe the data before enforcement. + pub fn has_restricted_query_auth_grant(&self) -> bool { + self.query_auth_grant + .as_ref() + .is_some_and(|g| g.has_server_restrictions()) + } + + /// Whether this split carries any grant that is not fully unrestricted. + /// Transport and re-authorization use this rather than + /// [`Self::has_restricted_query_auth_grant`]: a scope-only grant distorts no + /// statistics, but no wire format carries it either. + pub fn carries_query_auth_restriction(&self) -> bool { + self.query_auth_grant + .as_ref() + .is_some_and(|g| !g.is_unrestricted()) + } + + /// Stamp the query-auth grant this split must be read under. Called by scan + /// planning and write-path authorizers on every emitted split. + pub(crate) fn with_query_auth_grant(mut self, grant: Option>) -> Self { + self.query_auth_grant = grant; + self + } + /// Returns the deletion file for the data file at the given index, if any. `None` at that index means no deletion file. pub fn deletion_file_for_data_file_index(&self, index: usize) -> Option<&DeletionFile> { self.data_deletion_files @@ -599,6 +677,17 @@ impl DataSplit { /// /// Reference: [DataSplit.mergedRowCount()](https://github.com/apache/paimon/blob/release-1.3/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java#L133) pub fn merged_row_count(&self) -> Option { + // A row filter drops rows after the scan, so the manifest count is not + // the logical output count. Java's `QueryAuthSplit.mergedRowCount()` + // returns empty for the same reason; `Plan::row_counts_exact` only + // covers callers that go through the plan. + if self + .query_auth_grant + .as_ref() + .is_some_and(|g| g.has_row_filter()) + { + return None; + } if !self.row_counts_known() { return None; } @@ -673,7 +762,27 @@ impl DataSplit { /// Serialize the DataSplit fields to Java `DataSplit#serialize` (version 8) binary. /// Byte-compatible with `compatibility/datasplit-v8`. Row ranges are not part of the v8 /// format; `serialize_split_v1` wraps a row-range split as an `IndexedSplit` instead. + /// Fail closed when a split carrying a grant is about to leave the process: + /// no wire format (native bytes, the Java `SplitSerializer` frame, serde) + /// carries it. `what` names the attempted operation. + fn ensure_no_restricted_grant(&self, what: &str) -> crate::Result<()> { + match &self.query_auth_grant { + // Any stamped grant, unrestricted included: the receiver would get + // a grant-less split and fail closed anyway, so refuse at the + // boundary where the cause is still visible. + Some(_) => Err(crate::Error::Unsupported { + message: format!( + "cannot {what} a split planned under a query-auth grant: no wire format \ + carries the grant, so the receiver could not enforce the row filter, \ + column masking, or column scope it was planned under" + ), + }), + _ => Ok(()), + } + } + pub fn serialize(&self) -> crate::Result> { + self.ensure_no_restricted_grant("serialize")?; let mut out = Vec::new(); out.extend_from_slice(&SPLIT_MAGIC.to_be_bytes()); out.extend_from_slice(&SPLIT_VERSION.to_be_bytes()); @@ -824,6 +933,10 @@ impl DataSplit { /// `IndexedSplit` (type 3) wrapping the DataSplit body plus the ranges. Byte-compatible with /// `compatibility/split-v1-data` / `split-v1-indexed`. pub fn serialize_split_v1(&self) -> crate::Result> { + // The wire format has no place for the query-auth grant (Java carries it + // out of band in `QueryAuthSplit`), so a restricted split would cross the + // boundary as a plain split and be read raw. Fail closed instead. + self.ensure_no_restricted_grant("serialize")?; let mut out = Vec::new(); out.extend_from_slice(&SPLIT_SER_MAGIC.to_be_bytes()); out.extend_from_slice(&SPLIT_SER_VERSION.to_be_bytes()); @@ -1255,6 +1368,7 @@ impl DataSplitBuilder { data_deletion_files: self.data_deletion_files, row_ranges: self.row_ranges, raw_convertible: self.raw_convertible, + query_auth_grant: None, }) } } @@ -1273,15 +1387,60 @@ impl Default for DataSplitBuilder { #[derive(Debug)] pub struct Plan { splits: Vec, + /// False when a residual pass (e.g. a query-auth row filter) drops rows + /// after the scan, so split row counts overcount the read output. + row_counts_exact: bool, + /// Whether this plan was made under a server-restricted grant. Recorded on + /// the plan, not inferred from its splits: a fully pruned plan has none to + /// carry the grant but its scan metadata is just as pre-enforcement. + query_auth_restricted: bool, } impl Plan { pub fn new(splits: Vec) -> Self { - Self { splits } + Self { + splits, + row_counts_exact: true, + query_auth_restricted: false, + } + } + pub(crate) fn with_inexact_row_counts(mut self) -> Self { + self.row_counts_exact = false; + self + } + + /// Stamp the grant onto every split so [`crate::table::TableRead::to_arrow`] + /// enforces exactly the one this plan fetched. A no-op for `None` (not a + /// query-auth table). + pub(crate) fn stamp_query_auth_grant(mut self, grant: Option>) -> Self { + self.query_auth_restricted = grant.as_ref().is_some_and(|g| g.has_server_restrictions()); + if grant.is_some() { + self.splits = self + .splits + .into_iter() + .map(|s| s.with_query_auth_grant(grant.clone())) + .collect(); + } + self } pub fn splits(&self) -> &[DataSplit] { &self.splits } + /// Whether split row counts exactly reflect the rows a read will produce. + pub fn row_counts_exact(&self) -> bool { + self.row_counts_exact + } + + /// Whether this plan was made under a row filter or column masking. + /// + /// Recorded on the plan, unlike + /// [`DataSplit::has_restricted_query_auth_grant`] which inspects one split's + /// grant: a fully pruned plan has no split to carry it, yet its scan + /// metadata (split/file counts, pruning counters) is just as + /// pre-enforcement, so engines must not publish it either. + pub fn planned_under_restricted_grant(&self) -> bool { + self.query_auth_restricted + } } #[cfg(test)] @@ -1327,6 +1486,50 @@ mod tests { .unwrap() } + #[test] + fn test_query_auth_grant_split_cannot_be_serialized() { + use crate::spec::{DataType, IntType, PredicateBuilder}; + use crate::table::query_auth::QueryAuthGrant; + + let fields = vec![crate::spec::DataField::new( + 0, + "id".to_string(), + DataType::Int(IntType::new()), + )]; + let filter = PredicateBuilder::new(&fields) + .greater_than("id", crate::spec::Datum::Int(1)) + .unwrap(); + let restricted = Arc::new(QueryAuthGrant::new( + vec![filter], + Vec::new(), + None, + None, + crate::table::query_auth::GrantBinding::default(), + )); + let plain = split(vec![file("a", 1, None)], true); + let guarded = plain.clone().with_query_auth_grant(Some(restricted)); + + // No wire format carries the grant, so every serialization path must + // refuse rather than emit a plain split the receiver would read raw. + assert!(guarded.serialize().is_err(), "native bytes"); + assert!(guarded.serialize_split_v1().is_err(), "Java frame"); + assert!( + serde_json::to_vec(&guarded).is_err(), + "serde (pickle, JSON)" + ); + + // Only an unstamped split serializes. An unrestricted grant is refused + // too: the receiver would get a grant-less split and fail closed when + // reading it, so rejecting here reports the real cause at the boundary. + assert!(plain.serialize().is_ok()); + assert!(serde_json::to_vec(&plain).is_ok()); + let unrestricted = plain + .clone() + .with_query_auth_grant(Some(Arc::new(QueryAuthGrant::default()))); + assert!(unrestricted.serialize().is_err()); + assert!(serde_json::to_vec(&unrestricted).is_err()); + } + #[test] fn data_split_serde_json_round_trip() { let split = DataSplit::builder() diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index 809e45d9f..a52479298 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -156,6 +156,7 @@ impl TableCommit { filter_committed: bool, ) -> Result<()> { self.table.ensure_not_branch_reference_for_write()?; + self.table.authorize_unrestricted_write().await?; if commit_messages.is_empty() { return Ok(()); @@ -199,6 +200,7 @@ impl TableCommit { commit_identifier: i64, ) -> Result<()> { self.table.ensure_not_branch_reference_for_write()?; + self.table.authorize_unrestricted_write().await?; if commit_messages.is_empty() { return Ok(()); @@ -270,6 +272,7 @@ impl TableCommit { filter_committed: bool, ) -> Result<()> { self.table.ensure_not_branch_reference_for_write()?; + self.table.authorize_unrestricted_write().await?; if commit_messages.is_empty() && static_partitions.is_none() { return Ok(()); @@ -519,6 +522,7 @@ impl TableCommit { filter_committed: bool, ) -> Result<()> { self.table.ensure_not_branch_reference_for_write()?; + self.table.authorize_unrestricted_write().await?; if partitions.is_empty() { return Ok(()); @@ -566,6 +570,7 @@ impl TableCommit { commit_identifier: i64, ) -> Result<()> { self.table.ensure_not_branch_reference_for_write()?; + self.table.authorize_unrestricted_write().await?; if partitions.is_empty() { return Err(crate::Error::DataInvalid { @@ -597,6 +602,7 @@ impl TableCommit { filter_committed: bool, ) -> Result<()> { self.table.ensure_not_branch_reference_for_write()?; + self.table.authorize_unrestricted_write().await?; self.try_commit( CommitEntriesPlan::Overwrite { @@ -621,6 +627,9 @@ impl TableCommit { /// files or storage errors are ignored so abort cleanup never masks the /// original write failure. pub async fn abort(&self, commit_messages: &[CommitMessage]) -> Result<()> { + // No query-auth gate: abort only deletes files the caller just wrote. + // A restricted commit is rejected after `prepare_commit` wrote them, so + // gating here would strand them instead of cleaning up. self.table.ensure_not_branch_reference_for_write()?; for message in commit_messages { diff --git a/crates/paimon/src/table/table_read.rs b/crates/paimon/src/table/table_read.rs index 0a179b49d..6ebce829f 100644 --- a/crates/paimon/src/table/table_read.rs +++ b/crates/paimon/src/table/table_read.rs @@ -21,7 +21,7 @@ use super::format_table_read::FormatTableRead; use super::incremental_scan::{IncrementalPlan, IncrementalScanMode, IncrementalSplit}; use super::kv_file_reader::{KeyValueFileReader, KeyValueReadConfig}; use super::read_builder::split_scan_predicates; -use super::{ArrowRecordBatchStream, Table}; +use super::{query_auth, ArrowRecordBatchStream, Table}; use crate::arrow::build_target_arrow_schema; use crate::spec::{ BigIntType, CoreOptions, DataField, DataType, MergeEngine, Predicate, TinyIntType, @@ -108,6 +108,15 @@ impl<'a> TableRead<'a> { } } + /// A read-level row limit that must be applied after materialization + /// (format tables); Paimon reads push their limit to scan planning. + fn read_limit(&self) -> Option { + match &self.0 { + TableReadKind::Paimon(_) => None, + TableReadKind::Format(read) => read.limit(), + } + } + /// Set a filter predicate. pub fn with_filter(self, filter: Predicate) -> Self { match self.0 { @@ -133,7 +142,84 @@ impl<'a> TableRead<'a> { } /// Returns an [`ArrowRecordBatchStream`]. + /// + /// Query-auth is enforced off the grant stamped on the splits, never a + /// shared slot on `Table`, so it cannot leak from a concurrent query or a + /// write rewrite. A restricted grant is applied to the output stream; an + /// unrestricted one reads raw; no grant on a `query-auth.enabled` table + /// means an unauthorized path — fail closed. pub fn to_arrow(&self, data_splits: &[DataSplit]) -> crate::Result { + let Some(grant) = self.resolve_split_grant(data_splits)? else { + return self.to_arrow_dispatch(data_splits); + }; + query_auth::authorize_read( + &grant, + self.table(), + self.read_type(), + self.data_predicates(), + &[], + )?; + if grant.is_unrestricted() { + self.to_arrow_dispatch(data_splits) + } else { + self.to_arrow_auth_enforced(data_splits, grant) + } + } + + /// The single grant every split was planned under, or `None` for a + /// non-query-auth table. + /// + /// A disagreement fails closed: one split's permissive grant must not relax + /// the read of splits planned under a stricter one. So does a + /// `query-auth.enabled` table whose splits carry no grant at all. + fn resolve_split_grant<'s>( + &self, + data_splits: impl IntoIterator, + ) -> crate::Result>> { + let mut grant: Option<&Arc> = None; + let mut saw_ungranted = false; + let mut empty = true; + for split in data_splits { + empty = false; + match (split.query_auth_grant(), &grant) { + (Some(found), None) => grant = Some(found), + (Some(found), Some(seen)) if found != *seen => { + return Err(crate::Error::Unsupported { + message: "reading splits planned under different query-auth grants is \ + not supported; re-plan the scan" + .to_string(), + }); + } + (Some(_), Some(_)) => {} + // A grant found later must not retroactively authorize an + // earlier grant-less split, so record it and check after the + // loop — matching on what was seen so far is order-dependent. + (None, _) => saw_ungranted = true, + } + } + if saw_ungranted && grant.is_some() { + return Err(crate::Error::Unsupported { + message: "a query-auth split was mixed with an unauthorized split; \ + re-plan the scan" + .to_string(), + }); + } + match grant { + Some(grant) => Ok(Some(Arc::clone(grant))), + // Nothing to read: an empty table or a fully pruned scan produces no + // rows, so there is nothing to leak (an authorized plan legitimately + // has no split to stamp). + None if empty => Ok(None), + // Real splits with no grant: either not a query-auth table, or an + // unauthorized read path — fail closed. + None => self.table().ensure_read_without_grant().map(|()| None), + } + } + + fn to_arrow_dispatch( + &self, + data_splits: &[DataSplit], + ) -> crate::Result { match &self.0 { TableReadKind::Paimon(read) => read.to_arrow(data_splits), TableReadKind::Format(read) => read.to_arrow(data_splits), @@ -148,7 +234,7 @@ impl<'a> TableRead<'a> { &self, plan: &IncrementalPlan, ) -> crate::Result { - self.ensure_query_auth_allowed()?; + self.ensure_incremental_plan_authorized(plan, &[])?; plan.validate()?; match &self.0 { TableReadKind::Paimon(read) => read.to_incremental_arrow(plan), @@ -158,6 +244,40 @@ impl<'a> TableRead<'a> { } } + /// Incremental and audit-log reads consume their splits inside + /// `PaimonTableRead`, which cannot apply the filter / masking pass, so a + /// restricted grant fails closed here rather than return raw rows. + fn ensure_incremental_plan_authorized( + &self, + plan: &IncrementalPlan, + implicit_system_fields: &[&str], + ) -> crate::Result<()> { + // `all_data_splits` (not `data_splits`) so a Diff plan's pairs are seen: + // `data_splits` drops them, which would present no splits at all. + let Some(grant) = self.resolve_split_grant(plan.all_data_splits())? else { + return Ok(()); + }; + + if grant.has_server_restrictions() { + return Err(crate::Error::Unsupported { + message: "reading a query-auth row filter / column masking grant on an \ + incremental or audit-log scan is not supported" + .to_string(), + }); + } + + // A scoped grant still needs checking, and nothing downstream on this + // path does it (the batch path does, in `to_arrow_auth_enforced`). + query_auth::authorize_read( + &grant, + self.table(), + self.read_type(), + self.data_predicates(), + implicit_system_fields, + ) + .map(|_| ()) + } + /// Returns an audit-log [`ArrowRecordBatchStream`] for an incremental plan. /// /// Output schema is `rowkind` (+ optional `_SEQUENCE_NUMBER`) followed by @@ -168,7 +288,14 @@ impl<'a> TableRead<'a> { &self, plan: &IncrementalPlan, ) -> crate::Result { - self.ensure_query_auth_allowed()?; + // The audit schema prepends these on top of the read type, so neither + // reached the auth request via the projection. Check only what this read + // actually emits: `_SEQUENCE_NUMBER` is conditional. + let mut implicit = vec![ROW_KIND_FIELD_NAME]; + if audit_sequence_number_enabled(self.table()) { + implicit.push(SEQUENCE_NUMBER_FIELD_NAME); + } + self.ensure_incremental_plan_authorized(plan, &implicit)?; plan.validate()?; match &self.0 { TableReadKind::Paimon(read) => read.to_audit_log_arrow(plan), @@ -178,8 +305,111 @@ impl<'a> TableRead<'a> { } } - fn ensure_query_auth_allowed(&self) -> crate::Result<()> { - CoreOptions::new(self.table().schema().options()).ensure_read_authorized() + /// Read the union of the projection, filter columns and mask inputs; per + /// batch drop non-matching rows (on raw values, like Java), overwrite masked + /// columns, then project back to the requested columns. + fn to_arrow_auth_enforced( + &self, + data_splits: &[DataSplit], + grant: std::sync::Arc, + ) -> crate::Result { + use futures::StreamExt; + + let table = self.table(); + // The grant's filter and mask indices are POSITIONAL in the schema they + // were parsed against, so enforcing them on another schema would bind + // them to different columns. Refuse a grant issued for a different one. + let projected = query_auth::authorize_read( + &grant, + table, + self.read_type(), + self.data_predicates(), + &[], + )?; + let schema_fields = table.schema().fields().to_vec(); + + // Only masks whose target is caller-projected matter (others are + // projected away); keeping just those avoids masking a target that was + // added to the physical read solely because a filter references it. + // + // Comparing against `projected` is equivalent to matching field ids ONLY + // because `authorize_read` above rejected any non-canonical read type: + // an id present in the read type therefore always has its schema index + // here. Moving that call after this point would silently drop masks. + let masks: Vec = grant + .masks() + .iter() + .filter(|m| projected.contains(&m.column)) + .cloned() + .collect(); + + // Widen the physical read with filter columns and the applied masks' + // inputs, so both are always available to the in-memory pass. + let mut referenced = std::collections::HashSet::new(); + grant + .filters() + .iter() + .for_each(|f| f.collect_leaf_field_indices(&mut referenced)); + masks + .iter() + .for_each(|m| m.transform.collect_field_indices(&mut referenced)); + let mut physical = self.read_type().to_vec(); + for index in referenced { + let field = schema_fields + .get(index) + .ok_or_else(|| crate::Error::Unsupported { + message: format!("query-auth grant references unknown field #{index}"), + })?; + if !physical.iter().any(|f| f.id() == field.id()) { + physical.push(field.clone()); + } + } + + let projected_columns = self.read_type().len(); + let filters = grant.filters().to_vec(); + // The inner read must NOT apply the caller's limit: it would cap rows + // before the auth filter. Read everything, then truncate the output. + let caller_limit = self.read_limit(); + let inner = TableRead::new(table, physical.clone(), self.data_predicates().to_vec()); + let stream = inner.to_arrow_dispatch(data_splits)?.map(move |batch| { + let batch = batch?; + let filtered = + query_auth::strict_filter_batch(&batch, &filters, &schema_fields, &physical)?; + let masked = query_auth::mask_batch(&filtered, &masks, &schema_fields, &physical)?; + masked + .project(&(0..projected_columns).collect::>()) + .map_err(|e| crate::Error::DataInvalid { + message: format!("failed to re-project query-auth batch: {e}"), + source: Some(Box::new(e)), + }) + }); + match caller_limit { + None => Ok(Box::pin(stream)), + // `unfold` stops as soon as the cap is reached (state `emitted >= + // limit`) without polling the inner stream again, so a read error in + // a later batch can't surface after the limit is satisfied. + Some(limit) => Ok(Box::pin(futures::stream::unfold( + (Box::pin(stream), 0usize), + move |(mut inner, emitted)| async move { + if emitted >= limit { + return None; + } + match inner.next().await? { + Err(e) => Some((Err(e), (inner, limit))), + Ok(batch) => { + let remaining = limit - emitted; + let batch = if batch.num_rows() > remaining { + batch.slice(0, remaining) + } else { + batch + }; + let emitted = emitted + batch.num_rows(); + Some((Ok(batch), (inner, emitted))) + } + } + }, + ))), + } } } @@ -642,12 +872,12 @@ impl<'a> PaimonTableRead<'a> { reader.read(splits) } - /// Returns an [`ArrowRecordBatchStream`]. + /// Returns an [`ArrowRecordBatchStream`]. Query-auth (fail-closed + row + /// filter + masking) is enforced by the outer [`TableRead::to_arrow`] off + /// the grant stamped on the splits. pub fn to_arrow(&self, data_splits: &[DataSplit]) -> crate::Result { let has_primary_keys = !self.table.schema.primary_keys().is_empty(); let core_options = self.table.schema.core_options(); - // Fail closed for a direct `TableRead` (bypassing `ReadBuilder::new_read`). - core_options.ensure_read_authorized()?; let merge_engine = core_options.merge_engine()?; // Route supported PK merge engines through the split-aware reader. @@ -849,7 +1079,7 @@ fn audit_schema_for_read_type( build_target_arrow_schema(&fields) } -fn audit_sequence_number_enabled(table: &Table) -> bool { +pub(crate) fn audit_sequence_number_enabled(table: &Table) -> bool { table .schema() .options() @@ -1491,20 +1721,226 @@ mod tests { // Bypass `ReadBuilder` by constructing `TableRead` directly; the `to_arrow` guard // still fails closed. let read = TableRead::new(&table, table.schema.fields().to_vec(), Vec::new()); + // Real splits with no stamped grant: the read would return raw rows. + let ungranted = split(vec![file("a", 5, Some(0))], true); assert!( matches!( - read.to_arrow(&[]), + read.to_arrow(&[ungranted]), Err(crate::Error::Unsupported { ref message }) if message.contains("query-auth.enabled") ), "directly-constructed read of a query-auth.enabled table must fail closed" ); + // An empty slice reads no rows, so it is allowed (an authorized plan may + // legitimately produce no splits). + assert!(read.to_arrow(&[]).is_ok()); + } + + #[test] + fn test_grant_from_another_table_cannot_authorize_a_raw_read() { + // `authorize_rewrite_splits` is public, so a caller holding two tables + // can stamp one's unrestricted grant onto the other's splits. Both sit + // at schema id 0 — a per-table counter — hence the identity binding. + let table = query_auth_table(); + let other = { + let mut t = query_auth_table(); + t.location = "/tmp/test-query-auth-table-other".to_string(); + t + }; + assert_eq!(table.schema.id(), other.schema.id()); + + let foreign = Arc::new(query_auth::QueryAuthGrant::new( + Vec::new(), + Vec::new(), + None, + None, + query_auth::GrantBinding::of(&other), + )); + // Unrestricted grants used to skip straight to the raw dispatch, so the + // binding has to be checked before that branch. + assert!(foreign.is_unrestricted()); + let granted = split(vec![file("a", 5, Some(0))], true).with_query_auth_grant(Some(foreign)); + let read = TableRead::new(&table, table.schema.fields().to_vec(), Vec::new()); + assert!( + matches!( + read.to_arrow(&[granted]), + Err(crate::Error::Unsupported { ref message }) + if message.contains("different table or schema") + ), + "a grant bound to another table must not authorize a raw read" + ); + } + + #[test] + fn test_incremental_read_rejects_noncanonical_read_type() { + use super::super::incremental_scan::{IncrementalPlan, IncrementalScanMode}; + use std::collections::HashSet; + let table = query_auth_table(); + let real = table.schema.fields()[0].clone(); + // Authorized id under another column's name: scoped by id, read by + // name. Both paths must go through `canonical_projection`. + let forged = crate::spec::DataField::new( + real.id(), + "not_the_real_name".to_string(), + real.data_type().clone(), + ); + let grant = Arc::new(query_auth::QueryAuthGrant::new( + Vec::new(), + Vec::new(), + Some(HashSet::from([0])), + None, + query_auth::GrantBinding::of(&table), + )); + let granted = split(vec![file("a", 5, Some(0))], true).with_query_auth_grant(Some(grant)); + let plan = IncrementalPlan::new( + IncrementalScanMode::Delta, + vec![IncrementalSplit::Data(granted)], + ); + let read = TableRead::new(&table, vec![forged], Vec::new()); + assert!( + matches!( + read.to_incremental_arrow(&plan), + Err(crate::Error::Unsupported { ref message }) + if message.contains("is not a column of this table") + ), + "a read type pairing an authorized id with another column's name must fail closed" + ); + } + + #[test] + fn test_incremental_read_enforces_column_scope() { + use super::super::incremental_scan::{IncrementalPlan, IncrementalScanMode}; + use std::collections::HashSet; + + // A column-scoped grant has no filter or mask, so the filter check + // passes it through; without an explicit scope check a wider read + // would return unauthorized columns raw. + let table = query_auth_table(); + let fields = table.schema.fields().to_vec(); + let read = TableRead::new(&table, fields.clone(), Vec::new()); + let plan_for = |authorized: HashSet| { + let grant = Arc::new(query_auth::QueryAuthGrant::new( + Vec::new(), + Vec::new(), + Some(authorized), + None, + query_auth::GrantBinding::of(&table), + )); + let granted = + split(vec![file("a", 5, Some(0))], true).with_query_auth_grant(Some(grant)); + IncrementalPlan::new( + IncrementalScanMode::Delta, + vec![IncrementalSplit::Data(granted)], + ) + }; + + let out_of_scope = plan_for(HashSet::new()); + assert!( + matches!( + read.to_incremental_arrow(&out_of_scope), + Err(crate::Error::Unsupported { ref message }) + if message.contains("outside the authorized set") + ), + "incremental read wider than the grant's column scope must fail closed" + ); + + // Reading exactly the authorized columns is allowed. + let in_scope = plan_for((0..fields.len()).collect()); + assert!(read + .ensure_incremental_plan_authorized(&in_scope, &[]) + .is_ok()); + } + + #[test] + fn test_noncanonical_read_type_fails_closed() { + use std::collections::HashSet; + + // Authorization and mask selection resolve by field id, but the physical + // read resolves by name. A read type pairing an authorized id with + // another column's name would read that other column and skip its mask. + let table = query_auth_table(); + let schema_field = table.schema.fields()[0].clone(); + let forged = crate::spec::DataField::new( + schema_field.id(), + "not_the_real_name".to_string(), + schema_field.data_type().clone(), + ); + let grant = Arc::new(query_auth::QueryAuthGrant::new( + Vec::new(), + Vec::new(), + Some(HashSet::from([0])), + None, + query_auth::GrantBinding::of(&table), + )); + let granted = split(vec![file("a", 5, Some(0))], true).with_query_auth_grant(Some(grant)); + + // Direction A: a known id carrying another column's name. + let read = TableRead::new(&table, vec![forged], Vec::new()); + let Err(err) = read.to_arrow(std::slice::from_ref(&granted)) else { + panic!("a read type with a mismatched id/name pair must fail closed"); + }; + assert!( + err.to_string().contains("not a column of this table"), + "got: {err}" + ); + + // Direction B: an UNKNOWN id borrowing a real column's name. The + // physical read resolves by name, so this would read the real column + // while scoping and mask selection (both by id) see an unrelated field. + let forged_id = crate::spec::DataField::new( + 9999, + schema_field.name().to_string(), + schema_field.data_type().clone(), + ); + let read = TableRead::new(&table, vec![forged_id], Vec::new()); + let Err(err) = read.to_arrow(&[granted]) else { + panic!("an unknown id borrowing a real column name must fail closed"); + }; + assert!( + err.to_string().contains("not a column of this table"), + "got: {err}" + ); + } + + #[test] + fn test_mixed_granted_and_ungranted_splits_fail_closed_in_both_orders() { + use std::collections::HashSet; + + let table = query_auth_table(); + let read = TableRead::new(&table, table.schema.fields().to_vec(), Vec::new()); + let grant = Arc::new(query_auth::QueryAuthGrant::new( + Vec::new(), + Vec::new(), + Some(HashSet::from([0])), + None, + query_auth::GrantBinding::of(&table), + )); + let granted = + split(vec![file("a", 5, Some(0))], true).with_query_auth_grant(Some(grant.clone())); + let ungranted = split(vec![file("b", 5, Some(0))], true); + + // Order must not decide the verdict: a grant found later must never + // retroactively authorize an earlier grant-less split. + for slice in [ + vec![granted.clone(), ungranted.clone()], + vec![ungranted, granted], + ] { + let Err(err) = read.to_arrow(&slice) else { + panic!("mixing a granted and an ungranted split must fail closed"); + }; + assert!( + err.to_string().contains("mixed with an unauthorized split"), + "got: {err}" + ); + } } #[test] fn test_direct_incremental_read_fails_closed_when_query_auth_enabled() { let table = query_auth_table(); let read = TableRead::new(&table, table.schema.fields().to_vec(), Vec::new()); - let plan = IncrementalPlan::new(IncrementalScanMode::Delta, Vec::new()); + // Real splits carrying no grant: an unauthorized read path. + let ungranted = IncrementalSplit::Data(split(vec![file("a", 5, Some(0))], true)); + let plan = IncrementalPlan::new(IncrementalScanMode::Delta, vec![ungranted]); assert!( matches!( read.to_incremental_arrow(&plan), @@ -1512,13 +1948,19 @@ mod tests { ), "directly-constructed incremental read of a query-auth.enabled table must fail closed" ); + // An empty plan reads no rows, so it has nothing to leak (same rule as + // the batch path); authorization is per-split, not per-table. + let empty = IncrementalPlan::new(IncrementalScanMode::Delta, Vec::new()); + assert!(read.to_incremental_arrow(&empty).is_ok()); } #[test] fn test_direct_audit_log_read_fails_closed_when_query_auth_enabled() { let table = query_auth_table(); let read = TableRead::new(&table, table.schema.fields().to_vec(), Vec::new()); - let plan = IncrementalPlan::new(IncrementalScanMode::Delta, Vec::new()); + // Real splits carrying no grant: an unauthorized read path. + let ungranted = IncrementalSplit::Data(split(vec![file("a", 5, Some(0))], true)); + let plan = IncrementalPlan::new(IncrementalScanMode::Delta, vec![ungranted]); assert!( matches!( read.to_audit_log_arrow(&plan), @@ -1526,6 +1968,10 @@ mod tests { ), "directly-constructed audit-log read of a query-auth.enabled table must fail closed" ); + // An empty plan reads no rows, so it has nothing to leak (same rule as + // the batch path); authorization is per-split, not per-table. + let empty = IncrementalPlan::new(IncrementalScanMode::Delta, Vec::new()); + assert!(read.to_audit_log_arrow(&empty).is_ok()); } #[test] diff --git a/crates/paimon/src/table/table_scan.rs b/crates/paimon/src/table/table_scan.rs index 632b6ee35..13b4be103 100644 --- a/crates/paimon/src/table/table_scan.rs +++ b/crates/paimon/src/table/table_scan.rs @@ -44,6 +44,7 @@ use crate::table::bin_pack::split_for_batch; use crate::table::merge_tree_split_generator::{ merge_tree_split_for_batch, KeyComparator, SplitGroup, }; +use crate::table::query_auth::QueryAuthGrant; use crate::table::schema_manager::SchemaManager; use crate::table::source::{ any_range_overlaps_file, intersect_ranges_with_file, merge_row_ranges, DataSplit, @@ -1027,6 +1028,26 @@ impl<'a> TableScan<'a> { } } + pub(super) fn with_query_auth_scope( + self, + filter_columns: HashSet, + projected: Option>, + system_select: Vec, + ) -> Self { + match self.0 { + TableScanKind::Paimon(scan) => Self(TableScanKind::Paimon(scan.with_query_auth_scope( + filter_columns, + projected, + system_select, + ))), + TableScanKind::Format(scan) => Self(TableScanKind::Format(scan.with_query_auth_scope( + filter_columns, + projected, + system_select, + ))), + } + } + pub async fn plan(&self) -> crate::Result { match &self.0 { TableScanKind::Paimon(scan) => scan.plan().await, @@ -1091,7 +1112,7 @@ impl<'a> TableScan<'a> { fn apply_limit_pushdown(&self, splits: Vec) -> Vec { match &self.0 { TableScanKind::Paimon(scan) => scan.apply_limit_pushdown(splits), - TableScanKind::Format(scan) => scan.apply_limit_pushdown(splits), + TableScanKind::Format(scan) => scan.apply_limit_pushdown(splits, false), } } } @@ -1117,6 +1138,12 @@ struct PaimonTableScan<'a> { /// the complete file set. Normal read scans leave this as `false`. scan_all_files: bool, projected_read_field_ids: Option>, + /// Filter/projection columns for the query-auth scope check, evaluated + /// against the live grant at plan time (see `ensure_query_auth_allowed`). + query_auth_filter_columns: HashSet, + query_auth_projected: Option>, + /// Sent in `select`, but has no index to scope. + query_auth_system_select: Vec, } impl<'a> PaimonTableScan<'a> { @@ -1138,6 +1165,9 @@ impl<'a> PaimonTableScan<'a> { row_range_optimization_disabled: false, scan_all_files: false, projected_read_field_ids: None, + query_auth_filter_columns: HashSet::new(), + query_auth_projected: None, + query_auth_system_select: Vec::new(), } } @@ -1155,7 +1185,18 @@ impl<'a> PaimonTableScan<'a> { /// /// This replaces any existing row_ranges. Typically used to inject /// results from global index lookups (e.g. full-text search). + /// Slicing by physical row id selects rows by `_ROW_ID` with no predicate, + /// so it must reach the auth request even when set after `new_scan` fixed + /// the scope (`ReadBuilder::with_row_ranges` does the same on its side). pub fn with_row_ranges(mut self, ranges: Vec) -> Self { + let row_id = crate::spec::ROW_ID_FIELD_NAME.to_string(); + // Only while ranges actually apply: clearing them must not leave a + // stale system-column request that could get an otherwise valid read + // rejected. + self.query_auth_system_select.retain(|n| n != &row_id); + if !ranges.is_empty() { + self.query_auth_system_select.push(row_id); + } self.row_ranges = if ranges.is_empty() { None } else { @@ -1178,6 +1219,18 @@ impl<'a> PaimonTableScan<'a> { self } + pub(super) fn with_query_auth_scope( + mut self, + filter_columns: HashSet, + projected: Option>, + system_select: Vec, + ) -> Self { + self.query_auth_filter_columns = filter_columns; + self.query_auth_projected = projected; + self.query_auth_system_select = system_select; + self + } + /// Plan the full scan: resolve snapshot (via options or latest), then read manifests and build DataSplits. /// /// Time travel is resolved from table options: @@ -1193,49 +1246,154 @@ impl<'a> PaimonTableScan<'a> { /// Reference: [TimeTravelUtil.tryTravelToSnapshot](https://github.com/apache/paimon/blob/master/paimon-core/src/main/java/org/apache/paimon/table/source/snapshot/TimeTravelUtil.java) /// for `scan.version`; the strict selectors mirror Java's typed /// `scan.snapshot-id` / `scan.tag-name` handling. + /// Fail closed when the snapshot the plan will read was written under a + /// different schema than the grant was parsed against. + /// + /// The grant is fetched before the snapshot is resolved, so a schema-changing + /// commit landing in between would leave the rules bound to this copy's + /// schema while the plan consumes the new one — a drop-and-re-add in that + /// window binds them to unrelated field ids. + fn ensure_grant_matches_snapshot( + &self, + grant: Option<&Arc>, + snapshot: &Snapshot, + ) -> crate::Result<()> { + let Some(grant) = grant else { + return Ok(()); + }; + // Only a snapshot NEWER than this copy is a problem. A schema-only ALTER + // writes `schema-N` without committing a snapshot, so the table schema + // is routinely ahead of the latest data snapshot — the reader evolves + // those older files by field id, and the rules were parsed against the + // newer schema they are expressed in. + if !grant.has_server_restrictions() || snapshot.schema_id() <= self.table.schema().id() { + return Ok(()); + } + Err(crate::Error::Unsupported { + message: "the snapshot being planned was written under a newer schema than the \ + query-auth grant was issued for; re-plan the scan" + .to_string(), + }) + } + pub async fn plan(&self) -> crate::Result { - self.ensure_query_auth_allowed()?; - let data_evolution_read_field_ids = self.projected_read_field_ids()?; + let grant = self.ensure_query_auth_allowed().await?; + let has_row_filter = grant.as_deref().is_some_and(|g| g.has_row_filter()); + let data_evolution_read_field_ids = self.auth_widened_read_field_ids(grant.as_deref()); let snapshot = match super::time_travel::resolve_snapshot(self.table).await? { Some(snapshot) => snapshot, - None => return Ok(Plan::new(Vec::new())), + None => return Ok(self.finalize_plan(Plan::new(Vec::new()), grant.as_ref())), }; - self.plan_snapshot(snapshot, data_evolution_read_field_ids.as_ref(), None) - .await + self.ensure_grant_matches_snapshot(grant.as_ref(), &snapshot)?; + self.plan_snapshot( + snapshot, + data_evolution_read_field_ids.as_ref(), + None, + has_row_filter, + ) + .await + .map(|plan| self.finalize_plan(plan, grant.as_ref())) } /// Plan the full scan and return metadata-pruning trace counters. pub async fn plan_with_trace(&self) -> crate::Result<(Plan, ScanTrace)> { - self.ensure_query_auth_allowed()?; + let grant = self.ensure_query_auth_allowed().await?; + let has_row_filter = grant.as_deref().is_some_and(|g| g.has_row_filter()); let mut trace = ScanTrace { limit: self.limit, ..Default::default() }; - let data_evolution_read_field_ids = self.projected_read_field_ids()?; + let data_evolution_read_field_ids = self.auth_widened_read_field_ids(grant.as_deref()); let snapshot = match super::time_travel::resolve_snapshot(self.table).await? { Some(snapshot) => snapshot, - None => return Ok((Plan::new(Vec::new()), trace)), + None => { + return Ok(( + self.finalize_plan(Plan::new(Vec::new()), grant.as_ref()), + trace, + )) + } }; + self.ensure_grant_matches_snapshot(grant.as_ref(), &snapshot)?; trace.snapshot_id = Some(snapshot.id()); let plan = self .plan_snapshot( snapshot, data_evolution_read_field_ids.as_ref(), Some(&mut trace), + has_row_filter, ) .await?; - Ok((plan, trace)) + Ok((self.finalize_plan(plan, grant.as_ref()), trace)) + } + + /// Stamp the grant onto every split (so `TableRead::to_arrow` enforces + /// exactly this plan's grant) and mark row counts inexact when it carries a + /// row filter (dropped as a residual pass inside `TableRead`). + fn finalize_plan(&self, plan: Plan, grant: Option<&Arc>) -> Plan { + // A row filter drops rows and masking rewrites values, so a + // statistics-only `COUNT` would report raw counts and bypass + // enforcement. + let restricted = grant.is_some_and(|g| g.has_server_restrictions()); + let plan = plan.stamp_query_auth_grant(grant.cloned()); + if restricted { + plan.with_inexact_row_counts() + } else { + plan + } } /// Fail closed for a `query-auth.enabled` table: scan planning — including /// `with_scan_all_files`, which read-facing system tables like `files` use — /// exposes file paths, row counts, and stats the client can't authorize. - fn ensure_query_auth_allowed(&self) -> crate::Result<()> { - CoreOptions::new(self.table.schema().options()).ensure_read_authorized() + /// Returns the fetched grant (`None` = not a query-auth table) so the caller + /// can widen the projection and stamp the splits with it. + async fn ensure_query_auth_allowed(&self) -> crate::Result>> { + // Fetch/refresh the grant at plan time (Java parity), then guard + // against pruning on masked or out-of-scope columns. + let select = self.query_auth_projected.as_ref().map(|projected| { + projected + .iter() + .copied() + .chain(self.query_auth_filter_columns.iter().copied()) + .collect::>() + }); + let grant = self + .table + .verify_query_auth_for_read(select.as_ref(), Some(&self.query_auth_system_select)) + .await?; + if let Some(grant) = &grant { + // `scan_all_files` returns the un-merged files a normal PK scan + // hides, whose paths and statistics precede the filtering that only + // runs in `TableRead`. A pure column scope distorts none of that and + // stays allowed (the cross-partition bucket assigner needs it). + if self.scan_all_files && grant.has_server_restrictions() { + return Err(crate::Error::Unsupported { + message: "a query-auth row filter / column masking grant cannot be applied \ + to a scan-all-files plan: it returns raw physical files whose \ + paths and statistics precede enforcement" + .to_string(), + }); + } + crate::table::query_auth::scope_check( + grant, + self.table.schema().fields(), + &self.query_auth_filter_columns, + self.query_auth_projected.clone(), + )?; + } + Ok(grant) } - fn projected_read_field_ids(&self) -> crate::Result>> { - Ok(self.projected_read_field_ids.clone()) + /// Projected field ids for data-evolution column-slice pruning, widened with + /// the grant's filter / mask-input columns so pruning cannot drop a file + /// holding one (an omitted column reads as null and wrongly satisfies + /// `IS_NULL`). Needs the grant, so it cannot happen at `new_scan` time. + fn auth_widened_read_field_ids(&self, grant: Option<&QueryAuthGrant>) -> Option> { + let mut ids = self.projected_read_field_ids.clone(); + if let (Some(set), Some(grant)) = (ids.as_mut(), grant) { + set.extend(grant.read_field_ids(self.table.schema().fields())); + } + ids } /// Apply a limit-pushdown hint to the generated splits. @@ -1361,8 +1519,16 @@ impl<'a> PaimonTableScan<'a> { Ok(entries) } - fn can_push_down_limit_hint(&self, row_ranges: Option<&[RowRange]>) -> bool { + fn can_push_down_limit_hint( + &self, + row_ranges: Option<&[RowRange]>, + query_auth_row_filter: bool, + ) -> bool { + // A query-auth row filter is applied as a residual pass at read time, so + // split merged_row_count overcounts; count-based limit pruning would drop + // splits holding later authorized rows. can_push_down_limit_hint_for_scan(&self.data_predicates, row_ranges) + && !query_auth_row_filter } fn global_index_scan_settings( @@ -1525,14 +1691,17 @@ impl<'a> PaimonTableScan<'a> { /// Reuses the same split-building path as a full snapshot plan, but only /// reads the delta manifest list and keeps ADD entries. pub(crate) async fn plan_snapshot_delta(&self, snapshot: &Snapshot) -> crate::Result { - self.ensure_query_auth_allowed()?; - let data_evolution_read_field_ids = self.projected_read_field_ids()?; - self.plan_snapshot_manifest_list( - snapshot, - snapshot.delta_manifest_list(), - data_evolution_read_field_ids.as_ref(), - ) - .await + let grant = self.ensure_query_auth_allowed().await?; + let data_evolution_read_field_ids = self.auth_widened_read_field_ids(grant.as_deref()); + let plan = self + .plan_snapshot_manifest_list( + snapshot, + snapshot.delta_manifest_list(), + data_evolution_read_field_ids.as_ref(), + grant.as_deref().is_some_and(|g| g.has_row_filter()), + ) + .await?; + Ok(self.finalize_plan(plan, grant.as_ref())) } /// Plan data splits from a snapshot's changelog manifest list. @@ -1541,17 +1710,20 @@ impl<'a> PaimonTableScan<'a> { /// reads the changelog manifest list and keeps ADD entries. Snapshots /// without a changelog list yield an empty plan. pub(crate) async fn plan_snapshot_changelog(&self, snapshot: &Snapshot) -> crate::Result { - self.ensure_query_auth_allowed()?; + let grant = self.ensure_query_auth_allowed().await?; let Some(list_name) = snapshot.changelog_manifest_list() else { - return Ok(Plan::new(Vec::new())); + return Ok(self.finalize_plan(Plan::new(Vec::new()), grant.as_ref())); }; - let data_evolution_read_field_ids = self.projected_read_field_ids()?; - self.plan_snapshot_manifest_list( - snapshot, - list_name, - data_evolution_read_field_ids.as_ref(), - ) - .await + let data_evolution_read_field_ids = self.auth_widened_read_field_ids(grant.as_deref()); + let plan = self + .plan_snapshot_manifest_list( + snapshot, + list_name, + data_evolution_read_field_ids.as_ref(), + grant.as_deref().is_some_and(|g| g.has_row_filter()), + ) + .await?; + Ok(self.finalize_plan(plan, grant.as_ref())) } async fn plan_snapshot_manifest_list( @@ -1559,6 +1731,7 @@ impl<'a> PaimonTableScan<'a> { snapshot: &Snapshot, manifest_list_name: &str, data_evolution_read_field_ids: Option<&HashSet>, + query_auth_row_filter: bool, ) -> crate::Result { if matches!(self.limit, Some(0)) { return Ok(Plan::new(Vec::new())); @@ -1604,6 +1777,7 @@ impl<'a> PaimonTableScan<'a> { index_entries, effective_row_ranges, None, + query_auth_row_filter, ) .await } @@ -1619,7 +1793,8 @@ impl<'a> PaimonTableScan<'a> { before: &Snapshot, after: &Snapshot, ) -> crate::Result<(Plan, Plan)> { - self.ensure_query_auth_allowed()?; + let grant = self.ensure_query_auth_allowed().await?; + let has_row_filter = grant.as_deref().is_some_and(|g| g.has_row_filter()); let core_options = CoreOptions::new(self.table.schema().options()); if core_options.deletion_vectors_enabled() { return Err(crate::Error::Unsupported { @@ -1647,12 +1822,34 @@ impl<'a> PaimonTableScan<'a> { let before_entries = full_state_scan.plan_manifest_entries(before).await?; let after_entries = full_state_scan.plan_manifest_entries(after).await?; let before_plan = full_state_scan - .plan_snapshot_from_entries(before.clone(), before_entries, None, None, None, None) + .plan_snapshot_from_entries( + before.clone(), + before_entries, + None, + None, + None, + None, + has_row_filter, + ) .await?; let after_plan = full_state_scan - .plan_snapshot_from_entries(after.clone(), after_entries, None, None, None, None) + .plan_snapshot_from_entries( + after.clone(), + after_entries, + None, + None, + None, + None, + has_row_filter, + ) .await?; - Ok((before_plan, after_plan)) + // Both sides must carry the grant: the diff pairs them into + // `IncrementalSplit::DiffPair`, and the read gate authorizes off the + // splits it is handed. + Ok(( + self.finalize_plan(before_plan, grant.as_ref()), + self.finalize_plan(after_plan, grant.as_ref()), + )) } async fn validate_diff_bucket_layout( @@ -1799,6 +1996,7 @@ impl<'a> PaimonTableScan<'a> { snapshot: Snapshot, data_evolution_read_field_ids: Option<&HashSet>, mut trace: Option<&mut ScanTrace>, + query_auth_row_filter: bool, ) -> crate::Result { if matches!(self.limit, Some(0)) { if let Some(trace) = trace { @@ -1854,10 +2052,12 @@ impl<'a> PaimonTableScan<'a> { index_entries, effective_row_ranges, trace, + query_auth_row_filter, ) .await } + #[allow(clippy::too_many_arguments)] async fn plan_snapshot_from_entries( &self, snapshot: Snapshot, @@ -1866,6 +2066,7 @@ impl<'a> PaimonTableScan<'a> { index_entries: Option>, effective_row_ranges: Option>, mut trace: Option<&mut ScanTrace>, + query_auth_row_filter: bool, ) -> crate::Result { let table_path = self.table.location(); let table_schema_id = self.table.schema().id(); @@ -1993,7 +2194,8 @@ impl<'a> PaimonTableScan<'a> { .map(|entries| build_deletion_files_map(entries, base_path)); let mut data_file_field_ids_cache = DataFileFieldIdsCache::new(); - let can_push_down_limit = self.can_push_down_limit_hint(effective_row_ranges.as_deref()); + let can_push_down_limit = + self.can_push_down_limit_hint(effective_row_ranges.as_deref(), query_auth_row_filter); let mut limit_accumulator = match self.limit { Some(limit) if limit > 0 && can_push_down_limit => { Some(LimitPushdownAccumulator::new(limit)) diff --git a/crates/paimon/src/table/vector_search_builder.rs b/crates/paimon/src/table/vector_search_builder.rs index 82183f381..7edad4d98 100644 --- a/crates/paimon/src/table/vector_search_builder.rs +++ b/crates/paimon/src/table/vector_search_builder.rs @@ -200,8 +200,10 @@ impl<'a> VectorSearchBuilder<'a> { pub async fn execute_scored(&self) -> crate::Result { // Fail closed: returns data-derived row ranges outside `TableScan`/`TableRead`. + // Strict: search results bypass the query-auth row filter, so only a + // fully unrestricted grant may search. + self.table.authorize_unrestricted_read().await?; let core = CoreOptions::new(self.table.schema().options()); - core.ensure_read_authorized()?; let vector_column = self.vector_column .as_deref() @@ -275,9 +277,11 @@ impl<'a> VectorSearchBuilder<'a> { /// [`with_projection`](Self::with_projection)) plus `__paimon_search_score`; /// `_ROW_ID` and `_PKEY_VECTOR_POSITION` are always hidden. pub async fn execute_read(&self) -> crate::Result { - // Fail closed: returns data outside `TableScan`/`TableRead`. + // Fail closed: materializes rows outside `TableScan`/`TableRead`, so it + // cannot apply the row filter / masking — only a fully unrestricted + // grant may run it (same gate as the scored entry points). + self.table.authorize_unrestricted_read().await?; let core = CoreOptions::new(self.table.schema().options()); - core.ensure_read_authorized()?; let vector_column = self.vector_column .as_deref() @@ -1053,12 +1057,11 @@ impl<'a> BatchVectorSearchBuilder<'a> { } pub async fn execute(&self) -> crate::Result> { - // Fail closed: like `execute_read` and the single-query builder, this - // returns data-derived row ids/scores outside `TableScan`/`TableRead`, - // so it must refuse a `query-auth.enabled` table before any fast path - // (an empty snapshot would otherwise return empty results and bypass it). + // Batch vector search reads index files raw and ranks over masked or + // hidden rows — an oracle it cannot enforce the filter on. Checked + // before any fast path, or an empty snapshot would bypass it. + self.table.authorize_unrestricted_read().await?; let core = CoreOptions::new(self.table.schema().options()); - core.ensure_read_authorized()?; let vector_column = self.vector_column .as_deref() @@ -1172,9 +1175,11 @@ impl<'a> BatchVectorSearchBuilder<'a> { /// scored global row-ids, not materialized rows, so callers use /// [`execute`](Self::execute) instead. pub async fn execute_read(&self) -> crate::Result> { - // Fail closed: returns data outside `TableScan`/`TableRead`. + // Fail closed: materializes rows outside `TableScan`/`TableRead`, so it + // cannot apply the row filter / masking — only a fully unrestricted + // grant may run it (same gate as the scored entry points). + self.table.authorize_unrestricted_read().await?; let core = CoreOptions::new(self.table.schema().options()); - core.ensure_read_authorized()?; let vector_column = self.vector_column .as_deref() @@ -3532,11 +3537,9 @@ mod tests { #[tokio::test] async fn test_batch_execute_fails_closed_when_query_auth_enabled() { - // The batch scored entry returns data-derived row ids/scores outside - // `TableScan`/`TableRead`, so it must fail closed under - // `query-auth.enabled` exactly like the single-query builder. Its config - // is otherwise valid, so without the guard the empty-snapshot fast path - // would return empty results and silently bypass authorization. + // The batch builder reads index files raw, never through + // `plan`/`to_arrow`, so it gates query-auth itself. The config is valid, + // so without the guard the empty-snapshot path would bypass it. let table = crate::table::query_auth_table(); let err = table .new_batch_vector_search_builder() diff --git a/crates/paimon/src/table/vindex_index_build_builder.rs b/crates/paimon/src/table/vindex_index_build_builder.rs index 11579c6eb..41576b4b9 100644 --- a/crates/paimon/src/table/vindex_index_build_builder.rs +++ b/crates/paimon/src/table/vindex_index_build_builder.rs @@ -63,6 +63,11 @@ impl<'a> VindexIndexBuildBuilder<'a> { pub async fn execute(&self) -> Result { self.table.ensure_not_branch_reference_for_write()?; + // Authorize before reading any manifest or index metadata, and once for + // the whole build: doing it per shard both skipped the early-return + // paths and issued one REST round-trip per shard. + let build_grant = self.table.authorize_unrestricted_read().await?; + let grant = build_grant.as_ref(); if !is_vindex_index_type(&self.index_type) { return Err(Error::DataInvalid { @@ -158,7 +163,8 @@ impl<'a> VindexIndexBuildBuilder<'a> { let shard_count = shards.len(); let mut messages = Vec::with_capacity(shard_count); for shard in shards { - let vectors = extract_vectors(self.table, &shard, index_column, dimension).await?; + let vectors = + extract_vectors(self.table, &shard, index_column, dimension, grant).await?; let index_file = self .build_index_file( &shard, @@ -551,7 +557,10 @@ async fn extract_vectors( shard: &VindexIndexShard, index_column: &str, dimension: i32, + grant: Option<&std::sync::Arc>, ) -> Result> { + // Index building reads raw values, so `grant` must be the unrestricted one + // the caller obtained; stamp it so the read is authorized. let split = DataSplitBuilder::new() .with_snapshot(shard.snapshot_id) .with_partition(shard.partition.clone()) @@ -563,7 +572,8 @@ async fn extract_vectors( shard.row_range_start, shard.row_range_end, )]) - .build()?; + .build()? + .with_query_auth_grant(grant.cloned()); let mut read_builder = table.new_read_builder(); read_builder.with_projection(&[index_column, ROW_ID_FIELD_NAME])?; diff --git a/crates/paimon/tests/mock_server.rs b/crates/paimon/tests/mock_server.rs index 6c8a0048a..778247b38 100644 --- a/crates/paimon/tests/mock_server.rs +++ b/crates/paimon/tests/mock_server.rs @@ -34,8 +34,8 @@ use std::sync::{Arc, Mutex}; use tokio::task::JoinHandle; use paimon::api::{ - AlterDatabaseRequest, AlterTableRequest, AuditRESTResponse, ConfigResponse, - CreateFunctionRequest, CreateViewRequest, ErrorResponse, GetDatabaseResponse, + AlterDatabaseRequest, AlterTableRequest, AuditRESTResponse, AuthTableQueryResponse, + ConfigResponse, CreateFunctionRequest, CreateViewRequest, ErrorResponse, GetDatabaseResponse, GetFunctionResponse, GetTableResponse, GetViewResponse, ListDatabasesResponse, ListFunctionsResponse, ListTablesResponse, ListViewsResponse, RenameTableRequest, ResourcePaths, @@ -53,6 +53,8 @@ struct MockState { list_page_size: Option, no_permission_databases: HashSet, no_permission_tables: HashSet, + /// Per-table auth response for `POST .../tables/{table}/auth`; absent = unrestricted. + auth_responses: HashMap, /// ECS metadata role name (for token loader testing) ecs_role_name: Option, /// ECS metadata token (for token loader testing) @@ -705,6 +707,20 @@ impl RESTServer { (StatusCode::NOT_FOUND, Json(err)).into_response() } + /// Handle POST /databases/:db/tables/:table/auth - per-user query-auth check. + pub async fn auth_table_query( + Path((db, table)): Path<(String, String)>, + Extension(state): Extension>, + ) -> impl IntoResponse { + let s = state.inner.lock().unwrap(); + let response = s + .auth_responses + .get(&format!("{db}.{table}")) + .cloned() + .unwrap_or_default(); + (StatusCode::OK, Json(response)).into_response() + } + /// Handle DELETE /databases/:db/tables/:table - drop a table. pub async fn drop_table( Path((db, table)): Path<(String, String)>, @@ -967,6 +983,13 @@ impl RESTServer { ); } + /// Set the auth response returned for `POST .../tables/{table}/auth`. + pub fn set_auth_response(&self, database: &str, table: &str, response: AuthTableQueryResponse) { + let mut s = self.inner.lock().unwrap(); + s.auth_responses + .insert(format!("{database}.{table}"), response); + } + /// Add a no-permission table to the server state. pub fn add_no_permission_table(&self, database: &str, table: &str) { let mut s = self.inner.lock().unwrap(); @@ -1105,6 +1128,10 @@ pub async fn start_mock_server( &format!("{prefix}/databases/:db/functions/:function"), get(RESTServer::get_function), ) + .route( + &format!("{prefix}/databases/:db/tables/:table/auth"), + post(RESTServer::auth_table_query), + ) .route( &format!("{prefix}/tables/rename"), post(RESTServer::rename_table), diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index 2000f9e40..5c2eeb361 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -27,7 +27,7 @@ use arrow_array::{Array, BinaryArray, Int32Array, Int64Array, RecordBatch, Strin use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema}; use axum::http::StatusCode; use futures::TryStreamExt; -use paimon::api::ConfigResponse; +use paimon::api::{AuthTableQueryResponse, ConfigResponse}; use paimon::catalog::{Catalog, Function, FunctionDefinition, Identifier, RESTCatalog, ViewSchema}; use paimon::common::Options; use paimon::spec::{ @@ -346,6 +346,615 @@ async fn test_catalog_get_table() { assert!(table.is_ok(), "failed to get table: {table:?}"); } +/// The grant is scoped to the columns the plan requested (like Java passing +/// `readType.getFieldNames()`): a wider read against a scoped grant fails +/// closed until it re-plans. +#[cfg(not(windows))] +#[tokio::test] +async fn test_query_auth_grant_scoped_to_planned_columns() { + let tmp = tempfile::tempdir().unwrap(); + let warehouse = format!("file://{}", tmp.path().display()); + + // Real data, so the scoped plan actually produces splits to assert on. + let mut fs_options = Options::new(); + fs_options.set(CatalogOptions::WAREHOUSE, &warehouse); + let fs_catalog = FileSystemCatalog::new(fs_options).expect("create filesystem catalog"); + fs_catalog + .create_database("default", true, HashMap::new()) + .await + .unwrap(); + let columns = || { + Schema::builder() + .column("id", DataType::BigInt(BigIntType::new())) + .column("name", DataType::VarChar(VarCharType::new(255).unwrap())) + .option("bucket", "1") + .option("bucket-key", "id") + }; + let identifier = Identifier::new("default", "qa_scope"); + fs_catalog + .create_table(&identifier, columns().build().unwrap(), false) + .await + .unwrap(); + let writer = fs_catalog.get_table(&identifier).await.unwrap(); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int64, true), + ArrowField::new("name", ArrowDataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + arrow_schema, + vec![ + Arc::new(Int64Array::from(vec![1i64, 2])), + Arc::new(arrow_array::StringArray::from(vec!["a", "b"])), + ], + ) + .unwrap(); + write_batch(&writer, batch, "u1").await; + + let ctx = setup_catalog(vec!["default"]).await; + ctx.server.add_table_with_schema( + "default", + "qa_scope", + columns() + .option("query-auth.enabled", "true") + .build() + .unwrap(), + &format!("{warehouse}/default.db/qa_scope"), + ); + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + + // Plan a projection of {id}: the grant is scoped to that column and stamped + // on this plan's splits (not a shared slot on the table). + let mut projected = table.new_read_builder(); + projected.with_projection(&["id"]).unwrap(); + let scoped_plan = projected.new_scan().plan().await.unwrap(); + assert!( + !scoped_plan.splits().is_empty(), + "the scoped plan must produce splits for this assertion to mean anything" + ); + assert!( + projected + .new_read() + .unwrap() + .to_arrow(scoped_plan.splits()) + .is_ok(), + "the planned projection must read under its own grant" + ); + + // The grant rides on the splits it was planned with, so handing those + // splits to a wider read must fail closed rather than widen the scope. + let wide = table.new_read_builder().new_read().unwrap(); + match wide.to_arrow(scoped_plan.splits()) { + Err(paimon::Error::Unsupported { message }) => assert!( + message.contains("outside the authorized set"), + "unexpected rejection: {message}" + ), + Ok(_) => panic!("a wider read must not run under the {{id}} plan's scoped grant"), + Err(e) => panic!("unexpected error: {e}"), + } + + // A separate full-table read re-authorizes for all columns when it plans. + table.new_read_builder().new_scan().plan().await.unwrap(); +} + +/// Java #8447 baseline: a query-auth row filter disables count-based limit +/// pushdown, so a limited read still reaches authorized rows in later files. +#[cfg(not(windows))] +#[tokio::test] +async fn test_query_auth_row_filter_reads_past_limit_pushdown() { + let tmp = tempfile::tempdir().unwrap(); + let warehouse = format!("file://{}", tmp.path().display()); + + // Write two commits (-> two files) through a plain FileSystemCatalog table. + let mut fs_options = Options::new(); + fs_options.set(CatalogOptions::WAREHOUSE, &warehouse); + let fs_catalog = FileSystemCatalog::new(fs_options).expect("create filesystem catalog"); + fs_catalog + .create_database("default", true, HashMap::new()) + .await + .unwrap(); + let write_schema = Schema::builder() + .column("id", DataType::BigInt(BigIntType::new())) + .option("bucket", "1") + .option("bucket-key", "id") + .build() + .unwrap(); + let identifier = Identifier::new("default", "qa_limit"); + fs_catalog + .create_table(&identifier, write_schema, false) + .await + .unwrap(); + let writer = fs_catalog.get_table(&identifier).await.unwrap(); + let int_batch = |ids: Vec| { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + ArrowDataType::Int64, + true, + )])); + RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(ids))]).unwrap() + }; + write_batch(&writer, int_batch(vec![1, 2, 3, 4]), "u1").await; + write_batch(&writer, int_batch(vec![5, 6, 7, 8]), "u2").await; + + // Read the same files through the REST catalog with query-auth enabled and + // a row filter of id >= 6 (all matches live in the SECOND file). + let ctx = setup_catalog(vec!["default"]).await; + let read_schema = Schema::builder() + .column("id", DataType::BigInt(BigIntType::new())) + .option("bucket", "1") + .option("bucket-key", "id") + .option("query-auth.enabled", "true") + .build() + .unwrap(); + ctx.server.add_table_with_schema( + "default", + "qa_limit", + read_schema, + &format!("{warehouse}/default.db/qa_limit"), + ); + ctx.server.set_auth_response( + "default", + "qa_limit", + AuthTableQueryResponse { + filter: Some(vec![ + r#"{"kind":"LEAF","transform":{"name":"FIELD_REF","fieldRef":{"index":0,"name":"id","type":"BIGINT"}},"function":"GREATER_OR_EQUAL","literals":[6]}"# + .to_string(), + ]), + column_masking: None, + }, + ); + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + + let mut builder = table.new_read_builder(); + builder.with_limit(2); + let plan = builder.new_scan().plan().await.unwrap(); + // The filter runs as a residual pass, so the plan must not cap splits by + // the unfiltered limit and must report its row counts as inexact. + assert!(!plan.row_counts_exact()); + let batches: Vec = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids: Vec = batches + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + ids.sort_unstable(); + assert_eq!( + ids, + vec![6, 7, 8], + "authorized rows beyond file 1 must appear" + ); +} + +/// Java #8570 baseline: a cross-column mask (`alias := UPPER(name)`) and a row +/// filter on an unprojected column (`score`) must still enforce when the caller +/// projects neither `name` nor `score` — the read is widened with the grant's +/// columns and every batch of every split is projected back to the caller's +/// columns (no auth-added column may leak from later splits). +#[cfg(not(windows))] +#[tokio::test] +async fn test_query_auth_cross_column_mask_with_narrow_projection() { + let tmp = tempfile::tempdir().unwrap(); + let warehouse = format!("file://{}", tmp.path().display()); + + let mut fs_options = Options::new(); + fs_options.set(CatalogOptions::WAREHOUSE, &warehouse); + let fs_catalog = FileSystemCatalog::new(fs_options).expect("create filesystem catalog"); + fs_catalog + .create_database("default", true, HashMap::new()) + .await + .unwrap(); + let columns = || { + Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("name", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("alias", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("score", DataType::BigInt(BigIntType::new())) + .option("bucket", "1") + .option("bucket-key", "id") + }; + let identifier = Identifier::new("default", "qa_cross_mask"); + fs_catalog + .create_table(&identifier, columns().build().unwrap(), false) + .await + .unwrap(); + let writer = fs_catalog.get_table(&identifier).await.unwrap(); + let batch = |ids: Vec, names: Vec<&str>, aliases: Vec<&str>, scores: Vec| { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, true), + ArrowField::new("name", ArrowDataType::Utf8, true), + ArrowField::new("alias", ArrowDataType::Utf8, true), + ArrowField::new("score", ArrowDataType::Int64, true), + ])); + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(ids)), + Arc::new(StringArray::from(names)), + Arc::new(StringArray::from(aliases)), + Arc::new(Int64Array::from(scores)), + ], + ) + .unwrap() + }; + // Two commits -> two files, so enforcement is exercised across splits. + write_batch( + &writer, + batch(vec![1, 2], vec!["ann", "bob"], vec!["x", "y"], vec![5, 15]), + "u1", + ) + .await; + write_batch( + &writer, + batch(vec![3, 4], vec!["cid", "dan"], vec!["z", "w"], vec![20, 8]), + "u2", + ) + .await; + + let ctx = setup_catalog(vec!["default"]).await; + ctx.server.add_table_with_schema( + "default", + "qa_cross_mask", + columns() + .option("query-auth.enabled", "true") + .build() + .unwrap(), + &format!("{warehouse}/default.db/qa_cross_mask"), + ); + ctx.server.set_auth_response( + "default", + "qa_cross_mask", + AuthTableQueryResponse { + // Row filter on `score` (index 3), which the caller does not project. + filter: Some(vec![ + r#"{"kind":"LEAF","transform":{"name":"FIELD_REF","fieldRef":{"index":3,"name":"score","type":"BIGINT"}},"function":"GREATER_OR_EQUAL","literals":[10]}"# + .to_string(), + ]), + // Cross-column mask: `alias` is overwritten from `name` (index 1), + // which the caller does not project either. + column_masking: Some(HashMap::from([( + "alias".to_string(), + r#"{"name":"UPPER","inputs":[{"index":1,"name":"name","type":"STRING"}]}"# + .to_string(), + )])), + }, + ); + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + + let mut builder = table.new_read_builder(); + builder.with_projection(&["id", "alias"]).unwrap(); + let plan = builder.new_scan().plan().await.unwrap(); + let batches: Vec = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + + let mut rows = Vec::new(); + for b in &batches { + assert_eq!( + b.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec!["id", "alias"], + "auth-added columns (name, score) must not leak from any split" + ); + let ids = b.column(0).as_any().downcast_ref::().unwrap(); + let aliases = b.column(1).as_any().downcast_ref::().unwrap(); + for r in 0..b.num_rows() { + rows.push((ids.value(r), aliases.value(r).to_string())); + } + } + rows.sort_unstable(); + assert_eq!( + rows, + vec![(2, "BOB".to_string()), (3, "CID".to_string())], + "filter must drop score<10 rows in both files and alias must be UPPER(name)" + ); +} + +/// Visible end-to-end demo of query-auth enforcement over a mock REST catalog: +/// the same files are read once with no grant (raw) and once through a +/// `query-auth.enabled` table whose per-user grant applies a row filter +/// (`salary >= 90000`) plus column masking (`name -> UPPER(name)`). Run with: +/// cargo test -p paimon --test rest_catalog_test query_auth_enforcement_demo -- --nocapture +#[cfg(not(windows))] +#[tokio::test] +async fn test_query_auth_enforcement_demo() { + let tmp = tempfile::tempdir().unwrap(); + let warehouse = format!("file://{}", tmp.path().display()); + + // --- Write demo_employees (id, name, salary) via a plain FileSystemCatalog --- + let mut fs_options = Options::new(); + fs_options.set(CatalogOptions::WAREHOUSE, &warehouse); + let fs_catalog = FileSystemCatalog::new(fs_options).expect("create filesystem catalog"); + fs_catalog + .create_database("default", true, HashMap::new()) + .await + .unwrap(); + let columns = || { + Schema::builder() + .column("id", DataType::Int(IntType::new())) + .column("name", DataType::VarChar(VarCharType::new(255).unwrap())) + .column("salary", DataType::BigInt(BigIntType::new())) + .option("bucket", "1") + .option("bucket-key", "id") + }; + let identifier = Identifier::new("default", "demo_employees"); + fs_catalog + .create_table(&identifier, columns().build().unwrap(), false) + .await + .unwrap(); + let writer = fs_catalog.get_table(&identifier).await.unwrap(); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", ArrowDataType::Int32, true), + ArrowField::new("name", ArrowDataType::Utf8, true), + ArrowField::new("salary", ArrowDataType::Int64, true), + ])); + let batch = RecordBatch::try_new( + arrow_schema, + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec![ + "alice", "bob", "charlie", "diana", "eve", + ])), + Arc::new(Int64Array::from(vec![120000, 85000, 95000, 70000, 99000])), + ], + ) + .unwrap(); + write_batch(&writer, batch, "u1").await; + + let dump = |label: &str, batches: &[RecordBatch]| { + println!("\n {label}"); + println!(" {:<4} {:<10} {:>8}", "id", "name", "salary"); + for b in batches { + let ids = b.column(0).as_any().downcast_ref::().unwrap(); + let names = b.column(1).as_any().downcast_ref::().unwrap(); + let sal = b.column(2).as_any().downcast_ref::().unwrap(); + for r in 0..b.num_rows() { + println!( + " {:<4} {:<10} {:>8}", + ids.value(r), + names.value(r), + sal.value(r) + ); + } + } + }; + + // --- Raw read (no query-auth) --- + let raw: Vec = { + let b = writer.new_read_builder(); + b.new_read() + .unwrap() + .to_arrow(b.new_scan().plan().await.unwrap().splits()) + .unwrap() + .try_collect() + .await + .unwrap() + }; + dump("RAW (no grant): all rows, real names", &raw); + + // --- Enforced read through the REST catalog with a per-user grant --- + let ctx = setup_catalog(vec!["default"]).await; + ctx.server.add_table_with_schema( + "default", + "demo_employees", + columns() + .option("query-auth.enabled", "true") + .build() + .unwrap(), + &format!("{warehouse}/default.db/demo_employees"), + ); + ctx.server.set_auth_response( + "default", + "demo_employees", + AuthTableQueryResponse { + // Row filter: salary >= 90000 (field index 2). + filter: Some(vec![ + r#"{"kind":"LEAF","transform":{"name":"FIELD_REF","fieldRef":{"index":2,"name":"salary","type":"BIGINT"}},"function":"GREATER_OR_EQUAL","literals":[90000]}"# + .to_string(), + ]), + // Column masking: name -> UPPER(name) (field index 1). + column_masking: Some(HashMap::from([( + "name".to_string(), + r#"{"name":"UPPER","inputs":[{"index":1,"name":"name","type":"STRING"}]}"# + .to_string(), + )])), + }, + ); + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + let b = table.new_read_builder(); + // Plan first: scan planning fetches + verifies the per-user grant (mirroring + // Java `CatalogEnvironment.tableQueryAuth()`) and authorizes the shared read + // state; only then may the sync read gate (`new_read`/`to_arrow`) proceed. + let plan = b.new_scan().plan().await.unwrap(); + let enforced: Vec = b + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect() + .await + .unwrap(); + dump("ENFORCED (grant: salary>=90000, name->UPPER)", &enforced); + + let mut rows: Vec<(i32, String, i64)> = enforced + .iter() + .flat_map(|batch| { + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let names = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let sal = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + (0..batch.num_rows()) + .map(|r| (ids.value(r), names.value(r).to_string(), sal.value(r))) + .collect::>() + }) + .collect(); + rows.sort_unstable(); + assert_eq!( + rows, + vec![ + (1, "ALICE".to_string(), 120000), + (3, "CHARLIE".to_string(), 95000), + (5, "EVE".to_string(), 99000), + ], + "row filter must drop salary<90000 and masking must uppercase name" + ); +} + +/// Query-auth: scan planning transparently fetches the per-user grant +/// (mirroring Java's `CatalogEnvironment.tableQueryAuth()`); an unrestricted +/// user reads everything, a filtered/masked user gets an enforced read, and +/// paths that cannot enforce stay fail-closed. +#[tokio::test] +async fn test_catalog_get_table_query_auth() { + let ctx = setup_catalog(vec!["default"]).await; + let schema = Schema::builder() + .column("id", DataType::BigInt(BigIntType::new())) + .option("query-auth.enabled", "true") + .build() + .expect("Failed to build schema"); + ctx.server.add_table_with_schema( + "default", + "qa", + schema, + "file:///tmp/test_warehouse/default.db/qa", + ); + let identifier = Identifier::new("default", "qa"); + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + + // Reading no splits yields no rows, so it is allowed even before planning; + // real splits carrying no grant fail closed (read_builder unit tests). + let ungranted = table.new_read_builder().new_read().unwrap(); + assert!(ungranted.to_arrow(&[]).is_ok()); + + // The mock /auth endpoint reports unrestricted by default: planning a scan + // authorizes the table and stamps the grant on its splits. + let builder = table.new_read_builder(); + builder + .new_scan() + .plan() + .await + .expect("unrestricted user should be able to plan a query-auth scan"); + + // A parseable row filter (Java Predicate JSON) grants a filtered read: + // planning and building the read succeed; the filter is enforced inside + // `to_arrow`. + ctx.server.set_auth_response( + "default", + "qa", + AuthTableQueryResponse { + filter: Some(vec![ + r#"{"kind":"LEAF","transform":{"name":"FIELD_REF","fieldRef":{"index":0,"name":"id","type":"BIGINT"}},"function":"GREATER_THAN","literals":[5]}"# + .to_string(), + ]), + column_masking: None, + }, + ); + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + let builder = table.new_read_builder(); + builder + .new_scan() + .plan() + .await + .expect("a parseable row filter should allow planning a (filtered) read"); + // ... but paths that bypass the row filter stay strictly fail-closed. + let err = table + .new_vector_search_builder() + .execute() + .await + .unwrap_err(); + assert!( + err.to_string().contains("query-auth.enabled"), + "search must stay fail-closed for a filtered user, got: {err}" + ); + + // An unparseable filter fails the plan and keeps the table fail-closed. + ctx.server.set_auth_response( + "default", + "qa", + AuthTableQueryResponse { + filter: Some(vec!["{\"kind\":\"CUSTOM\"}".to_string()]), + column_masking: None, + }, + ); + let fresh = ctx.catalog.get_table(&identifier).await.unwrap(); + assert!(fresh.new_read_builder().new_scan().plan().await.is_err()); + + // Parseable column masking grants a (masked) read; a caller predicate on + // the masked column is rejected (it would leak the raw value). + ctx.server.set_auth_response( + "default", + "qa", + AuthTableQueryResponse { + filter: None, + column_masking: Some(HashMap::from([( + "id".to_string(), + "{\"name\":\"NULL\"}".to_string(), + )])), + }, + ); + let table = ctx.catalog.get_table(&identifier).await.unwrap(); + let builder = table.new_read_builder(); + builder.new_scan().plan().await.expect("masked read plans"); + // A caller predicate on a masked column fails closed at plan time (pruning + // on its raw value would leak it); the same guard runs again in `to_arrow`. + let mut filtered = table.new_read_builder(); + filtered.with_filter( + PredicateBuilder::new(table.schema().fields()) + .equal("id", Datum::Long(1)) + .unwrap(), + ); + let Err(err) = filtered.new_scan().plan().await else { + panic!("a caller predicate on a masked column must fail closed"); + }; + assert!( + err.to_string().contains("masked column"), + "a caller predicate on a masked column must fail closed, got: {err}" + ); + + // Every plan re-authorizes (like Java): revoking down to an unparseable + // grant fails the plan, and a read without a stamped grant stays closed. + ctx.server.set_auth_response( + "default", + "qa", + AuthTableQueryResponse { + filter: Some(vec!["{\"kind\":\"CUSTOM\"}".to_string()]), + column_masking: None, + }, + ); + assert!(table.new_read_builder().new_scan().plan().await.is_err()); +} + #[tokio::test] async fn test_rest_env_get_table_reuses_catalog_environment() { let ctx = setup_catalog(vec!["default"]).await;