> part
throw new TableNoPermissionException(identifier, e);
} catch (BadRequestException e) {
throw new IllegalArgumentException(e.getMessage());
+ } finally {
+ // The server may have committed even when the response was lost or malformed.
+ FormatTableCatalogProvider.advanceGeneration(identifier);
}
return;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
index 9a832176caaf..64325e70b7c8 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
@@ -18,9 +18,11 @@
package org.apache.paimon.table;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
import org.apache.paimon.annotation.Public;
import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogUtils;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.manifest.IndexManifestEntry;
@@ -29,6 +31,7 @@
import org.apache.paimon.stats.Statistics;
import org.apache.paimon.table.format.FormatBatchWriteBuilder;
import org.apache.paimon.table.format.FormatReadBuilder;
+import org.apache.paimon.table.format.FormatTableCatalogProvider;
import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.StreamWriteBuilder;
import org.apache.paimon.table.source.BatchVectorSearchBuilder;
@@ -56,8 +59,9 @@
* operations on this table allow for reading or writing to these files, facilitating the retrieval
* of existing data and the addition of new files.
*
- * Partitioned file format table just like the standard hive format. Partitions are discovered
- * and inferred based on directory structure.
+ *
A partitioned file format table uses the standard Hive directory layout. By default,
+ * partitions are discovered from that layout. Catalog-managed format tables use catalog metadata
+ * for partition visibility while retaining the same physical layout.
*
* @since 0.9.0
*/
@@ -75,6 +79,12 @@ public interface FormatTable extends Table {
CatalogContext catalogContext();
+ /** Catalog access used by managed partition discovery and registration. */
+ @Nullable
+ default FormatTableCatalogProvider catalogProvider() {
+ return null;
+ }
+
/** Currently supported formats. */
enum Format {
ORC,
@@ -115,6 +125,7 @@ class Builder {
private Map options;
@Nullable private String comment;
private CatalogContext catalogContext;
+ @Nullable private FormatTableCatalogProvider catalogProvider;
public Builder fileIO(FileIO fileIO) {
this.fileIO = fileIO;
@@ -161,6 +172,11 @@ public Builder catalogContext(CatalogContext catalogContext) {
return this;
}
+ public Builder catalogProvider(@Nullable FormatTableCatalogProvider catalogProvider) {
+ this.catalogProvider = catalogProvider;
+ return this;
+ }
+
public FormatTable build() {
return new FormatTableImpl(
fileIO,
@@ -171,7 +187,8 @@ public FormatTable build() {
format,
options,
comment,
- catalogContext);
+ catalogContext,
+ catalogProvider);
}
}
@@ -189,6 +206,7 @@ class FormatTableImpl implements FormatTable {
private final Map options;
@Nullable private final String comment;
private CatalogContext catalogContext;
+ @Nullable private final FormatTableCatalogProvider catalogProvider;
public FormatTableImpl(
FileIO fileIO,
@@ -200,6 +218,30 @@ public FormatTableImpl(
Map options,
@Nullable String comment,
CatalogContext catalogContext) {
+ this(
+ fileIO,
+ identifier,
+ rowType,
+ partitionKeys,
+ location,
+ format,
+ options,
+ comment,
+ catalogContext,
+ null);
+ }
+
+ public FormatTableImpl(
+ FileIO fileIO,
+ Identifier identifier,
+ RowType rowType,
+ List partitionKeys,
+ String location,
+ Format format,
+ Map options,
+ @Nullable String comment,
+ CatalogContext catalogContext,
+ @Nullable FormatTableCatalogProvider catalogProvider) {
this.fileIO = fileIO;
this.identifier = identifier;
this.rowType = rowType;
@@ -209,6 +251,7 @@ public FormatTableImpl(
this.options = options;
this.comment = comment;
this.catalogContext = catalogContext;
+ this.catalogProvider = catalogProvider;
}
@Override
@@ -265,6 +308,27 @@ public FileIO fileIO() {
public FormatTable copy(Map dynamicOptions) {
Map newOptions = new HashMap<>(options);
newOptions.putAll(dynamicOptions);
+
+ CoreOptions coreOptions = CoreOptions.fromMap(options);
+ CoreOptions copiedCoreOptions = CoreOptions.fromMap(newOptions);
+ boolean managed = coreOptions.partitionedTableInMetastore();
+ boolean copiedManaged = copiedCoreOptions.partitionedTableInMetastore();
+ if (managed != copiedManaged) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Dynamic option '%s' cannot change whether Format Table partitions are catalog-managed.",
+ CoreOptions.METASTORE_PARTITIONED_TABLE.key()));
+ }
+ if (managed
+ && coreOptions.formatTablePartitionOnlyValueInPath()
+ != copiedCoreOptions.formatTablePartitionOnlyValueInPath()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Dynamic option '%s' cannot change the physical partition layout of a catalog-managed Format Table.",
+ CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key()));
+ }
+ CatalogUtils.validateManagedFormatTableOptions(newOptions);
+
return new FormatTableImpl(
fileIO,
identifier,
@@ -274,7 +338,8 @@ public FormatTable copy(Map dynamicOptions) {
format,
newOptions,
comment,
- catalogContext);
+ catalogContext,
+ catalogProvider);
}
@Override
@@ -302,6 +367,12 @@ public FullTextSearchBuilder newFullTextSearchBuilder() {
public CatalogContext catalogContext() {
return this.catalogContext;
}
+
+ @Override
+ @Nullable
+ public FormatTableCatalogProvider catalogProvider() {
+ return catalogProvider;
+ }
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
index 924e67dbdfdc..d3970bb878ea 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
@@ -87,7 +87,8 @@ public BatchTableCommit newCommit() {
Identifier.fromString(table.fullName()),
staticPartition,
syncHiveUri,
- table.catalogContext());
+ table.catalogContext(),
+ table.catalogProvider());
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java
index 30a567b436a6..1c9010d0be04 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatReadBuilder.java
@@ -173,6 +173,16 @@ public TableScan newScan() {
partitionFilter = partitionPredicateOpt.get();
}
}
+ if (options.partitionedTableInMetastore()) {
+ if (table.catalogProvider() == null) {
+ throw new IllegalStateException(
+ String.format(
+ "Managed format table %s has no catalog partition provider. "
+ + "The catalog client does not support managed format tables.",
+ table.fullName()));
+ }
+ return new ManagedFormatTableScan(table, partitionFilter, limit);
+ }
return new FormatTableScan(table, partitionFilter, limit);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCatalogProvider.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCatalogProvider.java
new file mode 100644
index 000000000000..c297ef63e094
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCatalogProvider.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.PagedList;
+import org.apache.paimon.annotation.Experimental;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogLoader;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.utils.StringUtils;
+
+import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Cache;
+import org.apache.paimon.shade.caffeine2.com.github.benmanes.caffeine.cache.Caffeine;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
+/** Serializable catalog access for a managed format table. */
+@Experimental
+public class FormatTableCatalogProvider implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final int PARTITION_PAGE_SIZE = 1000;
+ private static final int MAX_CACHED_PATTERNS = 128;
+ private static final int MAX_TRACKED_TABLE_GENERATIONS = 10_000;
+ private static final Duration PARTITION_CACHE_TTL = Duration.ofSeconds(30);
+ private static final Duration TABLE_GENERATION_TTL = Duration.ofHours(1);
+
+ // Keyed by identifier full name: listings themselves live in per-instance caches, so sharing
+ // a generation counter across table incarnations (or across catalogs using the same name) can
+ // only cause an extra invalidation, never a stale read.
+ private static final Cache GENERATIONS =
+ Caffeine.newBuilder()
+ .expireAfterAccess(TABLE_GENERATION_TTL)
+ .maximumSize(MAX_TRACKED_TABLE_GENERATIONS)
+ .executor(Runnable::run)
+ .build();
+
+ private final Identifier identifier;
+ private final CatalogLoader catalogLoader;
+
+ @Nullable private transient Cache> partitionCache;
+ // Reused across list/create calls: constructing a Catalog (HTTP client, auth provider, and
+ // for some auth providers an initial token fetch) per partition operation is expensive, and
+ // RESTCatalog.close() is a no-op so a long-lived instance leaks nothing. Recreated lazily
+ // after deserialization.
+ @Nullable private transient Catalog catalog;
+
+ public FormatTableCatalogProvider(Identifier identifier, CatalogLoader catalogLoader) {
+ this.identifier = identifier;
+ this.catalogLoader = catalogLoader;
+ }
+
+ /** List every catalog-visible partition matching the optional name prefix pattern. */
+ public List listPartitions(@Nullable String partitionNamePattern) {
+ return cache().get(
+ partitionCacheKey(partitionNamePattern),
+ ignored -> loadPartitions(partitionNamePattern));
+ }
+
+ /**
+ * Advance the partition-listing generation of every provider of this table in this JVM, so
+ * same-process scans skip cached listings taken before the mutation. The catalog partition DDL
+ * path (create/dropPartitions) calls this from a {@code finally} block: the mutation's outcome
+ * can be ambiguous (the server may have committed even when the response was lost), so the
+ * cached listing must be dropped after every attempt, not only after a confirmed success.
+ */
+ public static void advanceGeneration(Identifier identifier) {
+ GENERATIONS.get(identifier.getFullName(), ignored -> new AtomicLong()).incrementAndGet();
+ }
+
+ /** Create partitions with the catalog's idempotent create contract. */
+ public void createPartitions(List> partitions) {
+ if (partitions.isEmpty()) {
+ return;
+ }
+ try {
+ // Register in bounded batches: one backfill commit can touch tens of thousands of
+ // partitions (e.g. years of hourly data in a single dynamic-partition INSERT), and a
+ // single unbounded request either monopolizes a catalog worker or trips server-side
+ // request caps, failing the job after its data files are already committed.
+ // Registration is an idempotent ADD-only upsert, so a mid-way failure leaves a state
+ // that a rerun or MSCK converges from.
+ for (int start = 0; start < partitions.size(); start += PARTITION_PAGE_SIZE) {
+ catalog()
+ .createPartitions(
+ identifier,
+ partitions.subList(
+ start,
+ Math.min(start + PARTITION_PAGE_SIZE, partitions.size())));
+ }
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(
+ String.format(
+ "Failed to register partitions for managed format table %s.",
+ identifier),
+ e);
+ } finally {
+ advanceGeneration(identifier);
+ cache().invalidateAll();
+ }
+ }
+
+ private List loadPartitions(@Nullable String partitionNamePattern) {
+ List partitions = new ArrayList<>();
+ Set seenPageTokens = new HashSet<>();
+ try {
+ Catalog catalog = catalog();
+ String pageToken = null;
+ do {
+ PagedList page =
+ catalog.listPartitionsPaged(
+ identifier, PARTITION_PAGE_SIZE, pageToken, partitionNamePattern);
+ partitions.addAll(page.getElements());
+ pageToken = page.getNextPageToken();
+ if (StringUtils.isNotEmpty(pageToken) && !seenPageTokens.add(pageToken)) {
+ throw new IllegalStateException(
+ String.format(
+ "Catalog returned repeated partition page token '%s' for managed format table %s.",
+ pageToken, identifier.getFullName()));
+ }
+ } while (StringUtils.isNotEmpty(pageToken));
+ return Collections.unmodifiableList(partitions);
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(
+ String.format(
+ "Failed to list partitions for managed format table %s.", identifier),
+ e);
+ }
+ }
+
+ private synchronized Catalog catalog() {
+ if (catalog == null) {
+ catalog = catalogLoader.load();
+ }
+ return catalog;
+ }
+
+ private synchronized Cache> cache() {
+ if (partitionCache == null) {
+ partitionCache =
+ Caffeine.newBuilder()
+ .expireAfterWrite(PARTITION_CACHE_TTL)
+ .maximumSize(MAX_CACHED_PATTERNS)
+ .executor(Runnable::run)
+ .build();
+ }
+ return partitionCache;
+ }
+
+ private String partitionCacheKey(@Nullable String partitionNamePattern) {
+ // Partition mutations advance a process-local generation so providers in this JVM skip
+ // stale entries. Providers in other JVMs converge when PARTITION_CACHE_TTL expires.
+ return generation().get() + "\000" + partitionNamePattern;
+ }
+
+ private AtomicLong generation() {
+ return GENERATIONS.get(identifier.getFullName(), ignored -> new AtomicLong());
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
index ed98c6ba50ff..8d38388907c5 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
@@ -48,7 +48,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.StringJoiner;
import static org.apache.paimon.table.format.FormatBatchWriteBuilder.validateStaticPartition;
@@ -63,6 +62,10 @@ public class FormatTableCommit implements BatchTableCommit {
protected boolean overwrite = false;
private Catalog hiveCatalog;
private Identifier tableIdentifier;
+ @Nullable private final FormatTableCatalogProvider catalogProvider;
+ // Volatile: commit() and abort() may run on different threads when callers drive the API
+ // directly; abort must observe that registration already began to keep its no-op guarantee.
+ private volatile boolean partitionRegistrationStarted;
public FormatTableCommit(
String location,
@@ -74,6 +77,30 @@ public FormatTableCommit(
@Nullable Map staticPartitions,
@Nullable String syncHiveUri,
CatalogContext catalogContext) {
+ this(
+ location,
+ partitionKeys,
+ fileIO,
+ formatTablePartitionOnlyValueInPath,
+ overwrite,
+ tableIdentifier,
+ staticPartitions,
+ syncHiveUri,
+ catalogContext,
+ null);
+ }
+
+ public FormatTableCommit(
+ String location,
+ List partitionKeys,
+ FileIO fileIO,
+ boolean formatTablePartitionOnlyValueInPath,
+ boolean overwrite,
+ Identifier tableIdentifier,
+ @Nullable Map staticPartitions,
+ @Nullable String syncHiveUri,
+ CatalogContext catalogContext,
+ @Nullable FormatTableCatalogProvider catalogProvider) {
this.location = location;
this.fileIO = fileIO;
this.formatTablePartitionOnlyValueInPath = formatTablePartitionOnlyValueInPath;
@@ -82,6 +109,7 @@ public FormatTableCommit(
this.overwrite = overwrite;
this.partitionKeys = partitionKeys;
this.tableIdentifier = tableIdentifier;
+ this.catalogProvider = catalogProvider;
if (syncHiveUri != null) {
try {
Options options = new Options();
@@ -143,7 +171,9 @@ public void commit(List commitMessages) {
for (TwoPhaseOutputStream.Committer committer : committers) {
committer.commit(this.fileIO);
- if (partitionKeys != null && !partitionKeys.isEmpty() && hiveCatalog != null) {
+ if (partitionKeys != null
+ && !partitionKeys.isEmpty()
+ && (hiveCatalog != null || catalogProvider != null)) {
partitionSpecs.add(
extractPartitionSpecFromPath(
committer.targetPath().getParent(), partitionKeys));
@@ -152,6 +182,14 @@ public void commit(List commitMessages) {
for (TwoPhaseOutputStream.Committer committer : committers) {
committer.clean(this.fileIO);
}
+ if (catalogProvider != null && !partitionSpecs.isEmpty()) {
+ partitionRegistrationStarted = true;
+ try {
+ catalogProvider.createPartitions(new ArrayList<>(partitionSpecs));
+ } catch (RuntimeException e) {
+ throw partitionRegistrationFailure(e);
+ }
+ }
for (Map partitionSpec : partitionSpecs) {
if (hiveCatalog != null) {
try {
@@ -172,11 +210,28 @@ public void commit(List commitMessages) {
}
} catch (Exception e) {
- this.abort(commitMessages);
- throw new RuntimeException(e);
+ if (!partitionRegistrationStarted) {
+ this.abort(commitMessages);
+ throw new RuntimeException(e);
+ }
+ throw e instanceof RuntimeException ? (RuntimeException) e : new RuntimeException(e);
}
}
+ private RuntimeException partitionRegistrationFailure(RuntimeException cause) {
+ String tableName = tableIdentifier.getFullName();
+ return new RuntimeException(
+ String.format(
+ "Managed partition registration failed for %s after data files were "
+ + "committed. Committed data files were preserved because the "
+ + "catalog state may be ambiguous. Verify the catalog partition "
+ + "metadata and, if partitions are missing, re-run a partition "
+ + "metadata sync for %s (e.g. the Spark procedure "
+ + "sys.sync_format_table_metadata or MSCK REPAIR TABLE).",
+ tableName, tableName),
+ cause);
+ }
+
private Method getHiveCreatePartitionsInHmsMethod() throws NoSuchMethodException {
Method hiveCreatePartitionsInHmsMethod =
hiveCatalog
@@ -192,12 +247,68 @@ private Method getHiveCreatePartitionsInHmsMethod() throws NoSuchMethodException
private LinkedHashMap extractPartitionSpecFromPath(
Path partitionPath, List partitionKeys) {
- if (formatTablePartitionOnlyValueInPath) {
- return PartitionPathUtils.extractPartitionSpecFromPathOnlyValue(
- partitionPath, partitionKeys);
- } else {
- return PartitionPathUtils.extractPartitionSpecFromPath(partitionPath);
+ // The writer always lays partitions out as //.../, so the partition
+ // spec is exactly the trailing partitionKeys.size() components of the partition
+ // directory. Never walk beyond them: the table location itself may contain foreign
+ // 'k=v' segments that must not leak into the registered spec. Registered values are
+ // RAW (unescaped); readers re-escape them when probing partition directories.
+ String[] components = new String[partitionKeys.size()];
+ Path current = partitionPath;
+ for (int i = partitionKeys.size() - 1; i >= 0; i--) {
+ if (current == null || current.getName().isEmpty()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Partition path '%s' has fewer than %s directory levels required "
+ + "by partition keys %s of table %s.",
+ partitionPath,
+ partitionKeys.size(),
+ partitionKeys,
+ tableIdentifier.getFullName()));
+ }
+ components[i] = current.getName();
+ current = current.getParent();
+ }
+
+ LinkedHashMap partitionSpec = new LinkedHashMap<>();
+ for (int i = 0; i < partitionKeys.size(); i++) {
+ String expectedKey = partitionKeys.get(i);
+ String component = components[i];
+ if (formatTablePartitionOnlyValueInPath) {
+ partitionSpec.put(expectedKey, PartitionPathUtils.unescapePathName(component));
+ } else {
+ int splitIndex = component.indexOf('=');
+ if (splitIndex < 0) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Partition directory '%s' of partition path '%s' is not in "
+ + "'key=value' form expected for partition key '%s' "
+ + "of table %s.",
+ component,
+ partitionPath,
+ expectedKey,
+ tableIdentifier.getFullName()));
+ }
+ String parsedKey =
+ PartitionPathUtils.unescapePathName(component.substring(0, splitIndex));
+ String parsedValue =
+ PartitionPathUtils.unescapePathName(component.substring(splitIndex + 1));
+ if (!expectedKey.equals(parsedKey)) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Partition directory '%s' of partition path '%s' declares "
+ + "partition key '%s' but partition key '%s' was "
+ + "expected at position %s for table %s.",
+ component,
+ partitionPath,
+ parsedKey,
+ expectedKey,
+ i,
+ tableIdentifier.getFullName()));
+ }
+ partitionSpec.put(expectedKey, parsedValue);
+ }
}
+ return partitionSpec;
}
private static Path buildPartitionPath(
@@ -208,24 +319,29 @@ private static Path buildPartitionPath(
if (partitionSpec.isEmpty() || partitionKeys.isEmpty()) {
throw new IllegalArgumentException("partitionSpec or partitionKeys is empty.");
}
- StringJoiner joiner = new StringJoiner("/");
+ LinkedHashMap orderedSpec = new LinkedHashMap<>();
for (int i = 0; i < partitionSpec.size(); i++) {
String key = partitionKeys.get(i);
if (partitionSpec.containsKey(key)) {
- if (formatTablePartitionOnlyValueInPath) {
- joiner.add(partitionSpec.get(key));
- } else {
- joiner.add(key + "=" + partitionSpec.get(key));
- }
+ orderedSpec.put(key, partitionSpec.get(key));
} else {
throw new RuntimeException("partitionSpec does not contain key: " + key);
}
}
- return new Path(location, joiner.toString());
+ return new Path(
+ location,
+ PartitionPathUtils.generatePartitionPathUtil(
+ orderedSpec, formatTablePartitionOnlyValueInPath));
}
@Override
public void abort(List commitMessages) {
+ // Once partition registration has started the data files are already committed and must be
+ // preserved: the catalog outcome is ambiguous (see partitionRegistrationFailure), so
+ // discarding the files here could delete data the catalog already references.
+ if (partitionRegistrationStarted) {
+ return;
+ }
try {
for (CommitMessage commitMessage : commitMessages) {
if (commitMessage instanceof TwoPhaseCommitMessage) {
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
index cb0edefc82fc..62fc6b09e3c5 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
@@ -59,6 +59,7 @@
import javax.annotation.Nullable;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -81,9 +82,9 @@ public class FormatTableScan implements InnerTableScan {
private static final Logger LOG = LoggerFactory.getLogger(FormatTableScan.class);
- private final FormatTable table;
- private final CoreOptions coreOptions;
- @Nullable private PartitionPredicate partitionFilter;
+ protected final FormatTable table;
+ protected final CoreOptions coreOptions;
+ @Nullable protected PartitionPredicate partitionFilter;
@Nullable private final Integer limit;
private final long targetSplitSize;
private final long openFileCost;
@@ -142,7 +143,7 @@ public static boolean isDataFileName(String fileName) {
return fileName != null && !fileName.startsWith(".") && !fileName.startsWith("_");
}
- private BinaryRow toPartitionRow(LinkedHashMap partitionSpec) {
+ protected BinaryRow toPartitionRow(LinkedHashMap partitionSpec) {
RowType partitionType = table.partitionType();
GenericRow row =
convertSpecToInternalRow(partitionSpec, partitionType, table.defaultPartName());
@@ -160,7 +161,11 @@ public List splits() {
LinkedHashMap partitionSpec = pair.getKey();
BinaryRow partitionRow = toPartitionRow(partitionSpec);
if (partitionFilter == null || partitionFilter.test(partitionRow)) {
- splits.addAll(createSplits(fileIO, pair.getValue(), partitionRow));
+ try {
+ splits.addAll(createSplits(fileIO, pair.getValue(), partitionRow));
+ } catch (FileNotFoundException e) {
+ onPartitionFileNotFound(partitionSpec, pair.getValue(), e);
+ }
}
}
} else {
@@ -177,7 +182,7 @@ public List splits() {
}
}
- List, Path>> findPartitions() {
+ protected List, Path>> findPartitions() {
LOG.debug(
"Find partitions for format table {}, partition filter: {}",
table.name(),
@@ -222,6 +227,14 @@ List, Path>> findPartitions() {
}
}
+ protected void onPartitionFileNotFound(
+ LinkedHashMap partitionSpec,
+ Path partitionPath,
+ FileNotFoundException exception)
+ throws IOException {
+ throw exception;
+ }
+
protected static List, Path>> generatePartitions(
List partitionKeys,
RowType partitionType,
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/ManagedFormatTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/format/ManagedFormatTableScan.java
new file mode 100644
index 000000000000..7e16dee0839e
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/ManagedFormatTableScan.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.fs.FileStatus;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.manifest.PartitionEntry;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.utils.Pair;
+import org.apache.paimon.utils.PartitionPathUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/** A format table scan whose partition visibility is owned by a catalog. */
+public class ManagedFormatTableScan extends FormatTableScan {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ManagedFormatTableScan.class);
+
+ private final FormatTableCatalogProvider catalogProvider;
+
+ public ManagedFormatTableScan(
+ FormatTable table,
+ @Nullable PartitionPredicate partitionFilter,
+ @Nullable Integer limit) {
+ super(table, partitionFilter, limit);
+ FormatTableCatalogProvider provider = table.catalogProvider();
+ if (provider == null) {
+ throw new IllegalStateException(
+ String.format(
+ "Managed format table %s has no catalog partition provider.",
+ table.fullName()));
+ }
+ this.catalogProvider = provider;
+ }
+
+ @Override
+ protected List, Path>> findPartitions() {
+ String partitionNamePattern = createPartitionNamePattern();
+ List partitions = catalogProvider.listPartitions(partitionNamePattern);
+ if (partitions.isEmpty() && partitionNamePattern == null) {
+ warnIfFilesystemPartitionsExist();
+ }
+ List, Path>> result = new ArrayList<>(partitions.size());
+ Path tablePath = new Path(table.location());
+ // Do not trust the catalog to be duplicate-free: a repeated spec would double every split
+ // of that partition and silently duplicate query results.
+ Set seenPartitionPaths = new HashSet<>(partitions.size());
+ for (Partition partition : partitions) {
+ LinkedHashMap partitionSpec = normalizeSpec(partition.spec());
+ String partitionPath =
+ PartitionPathUtils.generatePartitionPathUtil(
+ partitionSpec, coreOptions.formatTablePartitionOnlyValueInPath());
+ if (!seenPartitionPaths.add(partitionPath)) {
+ continue;
+ }
+ result.add(Pair.of(partitionSpec, new Path(tablePath, partitionPath)));
+ }
+ return result;
+ }
+
+ @Override
+ public List listPartitionEntries() {
+ List partitions = catalogProvider.listPartitions(null);
+ if (partitions.isEmpty()) {
+ warnIfFilesystemPartitionsExist();
+ }
+ List entries = new ArrayList<>(partitions.size());
+ for (Partition partition : partitions) {
+ entries.add(
+ new PartitionEntry(
+ toPartitionRow(normalizeSpec(partition.spec())),
+ partition.recordCount(),
+ partition.fileSizeInBytes(),
+ partition.fileCount(),
+ partition.lastFileCreationTime(),
+ partition.totalBuckets()));
+ }
+ return entries;
+ }
+
+ /**
+ * Warn when the catalog knows no partitions but the table directory contains subdirectories:
+ * typically a table that predates managed partition support (or was written by a client that
+ * does not register partitions) and needs a metadata sync before its data becomes visible.
+ */
+ private void warnIfFilesystemPartitionsExist() {
+ try {
+ for (FileStatus status : table.fileIO().listStatus(new Path(table.location()))) {
+ if (status.isDir() && !status.getPath().getName().startsWith(".")) {
+ LOG.warn(
+ "Managed format table {} has no partitions registered in the catalog "
+ + "but its location {} contains directories. Data written "
+ + "before enabling catalog-managed partitions (or by clients "
+ + "that do not register partitions) is invisible until the "
+ + "partition metadata is synced, e.g. with the Spark procedure "
+ + "sys.sync_format_table_metadata or MSCK REPAIR TABLE.",
+ table.fullName(),
+ table.location());
+ return;
+ }
+ }
+ } catch (IOException ignored) {
+ // Best-effort hint only; never fail or slow down the scan because of it.
+ }
+ }
+
+ /**
+ * Build the partition-name prefix pattern pushed down to the catalog, or {@code null} to list
+ * all partitions without pushdown. See {@link
+ * PartitionPathUtils#buildPartitionNamePrefixPattern} for the pattern contract; this returns
+ * {@code null} when there is no partition predicate, when the predicate has no leading equality
+ * prefix, or when the prefix cannot be expressed in the pattern contract.
+ */
+ @Nullable
+ String createPartitionNamePattern() {
+ Optional predicate = extractPartitionPredicate(partitionFilter);
+ if (!predicate.isPresent()) {
+ return null;
+ }
+
+ Map equalityPrefix =
+ extractLeadingEqualityPartitionSpecWhenOnlyAnd(
+ table.partitionKeys(), predicate.get(), table.partitionType());
+ return PartitionPathUtils.buildPartitionNamePrefixPattern(
+ table.partitionKeys(), equalityPrefix);
+ }
+
+ private LinkedHashMap normalizeSpec(Map spec) {
+ if (spec == null
+ || spec.size() != table.partitionKeys().size()
+ || !spec.keySet().containsAll(table.partitionKeys())) {
+ throw corruptPartitionSpec(spec);
+ }
+ LinkedHashMap normalized = new LinkedHashMap<>();
+ boolean onlyValueInPath = coreOptions.formatTablePartitionOnlyValueInPath();
+ for (String partitionKey : table.partitionKeys()) {
+ String value = spec.get(partitionKey);
+ // Catalog metadata is not trusted for path construction. In a value-only layout,
+ // reject complete path components such as '.' and '..' that would escape the table.
+ try {
+ PartitionPathUtils.validatePartitionValueForPath(value, onlyValueInPath);
+ } catch (IllegalArgumentException e) {
+ throw corruptPartitionSpec(spec);
+ }
+ normalized.put(partitionKey, value);
+ }
+ return normalized;
+ }
+
+ private IllegalStateException corruptPartitionSpec(@Nullable Map spec) {
+ return new IllegalStateException(
+ String.format(
+ "Catalog returned corrupt partition metadata %s for managed format table %s; "
+ + "expected exactly the partition keys %s with values usable as "
+ + "path components.",
+ spec, table.fullName(), table.partitionKeys()));
+ }
+
+ @Override
+ protected void onPartitionFileNotFound(
+ LinkedHashMap partitionSpec,
+ Path partitionPath,
+ FileNotFoundException exception) {
+ // A registered partition without a directory reads as empty, matching Hive semantics
+ // (e.g. ADD PARTITION before the first INSERT). Warn so genuine drift — a directory
+ // removed behind the catalog's back — is still discoverable.
+ LOG.warn(
+ "Partition '{}' of managed format table {} is registered in the catalog but its "
+ + "directory '{}' does not exist; treating the partition as empty. If the "
+ + "directory was removed on purpose, drop the partition or repair the "
+ + "metadata, e.g. with the Spark procedure sys.sync_format_table_metadata "
+ + "or MSCK REPAIR TABLE.",
+ PartitionPathUtils.generatePartitionName(partitionSpec, false),
+ table.fullName(),
+ partitionPath);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
index fb7baf5b06b2..b17168cb3af2 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
@@ -109,13 +109,105 @@ public static String generatePartitionPathUtil(
suffixBuf.append(escapePathName(e.getKey()));
suffixBuf.append('=');
}
- suffixBuf.append(escapePathName(e.getValue()));
+ String value = e.getValue();
+ validatePartitionValueForPath(value, onlyValue);
+ suffixBuf.append(escapePathName(value));
i++;
}
suffixBuf.append(Path.SEPARATOR);
return suffixBuf.toString();
}
+ /**
+ * Generate a partition path without the trailing separator, e.g. {@code dt=20250101/hr=01}.
+ * This is the canonical partition name used when talking to a partition-managing catalog.
+ */
+ public static String generatePartitionName(
+ LinkedHashMap partitionSpec, boolean onlyValue) {
+ String path = generatePartitionPathUtil(partitionSpec, onlyValue);
+ return path.endsWith(Path.SEPARATOR)
+ ? path.substring(0, path.length() - Path.SEPARATOR.length())
+ : path;
+ }
+
+ /**
+ * Validate that a partition value is safe for the configured path layout. In a key-value
+ * layout, values such as {@code "."} are part of a component such as {@code "pt=."} and are
+ * safe. In a value-only layout, {@code "."} and {@code ".."} are complete path components and
+ * would resolve to a different directory.
+ */
+ public static void validatePartitionValueForPath(String value, boolean onlyValueInPath) {
+ if (value == null
+ || value.isEmpty()
+ || (onlyValueInPath && (".".equals(value) || "..".equals(value)))) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Partition value '%s' cannot be used as a partition path component.",
+ value));
+ }
+ }
+
+ /** Conservatively validate a value when the physical partition layout is unknown. */
+ public static void validatePartitionValueForPath(String value) {
+ validatePartitionValueForPath(value, true);
+ }
+
+ /** Validate every value of a partition spec for the configured path layout. */
+ public static void validatePartitionSpecForPath(
+ Map partitionSpec, boolean onlyValueInPath) {
+ for (String value : partitionSpec.values()) {
+ validatePartitionValueForPath(value, onlyValueInPath);
+ }
+ }
+
+ /** Conservatively validate a spec when the physical partition layout is unknown. */
+ public static void validatePartitionSpecForPath(Map partitionSpec) {
+ validatePartitionSpecForPath(partitionSpec, true);
+ }
+
+ /**
+ * Build the partition-name prefix pattern pushed down to a partition-managing catalog from the
+ * leading equality prefix of a partition predicate.
+ *
+ * Pattern contract (shared by every engine talking to the catalog): partition names are the
+ * escaped {@code key=value} form joined by {@code '/'}; {@code '%'} is the only wildcard and
+ * there is no escape sequence for it ({@code '_'} stays a literal). A complete spec matches the
+ * exact partition name; an incomplete prefix is suffixed with {@code '%'}.
+ *
+ *
Returns {@code null} whenever pushdown must be skipped and the caller should list all
+ * partitions instead: the equality prefix is empty, a prefix value is blank, or the escaped
+ * prefix contains a literal {@code '%'} that the contract cannot express.
+ */
+ @Nullable
+ public static String buildPartitionNamePrefixPattern(
+ List partitionKeys, Map equalityPrefix) {
+ if (equalityPrefix.isEmpty()) {
+ return null;
+ }
+ LinkedHashMap orderedPrefix = new LinkedHashMap<>();
+ for (String partitionKey : partitionKeys) {
+ if (!equalityPrefix.containsKey(partitionKey)) {
+ break;
+ }
+ String value = equalityPrefix.get(partitionKey);
+ if (StringUtils.isNullOrWhitespaceOnly(value)) {
+ return null;
+ }
+ orderedPrefix.put(partitionKey, value);
+ }
+ if (orderedPrefix.isEmpty()) {
+ return null;
+ }
+ String escapedPrefix = generatePartitionPath(orderedPrefix);
+ if (escapedPrefix.indexOf('%') >= 0) {
+ return null;
+ }
+ if (orderedPrefix.size() == partitionKeys.size()) {
+ return escapedPrefix.substring(0, escapedPrefix.length() - 1);
+ }
+ return escapedPrefix + '%';
+ }
+
public static List generatePartitionPaths(
List> partitions, RowType partitionType) {
return partitions.stream()
@@ -265,12 +357,45 @@ public static LinkedHashMap extractPartitionSpecFromPath(Path cu
return fullPartSpec;
}
+ /** Extract exactly the trailing key-value components for the declared partition keys. */
+ @Nullable
+ static LinkedHashMap extractPartitionSpecFromPath(
+ Path currPath, List partitionKeys) {
+ String[] values = new String[partitionKeys.size()];
+ Path current = currPath;
+ for (int i = partitionKeys.size() - 1; i >= 0; i--) {
+ if (current == null) {
+ return null;
+ }
+ Matcher matcher = PARTITION_NAME_PATTERN.matcher(current.getName());
+ if (!matcher.matches()
+ || !partitionKeys.get(i).equals(unescapePathName(matcher.group(1)))) {
+ return null;
+ }
+ values[i] = unescapePathName(matcher.group(2));
+ current = current.getParent();
+ }
+
+ LinkedHashMap spec = new LinkedHashMap<>();
+ for (int i = 0; i < partitionKeys.size(); i++) {
+ spec.put(partitionKeys.get(i), values[i]);
+ }
+ return spec;
+ }
+
public static LinkedHashMap extractPartitionSpecFromPathOnlyValue(
Path currPath, List partitionKeys) {
LinkedHashMap fullPartSpec = new LinkedHashMap<>();
String[] split = currPath.toString().split(Path.SEPARATOR);
for (int i = 0; i < partitionKeys.size(); i++) {
- fullPartSpec.put(partitionKeys.get(i), split[split.length - partitionKeys.size() + i]);
+ // Unescape the directory component so the extracted value is the RAW partition value,
+ // consistent with the key=value branch (extractPartitionSpecFromPath) and with the
+ // values the write path registers into a partition-managing catalog. Without this,
+ // directories containing escaped characters (e.g. a%3Ab) would round-trip to a
+ // different value than the one registered (a:b).
+ fullPartSpec.put(
+ partitionKeys.get(i),
+ unescapePathName(split[split.length - partitionKeys.size() + i]));
}
return fullPartSpec;
}
@@ -330,8 +455,9 @@ public static List, Path>> searchPartSpecAndP
part.getPath(), partitionKeys),
part.getPath()));
} else {
- LinkedHashMap spec = extractPartitionSpecFromPath(part.getPath());
- if (spec.size() != partitionKeys.size()) {
+ LinkedHashMap spec =
+ extractPartitionSpecFromPath(part.getPath(), partitionKeys);
+ if (spec == null) {
// illegal path, for example: /path/to/table/tmp/unknown, path without "="
continue;
}
@@ -360,8 +486,17 @@ private static FileStatus[] getFileStatusRecurse(
GenericRow values =
partitionType == null ? null : new GenericRow(partitionType.getFieldCount());
+ FileStatus fileStatus;
+ try {
+ fileStatus = fileIO.getFileStatus(path);
+ } catch (FileNotFoundException e) {
+ // A missing root simply means the table has no partitions yet.
+ return new FileStatus[0];
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to list files in " + path, e);
+ }
+
try {
- FileStatus fileStatus = fileIO.getFileStatus(path);
// Skip partition levels already fixed by the scan-path prefix.
int levelOffset = partitionKeys.size() - expectLevel;
listStatusRecursively(
@@ -378,9 +513,10 @@ private static FileStatus[] getFileStatusRecurse(
defaultPartValue,
levelOffset,
values);
- } catch (FileNotFoundException e) {
- return new FileStatus[0];
} catch (IOException e) {
+ // Never degrade a mid-scan failure into an empty listing: callers diff this result
+ // against partition metadata and an incomplete listing would deregister partitions
+ // that still exist.
throw new RuntimeException("Failed to list files in " + path, e);
}
@@ -412,7 +548,15 @@ private static void listStatusRecursively(
}
if (fileStatus.isDir()) {
- for (FileStatus stat : fileIO.listStatus(fileStatus.getPath())) {
+ FileStatus[] children;
+ try {
+ children = fileIO.listStatus(fileStatus.getPath());
+ } catch (FileNotFoundException e) {
+ // The directory vanished after the parent listed it: the partitions beneath it
+ // are gone, skipping just this subtree keeps the rest of the listing complete.
+ return;
+ }
+ for (FileStatus stat : children) {
int partitionKeyIndex = levelOffset + level;
String partitionKey = partitionKeys.get(partitionKeyIndex);
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
index 0961050f110d..3e91251ae812 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
@@ -368,6 +368,32 @@ public void testCreatePartitionsWithIgnoreIfExistsInvalidatesPartitionCache() th
assertThat(catalog.listPartitions(identifier)).containsExactly(created);
}
+ @Test
+ public void testCreatePartitionsFailureInvalidatesPartitionCache() throws Exception {
+ Catalog wrapped = Mockito.mock(Catalog.class);
+ TestableCachingCatalog catalog =
+ new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker);
+ Identifier identifier = new Identifier("db", "tbl");
+ Map spec = singletonMap("dt", "20260717");
+ Partition created = new Partition(spec, 0, 0, 0, 0, -1, false);
+ List stored = new ArrayList<>();
+ when(wrapped.listPartitions(identifier)).thenAnswer(ignored -> new ArrayList<>(stored));
+ RuntimeException responseLost = new RuntimeException("response lost");
+ Mockito.doAnswer(
+ ignored -> {
+ stored.add(created);
+ throw responseLost;
+ })
+ .when(wrapped)
+ .createPartitions(identifier, singletonList(spec));
+
+ assertThat(catalog.listPartitions(identifier)).isEmpty();
+ assertThatThrownBy(() -> catalog.createPartitions(identifier, singletonList(spec)))
+ .isSameAs(responseLost);
+
+ assertThat(catalog.listPartitions(identifier)).containsExactly(created);
+ }
+
@Test
public void testDeadlock() throws Exception {
Catalog underlyCatalog = this.catalog;
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java
new file mode 100644
index 000000000000..edcc79af157c
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.catalog;
+
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import static org.apache.paimon.CoreOptions.FILE_FORMAT;
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_IMPLEMENTATION;
+import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
+import static org.apache.paimon.CoreOptions.PATH;
+import static org.apache.paimon.CoreOptions.TYPE;
+import static org.apache.paimon.TableType.FORMAT_TABLE;
+import static org.apache.paimon.catalog.CatalogUtils.loadTable;
+import static org.apache.paimon.catalog.CatalogUtils.validateCreateTable;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/** Tests for {@link CatalogUtils}. */
+class CatalogUtilsTest {
+
+ @Test
+ void testRejectEngineImplementationForManagedFormatTableOnCreate() {
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .option(TYPE.key(), FORMAT_TABLE.toString())
+ .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(FORMAT_TABLE_IMPLEMENTATION.key(), "engine")
+ .build();
+
+ assertThatThrownBy(() -> validateCreateTable(schema, false))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(METASTORE_PARTITIONED_TABLE.key())
+ .hasMessageContaining(FORMAT_TABLE_IMPLEMENTATION.key());
+ }
+
+ @Test
+ void testEngineImplementationManagedFormatTableLoadsAsUnmanaged() throws Exception {
+ // The option combination is invalid for managed partitions, but an existing table must
+ // stay loadable: it degrades to an unmanaged format table instead of failing.
+ Table table = loadManagedFormatTable("engine", true);
+ assertUnmanagedFormatTable(table);
+ }
+
+ @Test
+ void testManagedFormatTableOutsideCapableCatalogLoadsAsUnmanaged() throws Exception {
+ // A catalog that cannot manage partitions (e.g. Hive) must not reject existing tables that
+ // carry the (previously inert) option; it loads them as unmanaged format tables.
+ Table table = loadManagedFormatTable("paimon", false);
+ assertUnmanagedFormatTable(table);
+ }
+
+ @Test
+ void testLoadManagedFormatTableFromCapableCatalog() throws Exception {
+ Table table = loadManagedFormatTable("paimon", true);
+ assertThat(table).isInstanceOf(FormatTable.class);
+ FormatTable formatTable = (FormatTable) table;
+ assertThat(
+ new org.apache.paimon.CoreOptions(formatTable.options())
+ .partitionedTableInMetastore())
+ .isTrue();
+ assertThat(formatTable.catalogProvider()).isNotNull();
+ }
+
+ private static void assertUnmanagedFormatTable(Table table) {
+ assertThat(table).isInstanceOf(FormatTable.class);
+ FormatTable formatTable = (FormatTable) table;
+ assertThat(
+ new org.apache.paimon.CoreOptions(formatTable.options())
+ .partitionedTableInMetastore())
+ .isFalse();
+ assertThat(formatTable.catalogProvider()).isNull();
+ }
+
+ private static Table loadManagedFormatTable(
+ String formatTableImplementation, boolean supportsManagedPartitions) throws Exception {
+ Identifier identifier = Identifier.create("managed_db", "managed_table");
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .option(TYPE.key(), FORMAT_TABLE.toString())
+ .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(FORMAT_TABLE_IMPLEMENTATION.key(), formatTableImplementation)
+ .option(FILE_FORMAT.key(), "parquet")
+ .option(PATH.key(), "file:///tmp/managed_db.db/managed_table")
+ .build();
+ TableMetadata metadata =
+ new TableMetadata(TableSchema.create(0, schema), false, "managed-table-id");
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.catalogLoader()).thenReturn(() -> catalog);
+ when(catalog.supportsManagedFormatTablePartitions()).thenReturn(supportsManagedPartitions);
+
+ return loadTable(
+ catalog,
+ identifier,
+ path -> new LocalFileIO(),
+ path -> new LocalFileIO(),
+ ignored -> metadata,
+ null,
+ null,
+ null,
+ supportsManagedPartitions);
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
index 04f67c018d1b..7bb122e1fea6 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
@@ -18,9 +18,15 @@
package org.apache.paimon.catalog;
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.TableType;
import org.apache.paimon.fs.Path;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaChange;
+import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.Table;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
@@ -28,6 +34,7 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for {@link FileSystemCatalog}. */
@@ -77,4 +84,49 @@ public void testAlterDatabase() throws Exception {
false))
.isInstanceOf(UnsupportedOperationException.class);
}
+
+ @Test
+ public void testResetManagedPartitionsOnStrandedFormatTable() throws Exception {
+ String database = "stranded_managed_format_table_db";
+ Identifier identifier = Identifier.create(database, "stranded_format_table");
+ catalog.createDatabase(database, false);
+ // Write the schema file directly to simulate a format table stranded with the
+ // REST-only option in a filesystem catalog.
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
+ .option(CoreOptions.FILE_FORMAT.key(), "parquet")
+ .option(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true")
+ .build();
+ Path tablePath =
+ ((FileSystemCatalog) DelegateCatalog.rootCatalog(catalog))
+ .getTableLocation(identifier);
+ new SchemaManager(fileIO, tablePath).createTable(schema);
+ // A catalog that cannot manage partitions must still load such a stranded table: the
+ // option is ignored and the table degrades to an unmanaged format table (readable), rather
+ // than becoming permanently unreadable.
+ Table stranded = catalog.getTable(identifier);
+ assertThat(stranded).isInstanceOf(FormatTable.class);
+ assertThat(new CoreOptions(stranded.options()).partitionedTableInMetastore()).isFalse();
+
+ // Alters that do not touch managed-sensitive options skip the managed validation.
+ catalog.alterTable(
+ identifier,
+ Lists.newArrayList(SchemaChange.setOption("custom.unrelated-option", "value")),
+ false);
+
+ // The documented remediation must succeed on a non-REST catalog.
+ catalog.alterTable(
+ identifier,
+ Lists.newArrayList(
+ SchemaChange.removeOption(CoreOptions.METASTORE_PARTITIONED_TABLE.key())),
+ false);
+
+ assertThat(catalog.getTable(identifier).options())
+ .doesNotContainKey(CoreOptions.METASTORE_PARTITIONED_TABLE.key())
+ .containsEntry("custom.unrelated-option", "value");
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
index b4af3194b72f..c05f45875ed4 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
@@ -26,6 +26,8 @@
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.catalog.TableQueryAuthResult;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.FieldRef;
@@ -46,6 +48,8 @@
import org.apache.paimon.rest.exceptions.NotImplementedException;
import org.apache.paimon.rest.responses.ConfigResponse;
import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.format.FormatTableCatalogProvider;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.JsonSerdeUtil;
@@ -247,6 +251,96 @@ void testManagedFormatTablePartitionListingDoesNotFallback() throws Exception {
.isInstanceOf(NotImplementedException.class);
}
+ @Test
+ void testCreatePartitionFailureInvalidatesManagedListingCache() throws Exception {
+ Identifier identifier = createManagedFormatTable();
+ FormatTable table = (FormatTable) restCatalog.getTable(identifier);
+ FormatTableCatalogProvider provider = table.catalogProvider();
+ assertThat(provider).isNotNull();
+ assertThat(provider.listPartitions(null)).isEmpty();
+ Map partition = Collections.singletonMap("dt", "20260717");
+ restCatalogServer.failNextCreatePartitionsResponse();
+
+ assertThatThrownBy(
+ () ->
+ restCatalog.createPartitions(
+ identifier, Collections.singletonList(partition)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Injected response failure");
+
+ assertThat(provider.listPartitions(null))
+ .extracting(org.apache.paimon.partition.Partition::spec)
+ .containsExactly(partition);
+ }
+
+ @Test
+ void testDropPartitionFailureInvalidatesManagedListingCache() throws Exception {
+ Identifier identifier = createManagedFormatTable();
+ FormatTable table = (FormatTable) restCatalog.getTable(identifier);
+ FormatTableCatalogProvider provider = table.catalogProvider();
+ assertThat(provider).isNotNull();
+ Map partition = Collections.singletonMap("dt", "20260717");
+ restCatalog.createPartitions(identifier, Collections.singletonList(partition));
+ assertThat(provider.listPartitions(null))
+ .extracting(org.apache.paimon.partition.Partition::spec)
+ .containsExactly(partition);
+ restCatalogServer.failNextDropPartitionsResponse();
+
+ assertThatThrownBy(
+ () ->
+ restCatalog.dropPartitions(
+ identifier, Collections.singletonList(partition)))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Injected response failure");
+
+ assertThat(provider.listPartitions(null)).isEmpty();
+ }
+
+ @Test
+ void testRejectExternalManagedFormatTableBeforeCreate() throws Exception {
+ Identifier identifier = Identifier.create("db1", "external_managed_format_table");
+ restCatalog.createDatabase(identifier.getDatabaseName(), true);
+ String externalPath = dataPath + "/external-managed-format-table";
+ Schema schema =
+ Schema.newBuilder()
+ .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
+ .option(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(CoreOptions.FILE_FORMAT.key(), "parquet")
+ .option(CoreOptions.PATH.key(), externalPath)
+ .column("id", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .build();
+
+ assertThatThrownBy(() -> restCatalog.createTable(identifier, schema, false))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("internal table");
+ assertThat(restCatalog.listTables(identifier.getDatabaseName()))
+ .doesNotContain(identifier.getTableName());
+ assertThat(LocalFileIO.create().exists(new Path(externalPath))).isFalse();
+ }
+
+ @Test
+ void testRoundTrippedManagedFormatTableReplacePassesClientValidation() throws Exception {
+ Identifier identifier = createManagedFormatTable();
+ FormatTable existing = (FormatTable) restCatalog.getTable(identifier);
+ Schema replacement =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .options(existing.options())
+ .build();
+ assertThat(replacement.options())
+ .containsEntry(CoreOptions.PATH.key(), existing.location());
+
+ // The mock service does not implement Format Table replacement. Reaching that response
+ // proves the REST client accepted the unchanged synthetic path from the loaded table.
+ assertThatThrownBy(() -> restCatalog.replaceTable(identifier, replacement, false))
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining("replaceTable does not support format tables");
+ }
+
private Identifier createManagedFormatTable() throws Exception {
Identifier identifier = Identifier.create("db1", "managed_partition_table");
restCatalog.createDatabase(identifier.getDatabaseName(), true);
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
index 3a0e1539eea1..385b27cb9fbf 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
@@ -212,6 +212,9 @@ public class RESTCatalogServer {
private final List> receivedHeaders = new ArrayList<>();
private volatile boolean partitionListingSupported = true;
+ private volatile boolean failNextCreatePartitionsResponse;
+ private volatile boolean failNextDropPartitionsResponse;
+ private volatile int tableGetCount;
public RESTCatalogServer(
String dataPath, AuthProvider authProvider, ConfigResponse config, String warehouse) {
@@ -279,6 +282,22 @@ public void setPartitionListingSupported(boolean partitionListingSupported) {
this.partitionListingSupported = partitionListingSupported;
}
+ public void failNextCreatePartitionsResponse() {
+ this.failNextCreatePartitionsResponse = true;
+ }
+
+ public void failNextDropPartitionsResponse() {
+ this.failNextDropPartitionsResponse = true;
+ }
+
+ public int tableGetCount() {
+ return tableGetCount;
+ }
+
+ public void resetTableGetCount() {
+ tableGetCount = 0;
+ }
+
public void addNoPermissionDatabase(String database) {
noPermissionDatabases.add(database);
}
@@ -1733,6 +1752,7 @@ private MockResponse tableHandle(String method, String data, Identifier identifi
}
switch (method) {
case "GET":
+ tableGetCount++;
TableMetadata tableMetadata;
if (identifier.isSystemTable()) {
TableSchema schema = catalog.loadTableSchema(identifier);
@@ -1897,6 +1917,16 @@ private MockResponse partitionsApiHandle(
existed.add(spec);
}
}
+ if (failNextCreatePartitionsResponse) {
+ failNextCreatePartitionsResponse = false;
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponse.RESOURCE_TYPE_PARTITION,
+ tableIdentifier.getFullName(),
+ "Injected response failure after creating partitions.",
+ 400),
+ 400);
+ }
return mockResponse(new CreatePartitionsResponse(created, existed), 200);
default:
return new MockResponse().setResponseCode(404);
@@ -1940,6 +1970,16 @@ private MockResponse dropPartitionsHandle(String data, Identifier tableIdentifie
}
return false;
});
+ if (failNextDropPartitionsResponse) {
+ failNextDropPartitionsResponse = false;
+ return mockResponse(
+ new ErrorResponse(
+ ErrorResponse.RESOURCE_TYPE_PARTITION,
+ tableIdentifier.getFullName(),
+ "Injected response failure after dropping partitions.",
+ 400),
+ 400);
+ }
return mockResponse(new DropPartitionsResponse(dropped, missing), 200);
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
index 504dc6fab6ec..3f13c1cab844 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
@@ -76,6 +76,7 @@
import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.FormatTable;
import org.apache.paimon.table.Instant;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.TableSnapshot;
@@ -1639,6 +1640,63 @@ void testDropPartitionsLoadsNonManagedTableOnce() throws Exception {
Mockito.verify(catalogSpy, Mockito.times(1)).getTable(identifier);
}
+ @Test
+ void testManagedFormatTableCommitAndScanMatchesFileSystemMode() throws Exception {
+ Identifier identifier = Identifier.create("format_partition_db", "managed_scan_table");
+ catalog.createDatabase(identifier.getDatabaseName(), true);
+ catalog.createTable(
+ identifier,
+ Schema.newBuilder()
+ .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
+ .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(CoreOptions.FILE_FORMAT.key(), "csv")
+ .column("id", DataTypes.INT())
+ .column("year", DataTypes.INT())
+ .column("month", DataTypes.INT())
+ .partitionKeys("year", "month")
+ .build(),
+ false);
+
+ FormatTable managedTable = (FormatTable) catalog.getTable(identifier);
+ BatchWriteBuilder writeBuilder = managedTable.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ write.write(GenericRow.of(1, 2024, 10));
+ write.write(GenericRow.of(2, 2025, 10));
+ write.write(GenericRow.of(3, 2025, 11));
+ commit.commit(write.prepareCommit());
+ }
+
+ List> registeredPartitions =
+ Arrays.asList(
+ ImmutableMap.of("year", "2024", "month", "10"),
+ ImmutableMap.of("year", "2025", "month", "10"),
+ ImmutableMap.of("year", "2025", "month", "11"));
+ assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec))
+ .containsExactlyInAnyOrderElementsOf(registeredPartitions);
+ assertThat(managedTable.newReadBuilder().newScan().getClass().getSimpleName())
+ .isEqualTo("ManagedFormatTableScan");
+
+ Map fileSystemOptions = new HashMap<>(managedTable.options());
+ fileSystemOptions.put(METASTORE_PARTITIONED_TABLE.key(), "false");
+ FormatTable fileSystemTable =
+ FormatTable.builder()
+ .fileIO(managedTable.fileIO())
+ .identifier(Identifier.create("format_partition_db", "filesystem_scan"))
+ .rowType(managedTable.rowType())
+ .partitionKeys(managedTable.partitionKeys())
+ .location(managedTable.location())
+ .format(managedTable.format())
+ .options(fileSystemOptions)
+ .catalogContext(managedTable.catalogContext())
+ .build();
+
+ Map partitionFilter = singletonMap("year", "2025");
+ assertThat(read(managedTable, null, null, partitionFilter, null))
+ .containsExactlyInAnyOrderElementsOf(
+ read(fileSystemTable, null, null, partitionFilter, null));
+ }
+
@Test
void testListPartitions() throws Exception {
innerTestListPartitions(true);
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java b/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java
new file mode 100644
index 000000000000..29fe9f4fd7fe
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table;
+
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.table.format.FormatTableCatalogProvider;
+import org.apache.paimon.table.format.FormatTableCommit;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH;
+import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
+import static org.apache.paimon.CoreOptions.READ_BATCH_SIZE;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Compatibility tests for the public {@link FormatTable} API. */
+class FormatTableCompatibilityTest {
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testCopyRejectsManagedPartitionStateChange(boolean managed) {
+ FormatTable table = formatTable(Boolean.toString(managed));
+
+ assertThatThrownBy(
+ () ->
+ table.copy(
+ Collections.singletonMap(
+ METASTORE_PARTITIONED_TABLE.key(),
+ Boolean.toString(!managed))))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(METASTORE_PARTITIONED_TABLE.key());
+ }
+
+ @Test
+ void testCopyAllowsSemanticallyEquivalentManagedPartitionOption() {
+ FormatTable table = formatTable("TRUE");
+
+ FormatTable copied =
+ table.copy(Collections.singletonMap(METASTORE_PARTITIONED_TABLE.key(), "true"));
+
+ assertThat(copied).isNotSameAs(table);
+ assertThat(copied.options())
+ .containsEntry(METASTORE_PARTITIONED_TABLE.key(), Boolean.TRUE.toString());
+ }
+
+ @Test
+ void testCopyRejectsChangingAbsentManagedPartitionDefault() {
+ FormatTable table = formatTable(Collections.emptyMap(), null);
+
+ assertThat(table.options()).doesNotContainKey(METASTORE_PARTITIONED_TABLE.key());
+ assertThatThrownBy(
+ () ->
+ table.copy(
+ Collections.singletonMap(
+ METASTORE_PARTITIONED_TABLE.key(), "true")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(METASTORE_PARTITIONED_TABLE.key());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testManagedCopyRejectsPhysicalPartitionLayoutChange(boolean onlyValueInPath) {
+ Map options = new LinkedHashMap<>();
+ options.put(METASTORE_PARTITIONED_TABLE.key(), "true");
+ options.put(
+ FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), Boolean.toString(onlyValueInPath));
+ FormatTable table = formatTable(options, null);
+
+ assertThatThrownBy(
+ () ->
+ table.copy(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.toString(!onlyValueInPath))))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testCopyPreservesManagedStateAndCatalogProviderForUnrelatedOption(boolean managed) {
+ Identifier identifier = Identifier.create("managed_db", "managed_table");
+ FormatTableCatalogProvider catalogProvider =
+ new FormatTableCatalogProvider(identifier, () -> null);
+ Map options = new LinkedHashMap<>();
+ options.put(METASTORE_PARTITIONED_TABLE.key(), Boolean.toString(managed));
+ FormatTable table = formatTable(options, catalogProvider);
+
+ FormatTable copied = table.copy(Collections.singletonMap(READ_BATCH_SIZE.key(), "128"));
+
+ assertThat(copied).isNotSameAs(table);
+ assertThat(copied.options())
+ .containsEntry(READ_BATCH_SIZE.key(), "128")
+ .containsEntry(METASTORE_PARTITIONED_TABLE.key(), Boolean.toString(managed));
+ assertThat(
+ new org.apache.paimon.CoreOptions(copied.options())
+ .partitionedTableInMetastore())
+ .isEqualTo(managed);
+ assertThat(copied.catalogProvider()).isSameAs(catalogProvider);
+ assertThat(table.options())
+ .containsExactlyEntriesOf(options)
+ .doesNotContainKey(READ_BATCH_SIZE.key());
+ }
+
+ @Test
+ void testManagedCatalogExtensionKeepsExistingImplementationsCompatible() throws Exception {
+ assertThat(FormatTable.class.getMethod("catalogProvider").isDefault()).isTrue();
+ assertThat(
+ FormatTable.FormatTableImpl.class.getConstructor(
+ FileIO.class,
+ Identifier.class,
+ RowType.class,
+ List.class,
+ String.class,
+ FormatTable.Format.class,
+ Map.class,
+ String.class,
+ CatalogContext.class))
+ .isNotNull();
+ assertThat(
+ FormatTableCommit.class.getConstructor(
+ String.class,
+ List.class,
+ FileIO.class,
+ boolean.class,
+ boolean.class,
+ Identifier.class,
+ Map.class,
+ String.class,
+ CatalogContext.class))
+ .isNotNull();
+ }
+
+ private static FormatTable formatTable(String managed) {
+ return formatTable(
+ Collections.singletonMap(METASTORE_PARTITIONED_TABLE.key(), managed), null);
+ }
+
+ private static FormatTable formatTable(
+ Map options, FormatTableCatalogProvider catalogProvider) {
+ return FormatTable.builder()
+ .fileIO(LocalFileIO.create())
+ .identifier(Identifier.create("managed_db", "managed_table"))
+ .rowType(
+ RowType.builder()
+ .field("id", DataTypes.INT())
+ .field("dt", DataTypes.STRING())
+ .build())
+ .partitionKeys(Collections.singletonList("dt"))
+ .location("file:///warehouse/managed_table")
+ .format(FormatTable.Format.PARQUET)
+ .options(options)
+ .catalogProvider(catalogProvider)
+ .build();
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java
new file mode 100644
index 000000000000..2bc74ca99bd6
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.RenamingTwoPhaseOutputStream;
+import org.apache.paimon.fs.TwoPhaseOutputStream;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.utils.PartitionPathUtils;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.entry;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+/** Tests for {@link FormatTableCommit}. */
+class FormatTableCommitTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ void testPartitionRegistrationFailurePreservesCommittedFiles() throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path targetPath = new Path(tablePath, "year=2025/month=10/data-1.csv");
+ RenamingTwoPhaseOutputStream outputStream =
+ new RenamingTwoPhaseOutputStream(fileIO, targetPath, false);
+ outputStream.write(1);
+ TwoPhaseOutputStream.Committer committer = outputStream.closeForCommit();
+ FormatTableCatalogProvider catalogProvider = mock(FormatTableCatalogProvider.class);
+ RuntimeException registrationFailure =
+ new RuntimeException("Catalog partition registration unavailable");
+ doThrow(registrationFailure).when(catalogProvider).createPartitions(anyList());
+
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Arrays.asList("year", "month"),
+ fileIO,
+ false,
+ false,
+ Identifier.create("managed_db", "managed_table"),
+ null,
+ null,
+ null,
+ catalogProvider);
+ CommitMessage message = new TwoPhaseCommitMessage(committer);
+
+ assertThatThrownBy(() -> commit.commit(Collections.singletonList(message)))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("Committed data files were preserved")
+ .hasMessageContaining("managed_db.managed_table")
+ .hasMessageContaining("MSCK REPAIR TABLE")
+ .hasRootCauseMessage("Catalog partition registration unavailable");
+
+ assertThat(fileIO.exists(targetPath)).isTrue();
+ verify(catalogProvider).createPartitions(anyList());
+
+ // Flink calls abort on the same commit object after a failed commit.
+ commit.abort(Collections.singletonList(message));
+ assertThat(fileIO.exists(targetPath)).isTrue();
+ }
+
+ @Test
+ void testFileCommitFailureStillDiscardsUncommittedFiles() throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ doThrow(new IOException("data commit failed")).when(committer).commit(fileIO);
+ FormatTableCatalogProvider catalogProvider = mock(FormatTableCatalogProvider.class);
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Arrays.asList("year", "month"),
+ fileIO,
+ false,
+ false,
+ Identifier.create("managed_db", "managed_table"),
+ null,
+ null,
+ null,
+ catalogProvider);
+ CommitMessage message = new TwoPhaseCommitMessage(committer);
+
+ assertThatThrownBy(() -> commit.commit(Collections.singletonList(message)))
+ .isInstanceOf(RuntimeException.class)
+ .hasRootCauseMessage("data commit failed");
+
+ verify(committer).discard(fileIO);
+ verify(catalogProvider, never()).createPartitions(anyList());
+ }
+
+ @Test
+ void testRegistersRawPartitionValuesForEscapedPath() throws Exception {
+ Path tablePath = new Path(tempDir.toUri());
+ LinkedHashMap rawSpec = new LinkedHashMap<>();
+ rawSpec.put("year", "2025");
+ rawSpec.put("month", "a b:c");
+ // The writer escapes partition values when building the directory layout.
+ String partitionDir = PartitionPathUtils.generatePartitionPathUtil(rawSpec, false);
+ assertThat(partitionDir).isEqualTo("year=2025/month=a b%3Ac/");
+
+ FormatTableCatalogProvider catalogProvider =
+ commitPartitionedFile(tablePath, false, partitionDir);
+
+ // The catalog must receive RAW values; readers re-escape them when probing directories.
+ assertThat(registeredSpec(catalogProvider))
+ .containsExactly(entry("year", "2025"), entry("month", "a b:c"));
+ }
+
+ @Test
+ void testForeignKeyValueSegmentsInLocationDoNotLeakIntoSpec() throws Exception {
+ Path tablePath = new Path(new Path(tempDir.toUri()), "env=prod/warehouse/tbl");
+
+ FormatTableCatalogProvider catalogProvider =
+ commitPartitionedFile(tablePath, false, "year=2025/month=10");
+
+ assertThat(registeredSpec(catalogProvider))
+ .containsExactly(entry("year", "2025"), entry("month", "10"));
+ }
+
+ @Test
+ void testValueOnlyPathUnderForeignKeyValueSegmentRegistersRawValues() throws Exception {
+ Path tablePath = new Path(new Path(tempDir.toUri()), "env=prod/warehouse/tbl");
+ LinkedHashMap rawSpec = new LinkedHashMap<>();
+ rawSpec.put("year", "2025");
+ rawSpec.put("month", "a:b");
+ String partitionDir = PartitionPathUtils.generatePartitionPathUtil(rawSpec, true);
+ assertThat(partitionDir).isEqualTo("2025/a%3Ab/");
+
+ FormatTableCatalogProvider catalogProvider =
+ commitPartitionedFile(tablePath, true, partitionDir);
+
+ assertThat(registeredSpec(catalogProvider))
+ .containsExactly(entry("year", "2025"), entry("month", "a:b"));
+ }
+
+ @Test
+ void testMismatchedPartitionKeyInPathFailsWithClearMessage() {
+ Path tablePath = new Path(tempDir.toUri());
+
+ assertThatThrownBy(() -> commitPartitionedFile(tablePath, false, "year=2025/day=10"))
+ .isInstanceOf(RuntimeException.class)
+ .hasRootCauseInstanceOf(IllegalArgumentException.class)
+ .rootCause()
+ .hasMessageContaining("declares partition key 'day'")
+ .hasMessageContaining("partition key 'month' was expected");
+ }
+
+ @Test
+ void testValueOnlyStaticPartitionCannotEscapeTableLocation() throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path parentPath = new Path(tempDir.toUri());
+ Path tablePath = new Path(parentPath, "table");
+ Path siblingPath = new Path(parentPath, "keep");
+ fileIO.mkdirs(tablePath);
+ fileIO.mkdirs(siblingPath);
+ Map staticPartition = Collections.singletonMap("year", "..");
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("year"),
+ fileIO,
+ true,
+ true,
+ Identifier.create("managed_db", "managed_table"),
+ staticPartition,
+ null,
+ null,
+ null);
+
+ assertThatThrownBy(() -> commit.commit(Collections.emptyList()))
+ .isInstanceOf(RuntimeException.class)
+ .hasRootCauseInstanceOf(IllegalArgumentException.class)
+ .hasRootCauseMessage(
+ "Partition value '..' cannot be used as a partition path component.");
+ assertThat(fileIO.exists(tablePath)).isTrue();
+ assertThat(fileIO.exists(siblingPath)).isTrue();
+ }
+
+ private FormatTableCatalogProvider commitPartitionedFile(
+ Path tableLocation, boolean onlyValueInPath, String partitionDir) throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path targetPath = new Path(new Path(tableLocation, partitionDir), "data-1.csv");
+ RenamingTwoPhaseOutputStream outputStream =
+ new RenamingTwoPhaseOutputStream(fileIO, targetPath, false);
+ outputStream.write(1);
+ TwoPhaseOutputStream.Committer committer = outputStream.closeForCommit();
+ FormatTableCatalogProvider catalogProvider = mock(FormatTableCatalogProvider.class);
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tableLocation.toString(),
+ Arrays.asList("year", "month"),
+ fileIO,
+ onlyValueInPath,
+ false,
+ Identifier.create("managed_db", "managed_table"),
+ null,
+ null,
+ null,
+ catalogProvider);
+ commit.commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+ return catalogProvider;
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private static Map registeredSpec(FormatTableCatalogProvider catalogProvider) {
+ ArgumentCaptor>> captor =
+ ArgumentCaptor.forClass((Class) List.class);
+ verify(catalogProvider).createPartitions(captor.capture());
+ assertThat(captor.getValue()).hasSize(1);
+ return captor.getValue().get(0);
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/ManagedFormatTableScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/ManagedFormatTableScanTest.java
new file mode 100644
index 000000000000..0ca741386eed
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/ManagedFormatTableScanTest.java
@@ -0,0 +1,553 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.PagedList;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.fs.FileStatus;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.manifest.PartitionEntry;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.source.Split;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.PartitionPathUtils;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/** Tests for managed format table partition discovery. */
+class ManagedFormatTableScanTest {
+
+ private static final Identifier IDENTIFIER = Identifier.create("managed_db", "managed_table");
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ void testProviderRegistersLargeBatchesInBoundedRequests() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ FormatTableCatalogProvider provider = provider(catalog);
+ List> specs = new ArrayList<>();
+ for (int index = 0; index < 2500; index++) {
+ specs.add(Collections.singletonMap("year", String.format("y%04d", index)));
+ }
+
+ provider.createPartitions(specs);
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ org.mockito.ArgumentCaptor>> batches =
+ (org.mockito.ArgumentCaptor) org.mockito.ArgumentCaptor.forClass(List.class);
+ verify(catalog, times(3)).createPartitions(eq(IDENTIFIER), batches.capture());
+ assertThat(batches.getAllValues().get(0)).hasSize(1000);
+ assertThat(batches.getAllValues().get(1)).hasSize(1000);
+ assertThat(batches.getAllValues().get(2)).hasSize(500);
+ assertThat(
+ batches.getAllValues().stream()
+ .flatMap(List::stream)
+ .collect(java.util.stream.Collectors.toList()))
+ .containsExactlyElementsOf(specs);
+ }
+
+ @Test
+ void testProviderPaginatesAndCachesMatchingPattern() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition october = partition("2025", "10");
+ Partition november = partition("2025", "11");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%")))
+ .thenReturn(new PagedList<>(Collections.singletonList(october), "next"));
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), eq("next"), eq("year=2025/%")))
+ .thenReturn(new PagedList<>(Collections.singletonList(november), null));
+
+ FormatTableCatalogProvider provider = provider(catalog);
+
+ assertThat(provider.listPartitions("year=2025/%")).containsExactly(october, november);
+ assertThat(provider.listPartitions("year=2025/%")).containsExactly(october, november);
+ verify(catalog, times(1)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%");
+ verify(catalog, times(1)).listPartitionsPaged(IDENTIFIER, 1000, "next", "year=2025/%");
+ }
+
+ @Test
+ void testProviderRejectsRepeatedPageTokenCycle() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.emptyList(), "a"));
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), eq("a"), isNull()))
+ .thenReturn(new PagedList<>(Collections.emptyList(), "b"))
+ .thenThrow(new AssertionError("provider requested page token 'a' twice"));
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), eq("b"), isNull()))
+ .thenReturn(new PagedList<>(Collections.emptyList(), "a"));
+
+ assertThatThrownBy(() -> provider(catalog).listPartitions(null))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("repeated partition page token 'a'")
+ .hasMessageContaining(IDENTIFIER.getFullName());
+ }
+
+ @Test
+ void testCreatePartitionsInvalidatesCachedPattern() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition oldPartition = partition("2025", "10");
+ Partition newPartition = partition("2025", "11");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%")))
+ .thenReturn(
+ new PagedList<>(Collections.singletonList(oldPartition), null),
+ new PagedList<>(Arrays.asList(oldPartition, newPartition), null));
+
+ FormatTableCatalogProvider provider = provider(catalog);
+ assertThat(provider.listPartitions("year=2025/%")).containsExactly(oldPartition);
+ assertThat(provider.listPartitions("year=2025/%")).containsExactly(oldPartition);
+
+ List> created = Collections.singletonList(newPartition.spec());
+ provider.createPartitions(created);
+
+ assertThat(provider.listPartitions("year=2025/%"))
+ .containsExactly(oldPartition, newPartition);
+ verify(catalog).createPartitions(IDENTIFIER, created);
+ verify(catalog, times(2)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%");
+ }
+
+ @Test
+ void testCreatePartitionsInvalidatesOtherProviderInSameProcess() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition oldPartition = partition("2025", "10");
+ Partition newPartition = partition("2025", "11");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%")))
+ .thenReturn(
+ new PagedList<>(Collections.singletonList(oldPartition), null),
+ new PagedList<>(Arrays.asList(oldPartition, newPartition), null));
+ FormatTableCatalogProvider readerProvider = provider(catalog);
+ FormatTableCatalogProvider writerProvider = provider(catalog);
+
+ assertThat(readerProvider.listPartitions("year=2025/%")).containsExactly(oldPartition);
+ writerProvider.createPartitions(Collections.singletonList(newPartition.spec()));
+
+ assertThat(readerProvider.listPartitions("year=2025/%"))
+ .containsExactly(oldPartition, newPartition);
+ verify(catalog, times(2)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%");
+ }
+
+ @Test
+ void testCreatePartitionsFailureInvalidatesOtherProviderCache() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition oldPartition = partition("2025", "10");
+ Partition newPartition = partition("2025", "11");
+ List storedPartitions = new ArrayList<>();
+ storedPartitions.add(oldPartition);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%")))
+ .thenAnswer(ignored -> new PagedList<>(new ArrayList<>(storedPartitions), null));
+ List> created = Collections.singletonList(newPartition.spec());
+ RuntimeException responseLost = new RuntimeException("response lost");
+ doAnswer(
+ ignored -> {
+ storedPartitions.add(newPartition);
+ throw responseLost;
+ })
+ .when(catalog)
+ .createPartitions(IDENTIFIER, created);
+ FormatTableCatalogProvider readerProvider = provider(catalog);
+ FormatTableCatalogProvider writerProvider = provider(catalog);
+
+ assertThat(readerProvider.listPartitions("year=2025/%")).containsExactly(oldPartition);
+ assertThatThrownBy(() -> writerProvider.createPartitions(created)).isSameAs(responseLost);
+
+ assertThat(readerProvider.listPartitions("year=2025/%"))
+ .containsExactly(oldPartition, newPartition);
+ verify(catalog, times(2)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%");
+ }
+
+ @Test
+ void testAdvanceGenerationInvalidatesProvidersInSameProcess() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition oldPartition = partition("2025", "10");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%")))
+ .thenReturn(
+ new PagedList<>(Arrays.asList(oldPartition, partition("2025", "11")), null),
+ new PagedList<>(Collections.singletonList(oldPartition), null));
+ FormatTableCatalogProvider provider = provider(catalog);
+
+ assertThat(provider.listPartitions("year=2025/%")).hasSize(2);
+ // The catalog partition DDL path (e.g. RESTCatalog.dropPartitions) advances the
+ // generation so same-process scans read their own writes within the cache TTL.
+ FormatTableCatalogProvider.advanceGeneration(IDENTIFIER);
+
+ assertThat(provider.listPartitions("year=2025/%")).containsExactly(oldPartition);
+ verify(catalog, times(2)).listPartitionsPaged(IDENTIFIER, 1000, null, "year=2025/%");
+ }
+
+ @Test
+ void testLeadingPatternResidualFilterAndUnregisteredDirectory() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%")))
+ .thenReturn(
+ new PagedList<>(
+ Arrays.asList(partition("2025", "10"), partition("2025", "11")),
+ null));
+ TrackingLocalFileIO fileIO = new TrackingLocalFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path octoberFile = writeDataFile(fileIO, tablePath, "year=2025/month=10");
+ Path novemberFile = writeDataFile(fileIO, tablePath, "year=2025/month=11");
+ Path unregisteredFile = writeDataFile(fileIO, tablePath, "year=2025/month=12");
+ FormatTable table = createTable(fileIO, tablePath, provider(catalog), false);
+ PredicateBuilder builder = new PredicateBuilder(table.partitionType());
+ Predicate predicate =
+ PredicateBuilder.and(builder.equal(0, 2025), builder.greaterThan(1, 10));
+ PartitionPredicate filter =
+ PartitionPredicate.fromPredicate(table.partitionType(), predicate);
+
+ ManagedFormatTableScan scan = new ManagedFormatTableScan(table, filter, null);
+ List plannedFiles = plannedFiles(scan.plan().splits());
+
+ assertThat(scan.createPartitionNamePattern()).isEqualTo("year=2025/%");
+ assertThat(plannedFiles).containsExactly(novemberFile);
+ assertThat(plannedFiles).doesNotContain(octoberFile, unregisteredFile);
+ assertThat(fileIO.listedPaths).containsExactly(new Path(tablePath, "year=2025/month=11"));
+ }
+
+ @Test
+ void testListPartitionEntriesUsesCatalogVisibility() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition catalogPartition =
+ new Partition(partition("2025", "11").spec(), 11L, 22L, 3L, 44L, 5, false);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(catalogPartition), null));
+ TrackingLocalFileIO fileIO = new TrackingLocalFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ writeDataFile(fileIO, tablePath, "year=2025/month=11");
+ writeDataFile(fileIO, tablePath, "year=2025/month=12");
+ FormatTable table = createTable(fileIO, tablePath, provider(catalog), false);
+
+ List entries =
+ new ManagedFormatTableScan(table, null, null).listPartitionEntries();
+
+ assertThat(entries)
+ .extracting(
+ entry -> entry.partition().getInt(0), entry -> entry.partition().getInt(1))
+ .containsExactly(org.assertj.core.groups.Tuple.tuple(2025, 11));
+ assertThat(entries.get(0).recordCount()).isEqualTo(11L);
+ assertThat(entries.get(0).fileSizeInBytes()).isEqualTo(22L);
+ assertThat(entries.get(0).fileCount()).isEqualTo(3L);
+ assertThat(entries.get(0).lastFileCreationTime()).isEqualTo(44L);
+ assertThat(entries.get(0).totalBuckets()).isEqualTo(5);
+ assertThat(fileIO.listedPaths).isEmpty();
+ assertThat(fileIO.statusListedPaths).isEmpty();
+ }
+
+ @Test
+ void testUnderscoreInPartitionNameRemainsLiteralPrefix() {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTable table =
+ createStringPartitionTable(fileIO, tablePath, provider(mock(Catalog.class)));
+ Predicate predicate =
+ new PredicateBuilder(table.partitionType())
+ .equal(0, BinaryString.fromString("a_b"));
+ PartitionPredicate filter =
+ PartitionPredicate.fromPredicate(table.partitionType(), predicate);
+
+ ManagedFormatTableScan scan = new ManagedFormatTableScan(table, filter, null);
+
+ assertThat(scan.createPartitionNamePattern()).isEqualTo("year=a_b/%");
+ }
+
+ @Test
+ void testValueOnlyPartitionPath() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(
+ new PagedList<>(Collections.singletonList(partition("2025", "11")), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path dataFile = writeDataFile(fileIO, tablePath, "2025/11");
+ FormatTable table = createTable(fileIO, tablePath, provider(catalog), true);
+
+ List plannedFiles =
+ plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits());
+
+ assertThat(plannedFiles).containsExactly(dataFile);
+ }
+
+ @Test
+ void testDuplicateCatalogPartitionPlansSplitsOnce() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(
+ new PagedList<>(
+ Arrays.asList(partition("2025", "11"), partition("2025", "11")),
+ null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path dataFile = writeDataFile(fileIO, tablePath, "year=2025/month=11");
+ FormatTable table = createTable(fileIO, tablePath, provider(catalog), false);
+
+ List plannedFiles =
+ plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits());
+
+ assertThat(plannedFiles).containsExactly(dataFile);
+ }
+
+ @Test
+ void testWhitespacePartitionValueIsVisible() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition whitespacePartition = partition(" ", "11");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(whitespacePartition), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path dataFile =
+ writeDataFile(
+ fileIO,
+ tablePath,
+ PartitionPathUtils.generatePartitionPath(
+ new LinkedHashMap<>(whitespacePartition.spec())));
+ FormatTable table = createStringPartitionTable(fileIO, tablePath, provider(catalog));
+
+ List plannedFiles =
+ plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits());
+
+ assertThat(plannedFiles).containsExactly(dataFile);
+ }
+
+ @Test
+ void testEmptyPartitionValueReportsCorruptMetadata() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition emptyValuePartition = partition("2025", "");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(emptyValuePartition), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTable table = createStringPartitionTable(fileIO, tablePath, provider(catalog));
+
+ assertThatThrownBy(() -> new ManagedFormatTableScan(table, null, null).plan().splits())
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("corrupt partition metadata")
+ .hasMessageContaining(IDENTIFIER.getFullName());
+ }
+
+ @Test
+ void testCatalogFailureDoesNotFallBackToFileSystem() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ RuntimeException catalogFailure =
+ new RuntimeException("Catalog partition listing unavailable");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenThrow(catalogFailure);
+ TrackingLocalFileIO fileIO = new TrackingLocalFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ writeDataFile(fileIO, tablePath, "year=2025/month=11");
+ FormatTable table = createTable(fileIO, tablePath, provider(catalog), false);
+
+ assertThatThrownBy(() -> new ManagedFormatTableScan(table, null, null).plan().splits())
+ .isSameAs(catalogFailure);
+ assertThat(fileIO.listedPaths).isEmpty();
+ }
+
+ @Test
+ @DisplayName(
+ "treats a registered partition with a missing directory as an empty partition "
+ + "(requires real object-store validation)")
+ void testMissingManagedPartitionReadsAsEmpty() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(
+ new PagedList<>(Collections.singletonList(partition("2025", "11")), null));
+ Path tablePath = new Path(tempDir.toUri());
+ Path missingPath = new Path(tablePath, "year=2025/month=11");
+ LocalFileIO fileIO =
+ new LocalFileIO() {
+ @Override
+ public FileStatus[] listFiles(Path path, boolean recursive) throws IOException {
+ assertThat(path).isEqualTo(missingPath);
+ throw new FileNotFoundException(path.toString());
+ }
+ };
+ FormatTable table = createTable(fileIO, tablePath, provider(catalog), false);
+
+ // A registered partition whose directory is missing (e.g. ADD PARTITION before the first
+ // INSERT) must not fail the whole scan; it reads as an empty partition, matching Hive.
+ List plannedFiles =
+ plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits());
+ assertThat(plannedFiles).isEmpty();
+ }
+
+ @Test
+ void testTraversalPartitionValueReportsCorruptMetadata() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition traversalPartition = partition("2025", "..");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(traversalPartition), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTable table = createStringPartitionTable(fileIO, tablePath, provider(catalog), true);
+
+ assertThatThrownBy(() -> new ManagedFormatTableScan(table, null, null).plan().splits())
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("corrupt partition metadata")
+ .hasMessageContaining(IDENTIFIER.getFullName());
+ }
+
+ @Test
+ void testDotDotValueInKeyValueLayoutRemainsVisible() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition partition = partition("2025", "..");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(partition), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path dataFile = writeDataFile(fileIO, tablePath, "year=2025/month=..");
+ FormatTable table = createStringPartitionTable(fileIO, tablePath, provider(catalog));
+
+ List plannedFiles =
+ plannedFiles(new ManagedFormatTableScan(table, null, null).plan().splits());
+
+ assertThat(plannedFiles).containsExactly(dataFile);
+ }
+
+ private FormatTableCatalogProvider provider(Catalog catalog) {
+ return new FormatTableCatalogProvider(IDENTIFIER, () -> catalog);
+ }
+
+ private FormatTable createTable(
+ LocalFileIO fileIO,
+ Path tablePath,
+ FormatTableCatalogProvider provider,
+ boolean valueOnlyPath) {
+ RowType rowType =
+ RowType.builder()
+ .field("year", DataTypes.INT())
+ .field("month", DataTypes.INT())
+ .field("id", DataTypes.INT())
+ .build();
+ return FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(IDENTIFIER)
+ .rowType(rowType)
+ .partitionKeys(Arrays.asList("year", "month"))
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.toString(valueOnlyPath)))
+ .catalogProvider(provider)
+ .build();
+ }
+
+ private FormatTable createStringPartitionTable(
+ LocalFileIO fileIO, Path tablePath, FormatTableCatalogProvider provider) {
+ return createStringPartitionTable(fileIO, tablePath, provider, false);
+ }
+
+ private FormatTable createStringPartitionTable(
+ LocalFileIO fileIO,
+ Path tablePath,
+ FormatTableCatalogProvider provider,
+ boolean valueOnlyPath) {
+ RowType rowType =
+ RowType.builder()
+ .field("year", DataTypes.STRING())
+ .field("month", DataTypes.STRING())
+ .field("id", DataTypes.INT())
+ .build();
+ return FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(IDENTIFIER)
+ .rowType(rowType)
+ .partitionKeys(Arrays.asList("year", "month"))
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.toString(valueOnlyPath)))
+ .catalogProvider(provider)
+ .build();
+ }
+
+ private Path writeDataFile(LocalFileIO fileIO, Path tablePath, String partitionPath)
+ throws IOException {
+ Path file = new Path(new Path(tablePath, partitionPath), "data.csv");
+ fileIO.mkdirs(file.getParent());
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false)) {
+ out.write(1);
+ }
+ return file;
+ }
+
+ private static Partition partition(String year, String month) {
+ Map spec = new LinkedHashMap<>();
+ spec.put("year", year);
+ spec.put("month", month);
+ return new Partition(spec, 0, 0, 0, 0, -1, false);
+ }
+
+ private static List plannedFiles(List splits) {
+ return splits.stream()
+ .map(FormatDataSplit.class::cast)
+ .flatMap(split -> split.files().stream())
+ .map(FormatDataSplit.FileMeta::filePath)
+ .collect(Collectors.toList());
+ }
+
+ private static class TrackingLocalFileIO extends LocalFileIO {
+
+ private final List listedPaths = new ArrayList<>();
+ private final List statusListedPaths = new ArrayList<>();
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ statusListedPaths.add(path);
+ return super.listStatus(path);
+ }
+
+ @Override
+ public FileStatus[] listFiles(Path path, boolean recursive) throws IOException {
+ listedPaths.add(path);
+ return super.listFiles(path, recursive);
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
index a21b168f54ea..6b2952778003 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
@@ -19,17 +19,25 @@
package org.apache.paimon.utils;
import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Arrays;
+import java.util.LinkedHashMap;
import static org.apache.paimon.utils.PartitionPathUtils.mightMatch;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.entry;
-/** Tests for {@link PartitionPathUtils#mightMatch}. */
+/** Tests for {@link PartitionPathUtils}. */
class PartitionPathUtilsTest {
private final RowType partitionType =
@@ -39,6 +47,46 @@ class PartitionPathUtilsTest {
.build();
private final PredicateBuilder builder = new PredicateBuilder(partitionType);
+ @ParameterizedTest
+ @ValueSource(strings = {".", ".."})
+ void testValueOnlyDotSegmentIsRejected(String rawValue) {
+ LinkedHashMap partitionSpec = new LinkedHashMap<>();
+ partitionSpec.put("pt", rawValue);
+
+ assertThatThrownBy(() -> PartitionPathUtils.generatePartitionPathUtil(partitionSpec, true))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(rawValue);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {".", ".."})
+ void testKeyedDotValueKeepsCompatibleSafeLayout(String rawValue) {
+ LinkedHashMap partitionSpec = new LinkedHashMap<>();
+ partitionSpec.put("pt", rawValue);
+
+ String partitionPath = PartitionPathUtils.generatePartitionPathUtil(partitionSpec, false);
+ Path resolvedPath = new Path(new Path("file:///warehouse/table"), partitionPath);
+
+ assertThat(partitionPath).isEqualTo("pt=" + rawValue + Path.SEPARATOR);
+ assertThat(PartitionPathUtils.extractPartitionSpecFromPath(resolvedPath))
+ .containsExactly(entry("pt", rawValue));
+ }
+
+ @Test
+ void testTrailingPartitionExtractionStopsAtTableBoundary() {
+ Path partitionPath = new Path("file:///warehouse/env=prod/table/dt=20260718/hh=10");
+
+ assertThat(
+ PartitionPathUtils.extractPartitionSpecFromPath(
+ partitionPath, Arrays.asList("dt", "hh")))
+ .containsExactly(entry("dt", "20260718"), entry("hh", "10"));
+ assertThat(
+ PartitionPathUtils.extractPartitionSpecFromPath(
+ new Path("file:///warehouse/dt=parent/table/wrong=20260718/hh=10"),
+ Arrays.asList("dt", "hh")))
+ .isNull();
+ }
+
@Test
void testNullPredicate() {
assertThat(mightMatch(null, 0, 0, GenericRow.of(2024, 5))).isTrue();
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
index f248ee70ae2d..f69c9083f84a 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkCatalog.java
@@ -36,6 +36,8 @@
import org.apache.paimon.spark.catalog.SupportView;
import org.apache.paimon.spark.catalog.functions.FunctionIdentifierConverter;
import org.apache.paimon.spark.catalog.functions.PaimonFunctions;
+import org.apache.paimon.spark.format.CatalogFormatTablePartitionCatalog;
+import org.apache.paimon.spark.format.FormatTablePartitionCatalog;
import org.apache.paimon.spark.utils.CatalogUtils;
import org.apache.paimon.table.FormatTable;
import org.apache.paimon.table.iceberg.IcebergTable;
@@ -805,7 +807,12 @@ protected org.apache.spark.sql.connector.catalog.Table loadSparkTable(
copyWithSQLConf(
catalog.getTable(tblIdent), catalogName, tblIdent, extraOptions);
if (table instanceof FormatTable) {
- return toSparkFormatTable(ident, (FormatTable) table);
+ FormatTable formatTable = (FormatTable) table;
+ FormatTablePartitionCatalog partitionCatalog =
+ formatTable.catalogProvider() != null
+ ? new CatalogFormatTablePartitionCatalog(catalog, tblIdent)
+ : null;
+ return toSparkFormatTable(ident, formatTable, partitionCatalog);
} else if (table instanceof IcebergTable) {
return new SparkIcebergTable(table);
} else if (table instanceof LanceTable) {
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java
index 6a2203b9cbaa..90f2bef3ca33 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java
@@ -39,6 +39,7 @@
import org.apache.paimon.spark.procedure.ExpireSnapshotsProcedure;
import org.apache.paimon.spark.procedure.ExpireTagsProcedure;
import org.apache.paimon.spark.procedure.FastForwardProcedure;
+import org.apache.paimon.spark.procedure.ListFormatTablePartitionsProcedure;
import org.apache.paimon.spark.procedure.MarkPartitionDoneProcedure;
import org.apache.paimon.spark.procedure.MergeBranchProcedure;
import org.apache.paimon.spark.procedure.MigrateDatabaseProcedure;
@@ -58,6 +59,7 @@
import org.apache.paimon.spark.procedure.RollbackProcedure;
import org.apache.paimon.spark.procedure.RollbackToTimestampProcedure;
import org.apache.paimon.spark.procedure.RollbackToWatermarkProcedure;
+import org.apache.paimon.spark.procedure.SyncFormatTableMetadataProcedure;
import org.apache.paimon.spark.procedure.TriggerTagAutomaticCreationProcedure;
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap;
@@ -127,6 +129,10 @@ private static Map> initProcedureBuilders() {
"trigger_tag_automatic_creation", TriggerTagAutomaticCreationProcedure::builder);
procedureBuilders.put("rewrite_file_index", RewriteFileIndexProcedure::builder);
procedureBuilders.put("copy", CopyFilesProcedure::builder);
+ procedureBuilders.put(
+ "list_format_table_partitions", ListFormatTablePartitionsProcedure::builder);
+ procedureBuilders.put(
+ "sync_format_table_metadata", SyncFormatTableMetadataProcedure::builder);
return procedureBuilders.build();
}
}
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java
index 3893e670685b..1493108729e1 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/catalog/FormatTableCatalog.java
@@ -24,6 +24,7 @@
import org.apache.paimon.options.Options;
import org.apache.paimon.spark.SparkSource;
import org.apache.paimon.spark.SparkTypeUtils;
+import org.apache.paimon.spark.format.FormatTablePartitionCatalog;
import org.apache.paimon.spark.format.PaimonFormatTable;
import org.apache.paimon.table.FormatTable;
import org.apache.paimon.utils.TypeUtils;
@@ -60,10 +61,17 @@ default boolean isFormatTable(@Nullable String provide) {
}
default Table toSparkFormatTable(Identifier ident, FormatTable formatTable) {
+ return toSparkFormatTable(ident, formatTable, null);
+ }
+
+ default Table toSparkFormatTable(
+ Identifier ident,
+ FormatTable formatTable,
+ @Nullable FormatTablePartitionCatalog partitionCatalog) {
Map optionsMap = formatTable.options();
CoreOptions coreOptions = new CoreOptions(optionsMap);
if (coreOptions.formatTableImplementationIsPaimon()) {
- return new PaimonFormatTable(formatTable);
+ return new PaimonFormatTable(formatTable, partitionCatalog);
} else {
SparkSession spark = PaimonSparkSession$.MODULE$.active();
List pathList = new ArrayList<>();
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BaseProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BaseProcedure.java
index c61f0c1fa5e9..34b8bd8ff2ad 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BaseProcedure.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BaseProcedure.java
@@ -20,6 +20,9 @@
import org.apache.paimon.spark.SparkTable;
import org.apache.paimon.spark.SparkUtils;
+import org.apache.paimon.spark.format.PaimonFormatTable;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.PartitionPathUtils;
import org.apache.paimon.utils.Preconditions;
import org.apache.spark.sql.PaimonSparkSession$;
@@ -34,6 +37,7 @@
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation;
import org.apache.spark.sql.paimon.shims.SparkShimLoader;
+import java.util.Map;
import java.util.function.Function;
import scala.Option;
@@ -104,6 +108,38 @@ protected SparkTable loadSparkTable(Identifier ident) {
}
}
+ protected PaimonFormatTable loadFormatTable(Identifier ident) {
+ try {
+ Table table = tableCatalog.loadTable(ident);
+ Preconditions.checkArgument(
+ table instanceof PaimonFormatTable, "%s is not a Paimon Format Table", ident);
+ return (PaimonFormatTable) table;
+ } catch (NoSuchTableException e) {
+ throw new RuntimeException("Couldn't load Format Table " + ident, e);
+ }
+ }
+
+ /** Validate that the table is a managed Format Table and return its partition gateway. */
+ static org.apache.paimon.spark.format.FormatTablePartitionCatalog managedPartitionCatalog(
+ PaimonFormatTable sparkTable) {
+ String name = sparkTable.table().fullName();
+ Preconditions.checkArgument(
+ new org.apache.paimon.CoreOptions(sparkTable.table().options())
+ .partitionedTableInMetastore(),
+ "%s is not a managed Format Table",
+ name);
+ org.apache.paimon.spark.format.FormatTablePartitionCatalog partitionCatalog =
+ sparkTable.partitionCatalog();
+ Preconditions.checkNotNull(
+ partitionCatalog, "Managed Format Table %s has no partition catalog gateway", name);
+ return partitionCatalog;
+ }
+
+ protected static String partitionPath(Map spec, RowType partitionType) {
+ String path = PartitionPathUtils.generatePartitionPath(spec, partitionType, false);
+ return path.endsWith("/") ? path.substring(0, path.length() - 1) : path;
+ }
+
protected DataSourceV2Relation createRelation(Identifier ident) {
return DataSourceV2Relation.create(
loadSparkTable(ident), Option.apply(tableCatalog), Option.apply(ident));
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListFormatTablePartitionsProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListFormatTablePartitionsProcedure.java
new file mode 100644
index 000000000000..8e4b76e0dcfa
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListFormatTablePartitionsProcedure.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.procedure;
+
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.serializer.InternalRowSerializer;
+import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.spark.format.FormatTablePartitionCatalog;
+import org.apache.paimon.spark.format.FormatTablePartitionPage;
+import org.apache.paimon.spark.format.PaimonFormatTable;
+import org.apache.paimon.spark.utils.SparkProcedureUtils;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.InternalRowPartitionComputer;
+import org.apache.paimon.utils.Preconditions;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.connector.catalog.Identifier;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.unsafe.types.UTF8String;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+
+import static org.apache.spark.sql.types.DataTypes.IntegerType;
+import static org.apache.spark.sql.types.DataTypes.StringType;
+
+/** Lists managed format-table partitions with bounded pagination. */
+public class ListFormatTablePartitionsProcedure extends BaseProcedure {
+
+ private static final int DEFAULT_LIMIT = 1000;
+ private static final int MAX_LIMIT = 10000;
+ private static final ProcedureParameter[] PARAMETERS =
+ new ProcedureParameter[] {
+ ProcedureParameter.required("table", StringType),
+ ProcedureParameter.optional("where", StringType),
+ ProcedureParameter.optional("limit", IntegerType),
+ ProcedureParameter.optional("page_token", StringType)
+ };
+
+ private static final StructType OUTPUT_TYPE =
+ new StructType(
+ new StructField[] {
+ new StructField("partition", StringType, false, Metadata.empty()),
+ new StructField("next_page_token", StringType, true, Metadata.empty())
+ });
+
+ protected ListFormatTablePartitionsProcedure(TableCatalog tableCatalog) {
+ super(tableCatalog);
+ }
+
+ @Override
+ public ProcedureParameter[] parameters() {
+ return PARAMETERS;
+ }
+
+ @Override
+ public StructType outputType() {
+ return OUTPUT_TYPE;
+ }
+
+ @Override
+ public InternalRow[] call(InternalRow args) {
+ Identifier ident = toIdentifier(args.getString(0), PARAMETERS[0].name());
+ String where = args.isNullAt(1) ? null : args.getString(1);
+ int limit = args.isNullAt(2) ? DEFAULT_LIMIT : args.getInt(2);
+ String pageToken = args.isNullAt(3) ? null : args.getString(3);
+ Preconditions.checkArgument(
+ limit > 0 && limit <= MAX_LIMIT,
+ "limit must be between 1 and %s, but was %s",
+ MAX_LIMIT,
+ limit);
+
+ PaimonFormatTable sparkTable = loadFormatTable(ident);
+ FormatTable formatTable = sparkTable.table();
+ FormatTablePartitionCatalog partitionCatalog = managedPartitionCatalog(sparkTable);
+
+ RowType partitionType = formatTable.partitionType();
+ PartitionPredicate partitionPredicate =
+ SparkProcedureUtils.convertToPartitionPredicate(
+ where, partitionType, spark(), createRelation(ident, sparkTable));
+ Predicate> predicate = null;
+ if (partitionPredicate != null) {
+ InternalRowSerializer serializer = new InternalRowSerializer(partitionType);
+ predicate =
+ spec -> {
+ GenericRow row =
+ InternalRowPartitionComputer.convertSpecToInternalRow(
+ spec, partitionType, formatTable.defaultPartName());
+ BinaryRow binaryRow = serializer.toBinaryRow(row);
+ return partitionPredicate.test(binaryRow);
+ };
+ }
+
+ FormatTablePartitionPage page =
+ listPage(partitionCatalog, Collections.emptyMap(), pageToken, limit, predicate);
+ UTF8String nextToken =
+ page.nextPageToken() == null ? null : UTF8String.fromString(page.nextPageToken());
+ return page.partitions().stream()
+ .map(
+ spec ->
+ newInternalRow(
+ UTF8String.fromString(partitionPath(spec, partitionType)),
+ nextToken))
+ .toArray(InternalRow[]::new);
+ }
+
+ public static ProcedureBuilder builder() {
+ return new BaseProcedure.Builder() {
+ @Override
+ protected ListFormatTablePartitionsProcedure doBuild() {
+ return new ListFormatTablePartitionsProcedure(tableCatalog());
+ }
+ };
+ }
+
+ @Override
+ public String description() {
+ return "ListFormatTablePartitionsProcedure";
+ }
+
+ /**
+ * Fetch one raw catalog page of {@code pageSize} partitions and, when a filter is present,
+ * return only the matching rows. Filtered pages may therefore be sparse (fewer than {@code
+ * pageSize} rows); the catalog's own next-page token is passed through verbatim. Raw pages
+ * whose rows are all filtered out are skipped within the same call, so an empty result always
+ * means the listing is exhausted.
+ */
+ static FormatTablePartitionPage listPage(
+ FormatTablePartitionCatalog catalog,
+ Map prefix,
+ String pageToken,
+ int pageSize,
+ Predicate> predicate) {
+ String token = normalizePageToken(pageToken);
+ while (true) {
+ FormatTablePartitionPage page = catalog.listPartitions(prefix, token, pageSize);
+ List> partitions = page.partitions();
+ // An overshooting page would silently violate the caller's limit and duplicate rows
+ // on the next page, so it is the one gateway contract worth asserting here.
+ Preconditions.checkState(
+ partitions.size() <= pageSize,
+ "Format table partition catalog returned %s partitions for a page of %s.",
+ partitions.size(),
+ pageSize);
+ String nextToken = normalizePageToken(page.nextPageToken());
+ if (predicate != null) {
+ List> filtered = new ArrayList<>();
+ for (Map spec : partitions) {
+ if (predicate.test(spec)) {
+ filtered.add(spec);
+ }
+ }
+ // The next-page token travels on result rows, so a page whose rows are all
+ // filtered out would lose the cursor. Skip ahead until at least one row matches
+ // or the listing is exhausted; an empty result therefore always means "done".
+ if (filtered.isEmpty() && nextToken != null) {
+ token = nextToken;
+ continue;
+ }
+ partitions = filtered;
+ }
+ return new FormatTablePartitionPage(partitions, nextToken);
+ }
+ }
+
+ private static String normalizePageToken(String pageToken) {
+ return pageToken == null || pageToken.isEmpty() ? null : pageToken;
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/SyncFormatTableMetadataProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/SyncFormatTableMetadataProcedure.java
new file mode 100644
index 000000000000..56bb498406eb
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/SyncFormatTableMetadataProcedure.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.procedure;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.spark.format.FormatTablePartitionCatalog;
+import org.apache.paimon.spark.format.FormatTablePartitionPage;
+import org.apache.paimon.spark.format.PaimonFormatTable;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Pair;
+import org.apache.paimon.utils.PartitionPathUtils;
+import org.apache.paimon.utils.Preconditions;
+
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.connector.catalog.Identifier;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
+import org.apache.spark.unsafe.types.UTF8String;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.spark.sql.types.DataTypes.BooleanType;
+import static org.apache.spark.sql.types.DataTypes.StringType;
+
+/** Reconciles filesystem partitions with managed format-table catalog metadata. */
+public class SyncFormatTableMetadataProcedure extends BaseProcedure {
+
+ private static final int CATALOG_PAGE_SIZE = 1000;
+
+ private static final ProcedureParameter[] PARAMETERS =
+ new ProcedureParameter[] {
+ ProcedureParameter.required("table", StringType),
+ ProcedureParameter.optional("mode", StringType),
+ ProcedureParameter.optional("dry_run", BooleanType)
+ };
+
+ private static final StructType OUTPUT_TYPE =
+ new StructType(
+ new StructField[] {
+ new StructField("partition", StringType, false, Metadata.empty()),
+ new StructField("action", StringType, false, Metadata.empty()),
+ new StructField("status", StringType, false, Metadata.empty())
+ });
+
+ protected SyncFormatTableMetadataProcedure(TableCatalog tableCatalog) {
+ super(tableCatalog);
+ }
+
+ @Override
+ public ProcedureParameter[] parameters() {
+ return PARAMETERS;
+ }
+
+ @Override
+ public StructType outputType() {
+ return OUTPUT_TYPE;
+ }
+
+ @Override
+ public InternalRow[] call(InternalRow args) {
+ Identifier ident = toIdentifier(args.getString(0), PARAMETERS[0].name());
+ String mode = args.isNullAt(1) ? "ADD" : args.getString(1);
+ boolean dryRun = args.isNullAt(2) || args.getBoolean(2);
+ validateMode(mode);
+
+ PaimonFormatTable sparkTable = loadFormatTable(ident);
+ FormatTable formatTable = sparkTable.table();
+ FormatTablePartitionCatalog partitionCatalog = managedPartitionCatalog(sparkTable);
+
+ List, String>> actions;
+ try {
+ actions =
+ syncPartitions(
+ partitionCatalog,
+ listFilesystemPartitionSpecs(formatTable),
+ formatTable.partitionKeys(),
+ dryRun,
+ mode);
+ } finally {
+ // A failed batch response cannot tell us whether the service committed, so the
+ // Spark-side caches are refreshed after every non-preview attempt.
+ if (!dryRun) {
+ refreshSparkCache(ident, sparkTable);
+ }
+ }
+
+ RowType partitionType = formatTable.partitionType();
+ return actions.stream()
+ .map(
+ entry ->
+ newInternalRow(
+ UTF8String.fromString(
+ partitionPath(entry.getKey(), partitionType)),
+ UTF8String.fromString(entry.getValue()),
+ UTF8String.fromString(
+ dryRun
+ ? "DRY_RUN"
+ : ("ADD".equals(entry.getValue())
+ ? "REGISTERED"
+ : "UNREGISTERED"))))
+ .toArray(InternalRow[]::new);
+ }
+
+ static void validateMode(String mode) {
+ Preconditions.checkArgument(
+ "ADD".equals(mode.toUpperCase(Locale.ROOT))
+ || "DROP".equals(mode.toUpperCase(Locale.ROOT))
+ || "SYNC".equals(mode.toUpperCase(Locale.ROOT)),
+ "Unsupported mode '%s'; supported modes are ADD, DROP and SYNC",
+ mode);
+ }
+
+ public static ProcedureBuilder builder() {
+ return new BaseProcedure.Builder() {
+ @Override
+ protected SyncFormatTableMetadataProcedure doBuild() {
+ return new SyncFormatTableMetadataProcedure(tableCatalog());
+ }
+ };
+ }
+
+ /**
+ * Shared engine for {@code MSCK REPAIR TABLE [{ADD|DROP|SYNC} PARTITIONS]} on a managed Format
+ * Table. MSCK applies immediately (never a dry run); the flag pair follows Spark's {@code
+ * RepairTable} plan (plain MSCK means ADD). Returns the number of applied actions.
+ */
+ public static int repairFormatTablePartitions(
+ PaimonFormatTable sparkTable, boolean addPartitions, boolean dropPartitions) {
+ Preconditions.checkArgument(
+ addPartitions || dropPartitions,
+ "MSCK REPAIR TABLE must enable ADD and/or DROP partitions");
+ FormatTable formatTable = sparkTable.table();
+ FormatTablePartitionCatalog partitionCatalog = managedPartitionCatalog(sparkTable);
+ String mode = addPartitions && dropPartitions ? "SYNC" : (addPartitions ? "ADD" : "DROP");
+ return syncPartitions(
+ partitionCatalog,
+ listFilesystemPartitionSpecs(formatTable),
+ formatTable.partitionKeys(),
+ false,
+ mode)
+ .size();
+ }
+
+ private static List> listFilesystemPartitionSpecs(FormatTable formatTable) {
+ // Discover partitions from the raw directory names rather than through the table scan:
+ // the scan casts each value to its column type and back (e.g. month=01 -> 1), producing
+ // specs that can no longer round-trip to the real directory. The write path registers the
+ // raw directory value, so MSCK/sync must diff against the same raw values to avoid
+ // spuriously adding/dropping partition metadata.
+ boolean onlyValueInPath =
+ new CoreOptions(formatTable.options()).formatTablePartitionOnlyValueInPath();
+ List, Path>> found =
+ PartitionPathUtils.searchPartSpecAndPaths(
+ formatTable.fileIO(),
+ new Path(formatTable.location()),
+ formatTable.partitionKeys().size(),
+ formatTable.partitionKeys(),
+ onlyValueInPath);
+ List> specs = new ArrayList<>(found.size());
+ for (Pair, Path> pair : found) {
+ PartitionPathUtils.validatePartitionSpecForPath(pair.getKey(), onlyValueInPath);
+ specs.add(pair.getKey());
+ }
+ return specs;
+ }
+
+ /**
+ * Diff the filesystem partition set against the catalog registration set and, unless dryRun,
+ * apply the requested mode. ADD registers "directory exists but unregistered"; DROP is
+ * metadata-only cleanup of "registered but directory missing" (data is never touched); SYNC is
+ * both. Scan-completeness guard: the filesystem listing that feeds {@code filesystemPartitions}
+ * ({@link PartitionPathUtils#searchPartSpecAndPaths}) fails on any mid-scan LIST error instead
+ * of returning a truncated set, so a DROP diff can only be produced from a complete listing and
+ * a transient failure never deregisters partitions that still exist.
+ */
+ static List, String>> syncPartitions(
+ FormatTablePartitionCatalog catalog,
+ List> filesystemPartitions,
+ List partitionKeys,
+ boolean dryRun,
+ String mode) {
+ String normalizedMode = mode.toUpperCase(Locale.ROOT);
+ Set> catalogPartitions = new HashSet<>();
+ String token = null;
+ do {
+ FormatTablePartitionPage page = catalog.listPartitions(null, token, CATALOG_PAGE_SIZE);
+ if (page.partitions() != null) {
+ catalogPartitions.addAll(page.partitions());
+ }
+ String nextToken = page.nextPageToken();
+ if (nextToken != null && nextToken.isEmpty()) {
+ // A null or empty next-page token is terminal, matching PaimonPartitionManagement.
+ nextToken = null;
+ }
+ token = nextToken;
+ } while (token != null);
+
+ Set> filesystemSet = new HashSet<>(filesystemPartitions);
+
+ List> addDiff = new ArrayList<>();
+ if (!"DROP".equals(normalizedMode)) {
+ for (Map partition : filesystemPartitions) {
+ if (!catalogPartitions.contains(partition)) {
+ addDiff.add(partition);
+ }
+ }
+ sortByCanonicalPath(addDiff, partitionKeys);
+ }
+ List> dropDiff = new ArrayList<>();
+ if (!"ADD".equals(normalizedMode)) {
+ for (Map partition : catalogPartitions) {
+ if (!filesystemSet.contains(partition)) {
+ dropDiff.add(partition);
+ }
+ }
+ sortByCanonicalPath(dropDiff, partitionKeys);
+ }
+
+ if (!dryRun) {
+ if (!addDiff.isEmpty()) {
+ applyInBatches(addDiff, batch -> catalog.createPartitions(batch, true));
+ }
+ if (!dropDiff.isEmpty()) {
+ applyInBatches(dropDiff, catalog::dropPartitions);
+ }
+ }
+ List, String>> actions = new ArrayList<>();
+ addDiff.forEach(spec -> actions.add(new java.util.AbstractMap.SimpleEntry<>(spec, "ADD")));
+ dropDiff.forEach(
+ spec -> actions.add(new java.util.AbstractMap.SimpleEntry<>(spec, "DROP")));
+ return actions;
+ }
+
+ /**
+ * Apply a repair diff in bounded batches instead of a single unbounded catalog mutation. A
+ * first repair of a pre-existing table can discover far more partitions than any regular write,
+ * and a single request carrying all of them monopolizes a server worker and one metadata
+ * transaction. Registration is an idempotent upsert and unregistration ignores missing
+ * partitions, so a mid-way failure leaves a state a rerun converges from.
+ */
+ private static void applyInBatches(
+ List> diff,
+ java.util.function.Consumer>> operation) {
+ for (int start = 0; start < diff.size(); start += CATALOG_PAGE_SIZE) {
+ operation.accept(diff.subList(start, Math.min(start + CATALOG_PAGE_SIZE, diff.size())));
+ }
+ }
+
+ /** Sort partitions by their canonical path for stable output and deterministic batches. */
+ private static void sortByCanonicalPath(
+ List> partitions, List partitionKeys) {
+ partitions.sort(Comparator.comparing(partition -> canonicalPath(partition, partitionKeys)));
+ }
+
+ private static String canonicalPath(Map partition, List partitionKeys) {
+ LinkedHashMap orderedSpec = new LinkedHashMap<>();
+ for (String key : partitionKeys) {
+ if (partition.containsKey(key)) {
+ orderedSpec.put(key, partition.get(key));
+ }
+ }
+ return PartitionPathUtils.generatePartitionPath(orderedSpec);
+ }
+
+ @Override
+ public String description() {
+ return "SyncFormatTableMetadataProcedure";
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
index cd5955b0ff39..c25cf20063be 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
@@ -19,11 +19,13 @@
package org.apache.paimon.spark
import org.apache.paimon.CoreOptions
+import org.apache.paimon.fs.Path
import org.apache.paimon.partition.PartitionStatistics
-import org.apache.paimon.table.{FileStoreTable, Table}
+import org.apache.paimon.spark.format.FormatTablePartitionCatalog
+import org.apache.paimon.table.{FileStoreTable, FormatTable, Table}
import org.apache.paimon.table.source.ScanMode
import org.apache.paimon.types.RowType
-import org.apache.paimon.utils.{InternalRowPartitionComputer, TypeUtils}
+import org.apache.paimon.utils.{InternalRowPartitionComputer, PartitionPathUtils, TypeUtils}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
@@ -35,58 +37,43 @@ import org.apache.spark.sql.types.StructType
import java.util.{Map => JMap, Objects}
import scala.collection.JavaConverters._
+import scala.collection.mutable.{ArrayBuffer, HashSet}
trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with Logging {
val table: Table
+ def partitionCatalog: FormatTablePartitionCatalog = null
+
lazy val partitionRowType: RowType = TypeUtils.project(table.rowType, table.partitionKeys)
override lazy val partitionSchema: StructType = SparkTypeUtils.fromPaimonRowType(partitionRowType)
private def toPaimonPartitions(rows: Array[InternalRow]): Array[java.util.Map[String, String]] = {
- table match {
- case fileStoreTable: FileStoreTable =>
- val partitionKeys = table.partitionKeys().asScala.toSeq
- val partitionDefaultName = fileStoreTable.coreOptions().partitionDefaultName()
- val legacyPartitionName = CoreOptions.fromMap(table.options()).legacyPartitionName
-
- rows.map {
- r =>
- val partitionFieldCount = r.numFields
- require(
- partitionFieldCount <= partitionKeys.length,
- s"Partition values length $partitionFieldCount exceeds partition keys " +
- s"${partitionKeys.mkString("[", ", ", "]")}."
- )
- val partitionNames = partitionKeys.take(partitionFieldCount)
- val currentPartitionRowType =
- if (partitionFieldCount == partitionRowType.getFieldCount) {
- partitionRowType
- } else {
- TypeUtils.project(table.rowType, partitionNames.asJava)
- }
- val currentPartitionSchema =
- if (partitionFieldCount == partitionSchema.length) {
- partitionSchema
- } else {
- SparkTypeUtils.fromPaimonRowType(currentPartitionRowType)
- }
- val rowConverter = CatalystTypeConverters.createToScalaConverter(
- CharVarcharUtils.replaceCharVarcharWithString(currentPartitionSchema))
- val rowDataPartitionComputer = new InternalRowPartitionComputer(
- partitionDefaultName,
- currentPartitionRowType,
- partitionNames.toArray,
- legacyPartitionName
- )
-
- rowDataPartitionComputer.generatePartValues(
- new SparkRow(currentPartitionRowType, rowConverter(r).asInstanceOf[Row]))
- }
- case _ =>
- throw new UnsupportedOperationException("Only FileStoreTable supports partitions.")
- }
+ val partitionKeys = table.partitionKeys().asScala.toSeq
+
+ rows.map(r => toPaimonPartition(r, partitionKeys.take(r.numFields)))
+ }
+
+ private def toPaimonPartition(
+ row: InternalRow,
+ partitionNames: Seq[String]): java.util.Map[String, String] = {
+ val coreOptions = CoreOptions.fromMap(table.options())
+ val partitionDefaultName = coreOptions.partitionDefaultName()
+ val legacyPartitionName = coreOptions.legacyPartitionName
+ val currentPartitionRowType = TypeUtils.project(table.rowType, partitionNames.asJava)
+ val currentPartitionSchema = SparkTypeUtils.fromPaimonRowType(currentPartitionRowType)
+ val rowConverter = CatalystTypeConverters.createToScalaConverter(
+ CharVarcharUtils.replaceCharVarcharWithString(currentPartitionSchema))
+ val rowDataPartitionComputer = new InternalRowPartitionComputer(
+ partitionDefaultName,
+ currentPartitionRowType,
+ partitionNames.toArray,
+ legacyPartitionName
+ )
+
+ rowDataPartitionComputer.generatePartValues(
+ new SparkRow(currentPartitionRowType, rowConverter(row).asInstanceOf[Row]))
}
override def dropPartitions(rows: Array[InternalRow]): Boolean = {
@@ -116,6 +103,142 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L
}
}
+ /**
+ * Resolves, with a single catalog list-by-names lookup, which of the given complete partition
+ * specs are registered for a managed Format Table. The result is aligned with the input arrays.
+ */
+ private[spark] def formatTablePartitionsRegistered(
+ partitionNames: Array[Array[String]],
+ rows: Array[InternalRow]): Array[Boolean] = {
+ if (rows.isEmpty) {
+ return Array.empty
+ }
+
+ table match {
+ case formatTable: FormatTable
+ if CoreOptions.fromMap(formatTable.options()).partitionedTableInMetastore() =>
+ val requested =
+ rows.zip(partitionNames).map { case (row, names) => toPaimonPartition(row, names.toSeq) }
+ val registered = requirePartitionCatalog().listPartitionsByNames(requested.toSeq.asJava)
+ val registeredSpecs = registered.asScala.map(_.asScala.toMap).toSet
+ requested.map(spec => registeredSpecs.contains(spec.asScala.toMap))
+ case _ =>
+ throw new UnsupportedOperationException(
+ "Only managed Format Table supports partition registration lookup.")
+ }
+ }
+
+ /**
+ * Drops the given partitions: complete specs are unregistered and their directories deleted
+ * as-is, partial specs are expanded to the registered leaf partitions they cover. Callers are
+ * responsible for resolving which complete specs are actually registered first (see
+ * [[formatTablePartitionsRegistered]]), so unregistered data directories are never deleted.
+ */
+ private[spark] def dropFormatTablePartitions(
+ partitionNames: Array[Array[String]],
+ rows: Array[InternalRow]): Boolean = {
+ table match {
+ case formatTable: FormatTable
+ if CoreOptions.fromMap(formatTable.options()).partitionedTableInMetastore() =>
+ val partitionKeyCount = formatTable.partitionKeys().size()
+ val requested =
+ rows.zip(partitionNames).map { case (row, names) => toPaimonPartition(row, names.toSeq) }
+ val partitions = ArrayBuffer.empty[java.util.Map[String, String]]
+ val seenPartitions = HashSet.empty[Map[String, String]]
+
+ def addPartition(partition: java.util.Map[String, String]): Unit = {
+ if (seenPartitions.add(partition.asScala.toMap)) {
+ partitions += partition
+ }
+ }
+
+ // Preserve exact requests as-is and let discovery add only missing complete leaves.
+ requested.filter(_.size() == partitionKeyCount).foreach(addPartition)
+ val partialSpecs = requested.filter(_.size() < partitionKeyCount).toSeq.distinct
+ if (partialSpecs.nonEmpty) {
+ def matchesRequestedPartial(partition: java.util.Map[String, String]): Boolean = {
+ partialSpecs.exists(_.asScala.forall {
+ case (key, value) => Objects.equals(value, partition.get(key))
+ })
+ }
+
+ // One unfiltered paged traversal resolves every partial spec; the requested constraints
+ // are enforced client-side.
+ val catalog = requirePartitionCatalog()
+ var pageToken: String = null
+ do {
+ val page = catalog.listPartitions(
+ new java.util.LinkedHashMap[String, String](),
+ pageToken,
+ PaimonPartitionManagement.ManagedPartitionDropPageSize)
+ page.partitions.asScala.foreach {
+ partition =>
+ val validated = validateManagedDropPartition(formatTable, partition)
+ if (matchesRequestedPartial(validated)) {
+ addPartition(validated)
+ }
+ }
+ pageToken = Option(page.nextPageToken).filter(_.nonEmpty).orNull
+ } while (pageToken != null)
+ }
+ dropManagedFormatTablePartitions(formatTable, partitions.toSeq)
+ case _ =>
+ throw new UnsupportedOperationException(
+ "Only managed Format Table supports named partition drop.")
+ }
+ }
+
+ private def dropManagedFormatTablePartitions(
+ formatTable: FormatTable,
+ partitions: Seq[java.util.Map[String, String]]): Boolean = {
+ // Unregister first so new queries stop seeing the partition, then delete the data directory
+ // with the table FileIO (client-side; the server never deletes data). A deletion failure leaves
+ // the possibly incomplete directory invisible; it must not be registered again automatically.
+ if (partitions.isEmpty) {
+ return true
+ }
+
+ val onlyValueInPath =
+ CoreOptions.fromMap(formatTable.options()).formatTablePartitionOnlyValueInPath()
+ // Resolve (and path-safety validate) every partition directory before any mutation, so a
+ // traversal attempt ('.'/'..') fails the whole DROP before unregistering anything.
+ val partitionPaths = partitions.map {
+ spec =>
+ resolvePartitionPathWithinTable(
+ formatTable,
+ orderedSpec(formatTable, spec),
+ onlyValueInPath)
+ }
+ logInfo("Try to drop managed format table partitions: " + partitions.mkString(","))
+ requirePartitionCatalog().dropPartitions(partitions.asJava)
+ val fileIO = formatTable.fileIO()
+ partitionPaths.foreach {
+ partitionPath =>
+ val deleted = fileIO.delete(partitionPath, true)
+ if (!deleted && fileIO.exists(partitionPath)) {
+ throw new java.io.IOException(
+ s"FileIO reported that partition directory $partitionPath was not deleted.")
+ }
+ }
+ true
+ }
+
+ private def validateManagedDropPartition(
+ formatTable: FormatTable,
+ partition: java.util.Map[String, String]): java.util.Map[String, String] = {
+ val partitionKeys = formatTable.partitionKeys().asScala
+ if (partitionKeys.exists(key => partition.get(key) == null)) {
+ throw new IllegalStateException(
+ s"Catalog must return a complete partition spec with keys " +
+ s"${partitionKeys.mkString("[", ", ", "]")} for managed Format Table " +
+ s"${formatTable.fullName()}, but returned $partition.")
+ }
+
+ val ordered = new java.util.LinkedHashMap[String, String]()
+ partitionKeys.foreach(key => ordered.put(key, partition.get(key)))
+ ordered
+ }
+
override def truncatePartitions(idents: Array[InternalRow]): Boolean = {
val partitions = toPaimonPartitions(idents).toSeq.asJava
val commit = table.newBatchWriteBuilder().newCommit()
@@ -216,4 +339,81 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L
throw new UnsupportedOperationException("Only FileStoreTable supports create partitions.")
}
}
+
+ private[spark] def createFormatTablePartitions(
+ rows: Array[InternalRow],
+ maps: Array[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ if (maps.exists(_.keySet().asScala.exists(_.equalsIgnoreCase("location")))) {
+ throw new UnsupportedOperationException(
+ s"ADD PARTITION with LOCATION is not supported for Format Table ${table.fullName()}.")
+ }
+ val formatTable = table.asInstanceOf[FormatTable]
+ val onlyValueInPath =
+ CoreOptions.fromMap(formatTable.options()).formatTablePartitionOnlyValueInPath()
+ val specs = toPaimonPartitions(rows).toSeq
+ // Resolve (and path-safety validate) every directory before mutating anything.
+ val partitionPaths = specs.map {
+ spec =>
+ resolvePartitionPathWithinTable(
+ formatTable,
+ orderedSpec(formatTable, spec),
+ onlyValueInPath)
+ }
+ requirePartitionCatalog().createPartitions(specs.asJava, ignoreIfExists)
+ // Create the partition directories client-side (symmetric with DROP deleting them), so an
+ // added partition exists on the filesystem and a subsequent scan returns an empty partition
+ // rather than depending on lazy directory creation, matching Hive ADD PARTITION semantics.
+ val fileIO = formatTable.fileIO()
+ partitionPaths.foreach(partitionPath => fileIO.mkdirs(partitionPath))
+ }
+
+ private def orderedSpec(
+ formatTable: FormatTable,
+ spec: java.util.Map[String, String]): java.util.LinkedHashMap[String, String] = {
+ val ordered = new java.util.LinkedHashMap[String, String]()
+ formatTable.partitionKeys().asScala.foreach {
+ key => if (spec.containsKey(key)) ordered.put(key, spec.get(key))
+ }
+ ordered
+ }
+
+ /**
+ * Build the partition directory for a spec and verify it stays strictly under the table location.
+ * Value-only path components are validated (including rejecting '.'/'..'), and the normalized
+ * path is checked against the table location so neither DROP (recursive delete) nor ADD (mkdirs)
+ * can escape the table directory via crafted or corrupt partition values.
+ */
+ private def resolvePartitionPathWithinTable(
+ formatTable: FormatTable,
+ orderedSpec: java.util.LinkedHashMap[String, String],
+ onlyValueInPath: Boolean): Path = {
+ PartitionPathUtils.validatePartitionSpecForPath(orderedSpec, onlyValueInPath)
+ val tablePath = new Path(formatTable.location())
+ val partitionPath = new Path(
+ tablePath,
+ PartitionPathUtils.generatePartitionPathUtil(orderedSpec, onlyValueInPath)
+ )
+ val normalizedTable = tablePath.toUri.normalize().getPath
+ val tablePrefix = if (normalizedTable.endsWith("/")) normalizedTable else normalizedTable + "/"
+ val normalizedPartition = partitionPath.toUri.normalize().getPath
+ if (!normalizedPartition.startsWith(tablePrefix)) {
+ throw new IllegalArgumentException(
+ s"Resolved partition path $partitionPath escapes the table location $tablePath for " +
+ s"partition spec $orderedSpec of Format Table ${formatTable.fullName()}.")
+ }
+ partitionPath
+ }
+
+ private def requirePartitionCatalog(): FormatTablePartitionCatalog = {
+ if (partitionCatalog == null) {
+ throw new UnsupportedOperationException(
+ s"Partition catalog is not configured for managed Format Table ${table.fullName()}.")
+ }
+ partitionCatalog
+ }
+}
+
+private object PaimonPartitionManagement {
+ val ManagedPartitionDropPageSize: Int = 1000
}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExec.scala
new file mode 100644
index 000000000000..26ce39238896
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExec.scala
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.commands
+
+import org.apache.paimon.CoreOptions
+import org.apache.paimon.spark.{SparkRow, SparkTypeUtils}
+import org.apache.paimon.spark.format.PaimonFormatTable
+import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec
+import org.apache.paimon.utils.{InternalRowPartitionComputer, TypeUtils}
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
+import org.apache.spark.sql.catalyst.analysis.ResolvedPartitionSpec
+import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils.escapePathName
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
+import org.apache.spark.unsafe.types.UTF8String
+
+import java.util.{Collections, Map => JMap}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+/** Physical command for bounded managed Format Table partition listing. */
+case class PaimonShowFormatTablePartitionsExec(
+ output: Seq[Attribute],
+ table: PaimonFormatTable,
+ partitionSpec: Option[ResolvedPartitionSpec],
+ maxResults: Int)
+ extends PaimonLeafV2CommandExec {
+
+ override protected def run(): Seq[InternalRow] = {
+ if (table.partitionCatalog == null) {
+ throw new UnsupportedOperationException(
+ s"Partition catalog is not configured for managed Format Table ${table.name()}.")
+ }
+
+ val prefix = toCatalogPrefix
+ val partitions = ArrayBuffer.empty[JMap[String, String]]
+ var pageToken: String = null
+ do {
+ val page =
+ table.partitionCatalog.listPartitions(prefix, pageToken, maxResults + 1 - partitions.size)
+ if (page.partitions != null) {
+ partitions ++= page.partitions.asScala
+ }
+ // A null or empty next-page token is terminal, matching PaimonPartitionManagement.
+ pageToken = Option(page.nextPageToken).filter(_.nonEmpty).orNull
+ } while (pageToken != null && partitions.size < maxResults + 1)
+
+ if (partitions.size > maxResults) {
+ throw new IllegalStateException(
+ s"SHOW PARTITIONS for managed Format Table ${table.name()} exceeded the configured " +
+ s"result limit $maxResults. Use CALL sys.list_format_table_partitions for paginated " +
+ "partition listing.")
+ }
+
+ val defaultPartitionName = table.table.defaultPartName()
+ partitions
+ .map(partition => formatPartition(partition, defaultPartitionName))
+ .sorted
+ .map(value => InternalRow(UTF8String.fromString(value)))
+ .toSeq
+ }
+
+ private def toCatalogPrefix: JMap[String, String] = {
+ partitionSpec match {
+ case None => Collections.emptyMap[String, String]()
+ case Some(spec) =>
+ val partitionNames = spec.names
+ val rowType = TypeUtils.project(table.table.rowType(), partitionNames.asJava)
+ val sparkSchema = SparkTypeUtils.fromPaimonRowType(rowType)
+ val rowConverter = CatalystTypeConverters.createToScalaConverter(
+ CharVarcharUtils.replaceCharVarcharWithString(sparkSchema))
+ val coreOptions = CoreOptions.fromMap(table.table.options())
+ val partitionComputer = new InternalRowPartitionComputer(
+ coreOptions.partitionDefaultName(),
+ rowType,
+ partitionNames.toArray,
+ coreOptions.legacyPartitionName)
+ partitionComputer.generatePartValues(
+ new SparkRow(rowType, rowConverter(spec.ident).asInstanceOf[Row]))
+ }
+ }
+
+ private def formatPartition(
+ partition: JMap[String, String],
+ defaultPartitionName: String): String = {
+ table.table
+ .partitionKeys()
+ .asScala
+ .map {
+ name =>
+ val catalogValue = partition.get(name)
+ val displayValue = if (defaultPartitionName == catalogValue) "null" else catalogValue
+ escapePathName(name) + "=" + escapePathName(displayValue)
+ }
+ .mkString("/")
+ }
+}
+
+object PaimonShowFormatTablePartitionsExec {
+
+ val DEFAULT_MAX_RESULTS: Int = 10000
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala
new file mode 100644
index 000000000000..2e7208efbd87
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.execution
+
+import org.apache.paimon.CoreOptions
+import org.apache.paimon.spark.format.PaimonFormatTable
+import org.apache.paimon.spark.procedure.SyncFormatTableMetadataProcedure
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec}
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec
+
+import java.util.{Map => JMap}
+
+import scala.collection.JavaConverters._
+
+object PaimonFormatTablePartitionDdlExec {
+
+ def isManaged(table: PaimonFormatTable): Boolean =
+ CoreOptions.fromMap(table.table.options()).partitionedTableInMetastore()
+
+ private[execution] def unsupportedUnmanaged(
+ operation: String,
+ table: PaimonFormatTable): UnsupportedOperationException =
+ new UnsupportedOperationException(
+ s"$operation PARTITION is not supported for unmanaged Format Table ${table.name()}.")
+}
+
+/**
+ * Adds managed Format Table partitions without Spark's client-side existence precheck. The whole
+ * batch and IF NOT EXISTS flag are forwarded to the catalog so it can apply them atomically.
+ */
+case class PaimonAddFormatTablePartitionsExec(
+ table: PaimonFormatTable,
+ partSpecs: Seq[ResolvedPartitionSpec],
+ ignoreIfExists: Boolean,
+ refreshCache: () => Unit)
+ extends LeafV2CommandExec {
+
+ override protected def run(): Seq[InternalRow] = {
+ if (!PaimonFormatTablePartitionDdlExec.isManaged(table)) {
+ throw PaimonFormatTablePartitionDdlExec.unsupportedUnmanaged("ADD", table)
+ }
+
+ if (partSpecs.nonEmpty) {
+ val properties: Array[JMap[String, String]] =
+ partSpecs.map(spec => spec.location.map("location" -> _).toMap.asJava).toArray
+ try {
+ table.createFormatTablePartitions(
+ partSpecs.map(_.ident).toArray,
+ properties,
+ ignoreIfExists)
+ } finally {
+ refreshCache()
+ }
+ }
+ Seq.empty
+ }
+
+ override def output: Seq[Attribute] = Seq.empty
+}
+
+/**
+ * Physical command providing explicit DROP semantics for managed Format Tables (unmanaged Format
+ * Tables always fail with a clear error). Complete specs are resolved against the catalog
+ * registration first: missing specs fail with [[NoSuchPartitionsException]] unless IF EXISTS was
+ * given, and only registered specs are unregistered and have their directories deleted, so data
+ * that is merely awaiting registration (e.g. by MSCK REPAIR TABLE) is never removed. Partial specs
+ * resolve to the registered leaf partitions they cover.
+ */
+case class PaimonDropFormatTablePartitionsExec(
+ table: PaimonFormatTable,
+ partSpecs: Seq[ResolvedPartitionSpec],
+ ifExists: Boolean,
+ purge: Boolean,
+ refreshCache: () => Unit)
+ extends LeafV2CommandExec {
+
+ override protected def run(): Seq[InternalRow] = {
+ if (!PaimonFormatTablePartitionDdlExec.isManaged(table)) {
+ throw PaimonFormatTablePartitionDdlExec.unsupportedUnmanaged("DROP", table)
+ }
+ if (purge) {
+ // Match Spark's v2 default for purgePartitions. Managed Format Table DROP PARTITION
+ // already removes the partition data, so PURGE adds nothing.
+ throw new UnsupportedOperationException(
+ s"DROP PARTITION ... PURGE is not supported for managed Format Table ${table.name()}. " +
+ "DROP PARTITION already removes the partition data.")
+ }
+ if (partSpecs.isEmpty) {
+ return Seq.empty
+ }
+
+ val partitionKeyCount = table.table.partitionKeys().size()
+ val (completeSpecs, partialSpecs) =
+ partSpecs.partition(_.ident.numFields == partitionKeyCount)
+ val registration = table
+ .formatTablePartitionsRegistered(
+ completeSpecs.map(_.names.toArray).toArray,
+ completeSpecs.map(_.ident).toArray)
+ .toSeq
+ val missingSpecs = completeSpecs.zip(registration).collect { case (spec, false) => spec }
+ if (missingSpecs.nonEmpty && !ifExists) {
+ throw new NoSuchPartitionsException(
+ table.name(),
+ missingSpecs.map(_.ident),
+ table.partitionSchema)
+ }
+
+ val specsToDrop =
+ completeSpecs.zip(registration).collect { case (spec, true) => spec } ++ partialSpecs
+ if (specsToDrop.nonEmpty) {
+ // Managed semantics (PaimonPartitionManagement#dropFormatTablePartitions): resolve partial
+ // specs, unregister the exact catalog partitions, then delete their directories with the
+ // table FileIO client-side. A directory-deletion failure stays unregistered so partially
+ // deleted data is not exposed again.
+ try {
+ table.dropFormatTablePartitions(
+ specsToDrop.map(_.names.toArray).toArray,
+ specsToDrop.map(_.ident).toArray)
+ } finally {
+ refreshCache()
+ }
+ }
+ Seq.empty
+ }
+
+ override def output: Seq[Attribute] = Seq.empty
+}
+
+/**
+ * MSCK REPAIR TABLE for managed Format Tables, reusing the sync-procedure engine. Spark rejects
+ * RepairTable for v2 tables in DataSourceV2Strategy, so PaimonStrategy intercepts first; plain MSCK
+ * means ADD, and MSCK always applies (never a dry run). Unmanaged tables are not intercepted and
+ * keep Spark's own v2 rejection.
+ */
+case class PaimonRepairFormatTablePartitionsExec(
+ table: PaimonFormatTable,
+ addPartitions: Boolean,
+ dropPartitions: Boolean,
+ refreshCache: () => Unit)
+ extends LeafV2CommandExec {
+
+ override protected def run(): Seq[InternalRow] = {
+ try {
+ SyncFormatTableMetadataProcedure
+ .repairFormatTablePartitions(table, addPartitions, dropPartitions)
+ } finally {
+ refreshCache()
+ }
+ Seq.empty
+ }
+
+ override def output: Seq[Attribute] = Seq.empty
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala
index 74c0e6c47dbd..82d26c89b501 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala
@@ -28,7 +28,9 @@ import org.apache.paimon.spark.{PaimonRecordReaderIterator, SparkCatalog, SparkG
import org.apache.paimon.spark.catalog.{SparkBaseCatalog, SupportView}
import org.apache.paimon.spark.catalyst.analysis.ResolvedPaimonView
import org.apache.paimon.spark.catalyst.plans.logical.{CopyIntoLocationCommand, CopyIntoLocationSource, CopyIntoTableCommand, CreateOrReplaceTagCommand, CreatePaimonView, DeleteTagCommand, DropPaimonView, LateralVectorSearch, PaimonCallCommand, PaimonDropPartitions, PaimonTableValuedFunctions, RenameTagCommand, ResolvedIdentifier, ShowPaimonViews, ShowTagsCommand, TruncatePaimonTableWithFilter}
+import org.apache.paimon.spark.commands.PaimonShowFormatTablePartitionsExec
import org.apache.paimon.spark.data.SparkInternalRow
+import org.apache.paimon.spark.format.PaimonFormatTable
import org.apache.paimon.spark.read.VectorSearchResultUtils
import org.apache.paimon.spark.schema.PaimonMetadataColumn
import org.apache.paimon.table.{InnerTable, SpecialFields, Table}
@@ -40,9 +42,9 @@ import org.apache.spark.TaskContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedTable}
+import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedPartitionSpec, ResolvedTable}
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, GenericInternalRow, JoinedRow, PredicateHelper, UnsafeProjection}
-import org.apache.spark.sql.catalyst.plans.logical.{CreateTableAsSelect, DescribeRelation, LogicalPlan, ReplaceTable, ReplaceTableAsSelect, ShowCreateTable}
+import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, CreateTableAsSelect, DescribeRelation, DropPartitions, LogicalPlan, RepairTable, ReplaceTable, ReplaceTableAsSelect, ShowCreateTable, ShowPartitions}
import org.apache.spark.sql.catalyst.util.ArrayData
import org.apache.spark.sql.connector.catalog.{Identifier, PaimonLookupCatalog, TableCatalog}
import org.apache.spark.sql.execution.{PaimonDescribeTableExec, SparkPlan, SparkStrategy}
@@ -161,6 +163,58 @@ case class PaimonStrategy(spark: SparkSession)
case _ => Nil
}
+ case AddPartitions(r @ ResolvedTable(_, _, table: PaimonFormatTable, _), parts, ifNotExists) =>
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ parts.asResolvedPartitionSpecs,
+ ifNotExists,
+ recacheTable(r)) :: Nil
+
+ // Spark's DataSourceV2Strategy rejects RepairTable for every v2 table; managed Format Tables
+ // support it through the sync engine, so intercept here (extension strategies run first).
+ // Unmanaged tables fall through and keep the upstream rejection.
+ case RepairTable(
+ r @ ResolvedTable(_, _, table: PaimonFormatTable, _),
+ enableAddPartitions,
+ enableDropPartitions) if PaimonFormatTablePartitionDdlExec.isManaged(table) =>
+ PaimonRepairFormatTablePartitionsExec(
+ table,
+ enableAddPartitions,
+ enableDropPartitions,
+ recacheTable(r)) :: Nil
+
+ case ShowPartitions(ResolvedTable(_, _, table: PaimonFormatTable, _), partitionSpec, output)
+ if PaimonFormatTablePartitionDdlExec.isManaged(table) =>
+ PaimonShowFormatTablePartitionsExec(
+ output,
+ table,
+ partitionSpec.map(_.asInstanceOf[ResolvedPartitionSpec]),
+ PaimonShowFormatTablePartitionsExec.DEFAULT_MAX_RESULTS) :: Nil
+
+ case DropPartitions(
+ r @ ResolvedTable(_, _, table: PaimonFormatTable, _),
+ parts,
+ ifExists,
+ purge) =>
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ parts.asResolvedPartitionSpecs,
+ ifExists,
+ purge,
+ recacheTable(r)) :: Nil
+
+ case PaimonDropPartitions(
+ r @ ResolvedTable(_, _, table: PaimonFormatTable, _),
+ parts,
+ ifExists,
+ purge) =>
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ parts.asResolvedPartitionSpecs,
+ ifExists,
+ purge,
+ recacheTable(r)) :: Nil
+
case PaimonDropPartitions(
r @ ResolvedTable(_, _, table: SparkTable, _),
parts,
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalog.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalog.scala
new file mode 100644
index 000000000000..dffac642c0e7
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalog.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format
+
+import org.apache.paimon.catalog.{Catalog, Identifier}
+import org.apache.paimon.partition.Partition
+
+import java.util.{ArrayList, List => JList, Map => JMap}
+
+import scala.collection.JavaConverters._
+
+/** Catalog-backed partition gateway for a single Format Table. */
+case class CatalogFormatTablePartitionCatalog(catalog: Catalog, identifier: Identifier)
+ extends FormatTablePartitionCatalog {
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ // The flag reaches the server unchanged: the managed ADD command skips Spark's client-side
+ // existence precheck, so strict-ADD atomicity (reject the whole batch when any partition
+ // exists) is enforced by the catalog service.
+ catalog.createPartitions(identifier, partitions, ignoreIfExists)
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ // Catalog contract: dropPartitions ignores non-existent partitions and, for managed format
+ // tables, unregisters metadata only. Data deletion is performed by the caller afterwards.
+ // Unregister in bounded batches: a partial-spec DROP can expand to more leaf partitions than
+ // a catalog service accepts in one request (e.g. dropping a year of a minute-partitioned
+ // table). Dropping ignores missing partitions, so a mid-way failure leaves a state a retry
+ // converges from.
+ var start = 0
+ while (start < partitions.size()) {
+ val end =
+ math.min(start + CatalogFormatTablePartitionCatalog.MUTATION_BATCH_SIZE, partitions.size())
+ catalog.dropPartitions(identifier, partitions.subList(start, end))
+ start = end
+ }
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = {
+ toSpecs(catalog.listPartitionsByNames(identifier, partitions))
+ }
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ val page = catalog.listPartitionsPaged(identifier, Int.box(pageSize), pageToken, null)
+ val partitions = Option(page.getElements)
+ .map(_.asScala)
+ .getOrElse(Seq.empty)
+ .filter(partition => matchesPrefix(partition, prefix))
+ .asJava
+ FormatTablePartitionPage(toSpecs(partitions), page.getNextPageToken)
+ }
+
+ private def matchesPrefix(partition: Partition, prefix: JMap[String, String]): Boolean = {
+ prefix == null || prefix.asScala.forall {
+ case (key, value) => value == partition.spec().get(key)
+ }
+ }
+
+ private def toSpecs(partitions: JList[Partition]): JList[JMap[String, String]] = {
+ val result = new ArrayList[JMap[String, String]](partitions.size())
+ partitions.asScala.foreach(partition => result.add(partition.spec()))
+ result
+ }
+}
+
+object CatalogFormatTablePartitionCatalog {
+
+ /** Mirror of the catalog page size; keeps a single mutation request bounded. */
+ val MUTATION_BATCH_SIZE = 1000
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWriteBase.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWriteBase.scala
index 853c7f41467b..bec591bbd7d1 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWriteBase.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTableBatchWriteBase.scala
@@ -49,6 +49,8 @@ abstract class FormatTableBatchWriteBase(
extends Logging
with Serializable {
+ @volatile protected var commitStarted: Boolean = false
+
protected val batchWriteBuilder: BatchWriteBuilder = {
val builder = table.newBatchWriteBuilder()
// todo: add test for static overwrite the whole table
@@ -65,6 +67,7 @@ abstract class FormatTableBatchWriteBase(
}
protected def commitMessages(messages: Array[WriterCommitMessage]): Unit = {
+ commitStarted = true
logInfo(s"Committing to FormatTable ${table.name()}")
val batchTableCommit = batchWriteBuilder.newCommit()
val commitMessages = WriteTaskResult.merge(messages).asJava
@@ -80,6 +83,12 @@ abstract class FormatTableBatchWriteBase(
}
protected def abortMessages(messages: Array[WriterCommitMessage]): Unit = {
+ if (commitStarted) {
+ logWarning(
+ s"Skip abort cleanup for FormatTable ${table.name()} because commit has already started")
+ return
+ }
+
logInfo(s"Aborting write to FormatTable ${table.name()}")
val batchTableCommit = batchWriteBuilder.newCommit()
val commitMessages = WriteTaskResult.merge(messages).asJava
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTablePartitionCatalog.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTablePartitionCatalog.scala
new file mode 100644
index 000000000000..79074555939c
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/FormatTablePartitionCatalog.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format
+
+import java.util.{List => JList, Map => JMap}
+
+/** Spark-side gateway to catalog-managed Format Table partition metadata. */
+trait FormatTablePartitionCatalog {
+
+ def createPartitions(partitions: JList[JMap[String, String]], ignoreIfExists: Boolean): Unit
+
+ /**
+ * Unregister partitions from the catalog (metadata only, ignores missing partitions). Data
+ * deletion is the caller's responsibility.
+ */
+ def dropPartitions(partitions: JList[JMap[String, String]]): Unit
+
+ def listPartitionsByNames(partitions: JList[JMap[String, String]]): JList[JMap[String, String]]
+
+ def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage
+}
+
+case class FormatTablePartitionPage(partitions: JList[JMap[String, String]], nextPageToken: String)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala
index dc7effd39f72..42482c5366d9 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala
@@ -41,6 +41,16 @@ case class PaimonFormatTable(table: FormatTable)
with SupportsRead
with SupportsWrite {
+ // Driver-only gateway owned and injected by SparkCatalog. It is intentionally transient.
+ @transient private var driverPartitionCatalog: FormatTablePartitionCatalog = null
+
+ def this(table: FormatTable, partitionCatalog: FormatTablePartitionCatalog) = {
+ this(table)
+ driverPartitionCatalog = partitionCatalog
+ }
+
+ override def partitionCatalog: FormatTablePartitionCatalog = driverPartitionCatalog
+
override def capabilities(): util.Set[TableCapability] = {
util.EnumSet.of(BATCH_READ, BATCH_WRITE, OVERWRITE_DYNAMIC, OVERWRITE_BY_FILTER)
}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
index 37521ceda653..31065e53244c 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
@@ -19,10 +19,10 @@
package org.apache.paimon.spark.util
import org.apache.paimon.CoreOptions
-import org.apache.paimon.catalog.Identifier
+import org.apache.paimon.catalog.{CatalogUtils, Identifier}
import org.apache.paimon.options.ConfigOption
import org.apache.paimon.spark.{SparkCatalogOptions, SparkConnectorOptions}
-import org.apache.paimon.table.Table
+import org.apache.paimon.table.{FormatTable, Table}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.SparkSession
@@ -186,10 +186,44 @@ object OptionUtils extends SQLConfHelper with Logging {
if (mergedOptions.isEmpty) {
table
} else {
+ normalizeManagedFormatTableDynamicOptions(table, mergedOptions)
table.copy(mergedOptions).asInstanceOf[T]
}
}
+ /**
+ * Whether a format table is catalog-managed is a persisted property and cannot be flipped by a
+ * dynamic option. A dynamic {@code metastore.partitioned-table} (typically a session-global
+ * {@code spark.paimon.*} config that used to be a harmless no-op) is dropped so the persisted
+ * value always wins, warning only when it actually disagrees, instead of failing every format
+ * table load in that session.
+ */
+ private def normalizeManagedFormatTableDynamicOptions(
+ table: Table,
+ dynamicOptions: JMap[String, String]): Unit = {
+ table match {
+ case formatTable: FormatTable =>
+ val managedKey = CoreOptions.METASTORE_PARTITIONED_TABLE.key()
+ val persistedManaged = new CoreOptions(formatTable.options()).partitionedTableInMetastore()
+ if (dynamicOptions.containsKey(managedKey)) {
+ val dynamicManaged = new CoreOptions(dynamicOptions).partitionedTableInMetastore()
+ if (dynamicManaged != persistedManaged) {
+ logWarning(s"Ignoring dynamic option '$managedKey=$dynamicManaged' for format table " +
+ s"${formatTable.fullName()}: whether a format table is catalog-managed is fixed " +
+ s"at creation time (persisted value: $persistedManaged). Use ALTER TABLE to change it.")
+ }
+ dynamicOptions.remove(managedKey)
+ }
+
+ if (persistedManaged) {
+ val effectiveOptions = new JHashMap[String, String](formatTable.options())
+ effectiveOptions.putAll(dynamicOptions)
+ CatalogUtils.validateManagedFormatTableOptions(effectiveOptions)
+ }
+ case _ =>
+ }
+ }
+
private def getMergedOptions(
catalogName: String = null,
ident: Identifier = null,
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala
index 5227d1f0f604..23ec0631c4dc 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala
@@ -18,14 +18,18 @@
package org.apache.spark.sql.catalyst.parser.extensions
-import org.apache.paimon.spark.catalog.SupportView
-import org.apache.paimon.spark.catalyst.plans.logical.{PaimonDropPartitions, ResolvedIdentifier}
+import org.apache.paimon.spark.SparkTable
+import org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions
+import org.apache.paimon.spark.execution.PaimonFormatTablePartitionDdlExec
+import org.apache.paimon.spark.format.PaimonFormatTable
import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier
+import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, UnresolvedTable}
import org.apache.spark.sql.catalyst.plans.logical.{DropPartitions, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog}
+import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog, TableCatalog}
+
+import scala.util.Try
case class RewriteSparkDDLCommands(spark: SparkSession)
extends Rule[LogicalPlan]
@@ -37,8 +41,41 @@ case class RewriteSparkDDLCommands(spark: SparkSession)
// A new member was added to CreatePaimonView since spark4.0,
// unapply pattern matching is not used here to ensure compatibility across multiple spark versions.
- case DropPartitions(UnresolvedPaimonRelation(aliasedTable), parts, ifExists, purge) =>
+ //
+ // Both Paimon data tables and catalog-managed Format Tables route DROP PARTITION through
+ // PaimonDropPartitions (managed Format Tables resolve partial specs through the partition
+ // gateway, so they must bypass Spark's strict partition-spec resolution too). A single
+ // extractor loads the table once and classifies it, avoiding a second catalog.loadTable per
+ // analyzer iteration. Unmanaged Format Tables are not matched and keep Spark's own resolution
+ // plus the explicit unsupported error at execution.
+ case DropPartitions(UnresolvedPaimonDropTarget(aliasedTable), parts, ifExists, purge) =>
PaimonDropPartitions(aliasedTable, parts, ifExists, purge)
+ }
+
+ private object UnresolvedPaimonDropTarget {
+ def unapply(plan: LogicalPlan): Option[LogicalPlan] = {
+ EliminateSubqueryAliases(plan) match {
+ case UnresolvedTable(multipartIdentifier, _, _)
+ if isPaimonDropTarget(multipartIdentifier) =>
+ Some(plan)
+ case _ => None
+ }
+ }
+ }
+ private def isPaimonDropTarget(multipartIdentifier: Seq[String]): Boolean = {
+ multipartIdentifier match {
+ case CatalogAndIdentifier(catalog: TableCatalog, ident) =>
+ Try(catalog.loadTable(ident))
+ .map {
+ case _: SparkTable => true
+ case formatTable: PaimonFormatTable =>
+ PaimonFormatTablePartitionDdlExec.isManaged(formatTable) &&
+ formatTable.partitionCatalog != null
+ case _ => false
+ }
+ .getOrElse(false)
+ case _ => false
+ }
}
}
diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/FormatTableMetadataProcedureTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/FormatTableMetadataProcedureTest.java
new file mode 100644
index 000000000000..beba3da65cf0
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/procedure/FormatTableMetadataProcedureTest.java
@@ -0,0 +1,1155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.procedure;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.FileStatus;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.spark.SparkProcedures;
+import org.apache.paimon.spark.format.FormatTablePartitionCatalog;
+import org.apache.paimon.spark.format.FormatTablePartitionPage;
+import org.apache.paimon.spark.format.PaimonFormatTable;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
+import org.apache.spark.sql.connector.catalog.TableCatalog;
+import org.apache.spark.unsafe.types.UTF8String;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.lang.reflect.Proxy;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for managed format-table metadata procedures. */
+class FormatTableMetadataProcedureTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ void procedureImplementationsAreAvailable() {
+ assertThatCode(
+ () ->
+ Class.forName(
+ "org.apache.paimon.spark.procedure.ListFormatTablePartitionsProcedure"))
+ .doesNotThrowAnyException();
+ assertThatCode(
+ () ->
+ Class.forName(
+ "org.apache.paimon.spark.procedure.SyncFormatTableMetadataProcedure"))
+ .doesNotThrowAnyException();
+ }
+
+ @Test
+ void implementationsExposeProcedureBuilders() throws Exception {
+ assertThat(Procedure.class).isAssignableFrom(ListFormatTablePartitionsProcedure.class);
+ assertThat(Procedure.class).isAssignableFrom(SyncFormatTableMetadataProcedure.class);
+ assertThat(
+ ListFormatTablePartitionsProcedure.class
+ .getDeclaredMethod("builder")
+ .getReturnType())
+ .isEqualTo(ProcedureBuilder.class);
+ assertThat(SparkProcedures.names())
+ .contains("list_format_table_partitions", "sync_format_table_metadata");
+ assertThat(
+ SyncFormatTableMetadataProcedure.class
+ .getDeclaredMethod("builder")
+ .getReturnType())
+ .isEqualTo(ProcedureBuilder.class);
+ }
+
+ @Test
+ void listProcedureDeclaresPagedPredicateContract() {
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("list-format-table-partitions-procedure-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ Procedure procedure =
+ ListFormatTablePartitionsProcedure.builder().withTableCatalog(null).build();
+
+ assertThat(
+ Arrays.stream(procedure.parameters())
+ .map(ProcedureParameter::name)
+ .collect(Collectors.toList()))
+ .containsExactly("table", "where", "limit", "page_token");
+ assertThat(
+ Arrays.stream(procedure.parameters())
+ .map(ProcedureParameter::required)
+ .collect(Collectors.toList()))
+ .containsExactly(true, false, false, false);
+ assertThat(procedure.outputType().fieldNames())
+ .containsExactly("partition", "next_page_token");
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void syncProcedureDeclaresSafeRepairContract() {
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("sync-format-table-metadata-procedure-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ Procedure procedure =
+ SyncFormatTableMetadataProcedure.builder().withTableCatalog(null).build();
+
+ assertThat(
+ Arrays.stream(procedure.parameters())
+ .map(ProcedureParameter::name)
+ .collect(Collectors.toList()))
+ .containsExactly("table", "mode", "dry_run");
+ assertThat(
+ Arrays.stream(procedure.parameters())
+ .map(ProcedureParameter::required)
+ .collect(Collectors.toList()))
+ .containsExactly(true, false, false);
+ assertThat(procedure.outputType().fieldNames())
+ .containsExactly("partition", "action", "status");
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void syncProcedureCallDefaultsToDryRunAndApplyCreatesOneBatch() throws Exception {
+ java.nio.file.Path partitionDirectory =
+ Files.createDirectories(tempDir.resolve("dt=20260701"));
+ Files.write(
+ partitionDirectory.resolve("data.csv"),
+ Collections.singletonList("1"),
+ StandardCharsets.UTF_8);
+
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("sync-format-table-metadata-call-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ RecordingCatalog partitionCatalog = new RecordingCatalog();
+ partitionCatalog.addPage(
+ null, new FormatTablePartitionPage(Collections.emptyList(), null));
+ PaimonFormatTable sparkTable =
+ new PaimonFormatTable(
+ formatTable(tempDir.toUri().toString()), partitionCatalog);
+ AtomicInteger refreshCount = new AtomicInteger();
+ Procedure procedure =
+ new SyncFormatTableMetadataProcedure(tableCatalog(sparkTable)) {
+ @Override
+ protected void refreshSparkCache(
+ org.apache.spark.sql.connector.catalog.Identifier ident,
+ org.apache.spark.sql.connector.catalog.Table table) {
+ refreshCount.incrementAndGet();
+ }
+ };
+
+ InternalRow[] dryRunRows =
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {UTF8String.fromString("db.t"), null, null}));
+
+ assertThat(dryRunRows).hasSize(1);
+ assertThat(dryRunRows[0].getUTF8String(0).toString()).isEqualTo("dt=20260701");
+ assertThat(dryRunRows[0].getUTF8String(1).toString()).isEqualTo("ADD");
+ assertThat(dryRunRows[0].getUTF8String(2).toString()).isEqualTo("DRY_RUN");
+ assertThat(partitionCatalog.createdPartitions).isEmpty();
+ assertThat(refreshCount).hasValue(0);
+
+ InternalRow[] applyRows =
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {UTF8String.fromString("db.t"), null, false}));
+
+ assertThat(applyRows).hasSize(1);
+ assertThat(applyRows[0].getUTF8String(2).toString()).isEqualTo("REGISTERED");
+ assertThat(partitionCatalog.createdPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260701")));
+ assertThat(partitionCatalog.createIgnoreFlags).containsExactly(true);
+ assertThat(refreshCount).hasValue(1);
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void syncProcedureAppliesLargeDiffInBoundedBatches() throws Exception {
+ int partitionCount = 1001;
+ for (int index = 0; index < partitionCount; index++) {
+ Files.createDirectories(tempDir.resolve(String.format("dt=%04d", index)));
+ }
+
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("sync-format-table-metadata-batch-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ RecordingCatalog partitionCatalog = new RecordingCatalog();
+ partitionCatalog.addPage(
+ null, new FormatTablePartitionPage(Collections.emptyList(), null));
+ PaimonFormatTable sparkTable =
+ new PaimonFormatTable(
+ formatTable(tempDir.toUri().toString()), partitionCatalog);
+ Procedure procedure =
+ new SyncFormatTableMetadataProcedure(tableCatalog(sparkTable)) {
+ @Override
+ protected void refreshSparkCache(
+ org.apache.spark.sql.connector.catalog.Identifier ident,
+ org.apache.spark.sql.connector.catalog.Table table) {}
+ };
+
+ InternalRow[] applyRows =
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {UTF8String.fromString("db.t"), null, false}));
+
+ assertThat(applyRows).hasSize(partitionCount);
+ assertThat(partitionCatalog.createdPartitions).hasSize(2);
+ assertThat(partitionCatalog.createdPartitions.get(0)).hasSize(1000);
+ assertThat(partitionCatalog.createdPartitions.get(1)).hasSize(1);
+ assertThat(partitionCatalog.createIgnoreFlags).containsExactly(true, true);
+ assertThat(partitionCatalog.createdPartitions.stream().mapToInt(List::size).sum())
+ .isEqualTo(partitionCount);
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void listProcedurePropagatesPartitionCatalogFailure() {
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("list-format-table-partitions-error-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ SecurityException failure = new SecurityException("partition access denied");
+ RecordingCatalog partitionCatalog =
+ new RecordingCatalog() {
+ @Override
+ public FormatTablePartitionPage listPartitions(
+ Map prefix, String pageToken, int pageSize) {
+ throw failure;
+ }
+ };
+ Procedure procedure =
+ ListFormatTablePartitionsProcedure.builder()
+ .withTableCatalog(
+ tableCatalog(
+ new PaimonFormatTable(formatTable(), partitionCatalog)))
+ .build();
+
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"), null, 1, null
+ })))
+ .isSameAs(failure);
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void listProcedureRejectsOutOfRangeLimit() {
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("list-format-table-partitions-limit-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ RecordingCatalog partitionCatalog = new RecordingCatalog();
+ Procedure procedure =
+ ListFormatTablePartitionsProcedure.builder()
+ .withTableCatalog(
+ tableCatalog(
+ new PaimonFormatTable(formatTable(), partitionCatalog)))
+ .build();
+
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"), null, 0, null
+ })))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("limit must be between 1 and 10000");
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"),
+ null,
+ 10001,
+ null
+ })))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("limit must be between 1 and 10000");
+ assertThat(partitionCatalog.requestedTokens).isEmpty();
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void listProcedureAppliesSqlPartitionPredicatePerCatalogPage() {
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("list-format-table-partitions-call-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ RecordingCatalog partitionCatalog = new RecordingCatalog();
+ partitionCatalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")), "p2"));
+ partitionCatalog.addPage(
+ "p2",
+ new FormatTablePartitionPage(
+ Arrays.asList(spec("dt", "20260703"), spec("dt", "20260704")), null));
+ PaimonFormatTable sparkTable = new PaimonFormatTable(formatTable(), partitionCatalog);
+ TableCatalog tableCatalog = tableCatalog(sparkTable);
+ Procedure procedure =
+ ListFormatTablePartitionsProcedure.builder()
+ .withTableCatalog(tableCatalog)
+ .build();
+
+ // A filtered page is the matching subset of one raw catalog page: it may be sparse
+ // and carries the catalog's own next-page token verbatim.
+ InternalRow[] rows =
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"),
+ UTF8String.fromString("dt >= '20260702'"),
+ 2,
+ null
+ }));
+
+ assertThat(rows).hasSize(1);
+ assertThat(rows[0].getUTF8String(0).toString()).isEqualTo("dt=20260702");
+ assertThat(rows[0].getUTF8String(1).toString()).isEqualTo("p2");
+ assertThat(partitionCatalog.requestedTokens).containsExactly((String) null);
+
+ InternalRow[] nextRows =
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"),
+ UTF8String.fromString("dt >= '20260702'"),
+ 2,
+ UTF8String.fromString("p2")
+ }));
+
+ assertThat(nextRows).hasSize(2);
+ assertThat(nextRows[0].getUTF8String(0).toString()).isEqualTo("dt=20260703");
+ assertThat(nextRows[1].getUTF8String(0).toString()).isEqualTo("dt=20260704");
+ assertThat(nextRows[0].isNullAt(1)).isTrue();
+ assertThat(nextRows[1].isNullAt(1)).isTrue();
+ assertThat(partitionCatalog.requestedTokens).containsExactly(null, "p2");
+ assertThat(partitionCatalog.requestedPageSizes).containsExactly(2, 2);
+
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"),
+ UTF8String.fromString("id > 0"),
+ 2,
+ null
+ })))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("Only partition predicate is supported");
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void filteredListPagesAreSparseAndIterateByRawTokens() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Arrays.asList(spec("dt", "1"), spec("dt", "skip-a")), "p2"));
+ catalog.addPage(
+ "p2",
+ new FormatTablePartitionPage(
+ Arrays.asList(spec("dt", "skip-b"), spec("dt", "skip-c")), "tail"));
+ catalog.addPage(
+ "tail",
+ new FormatTablePartitionPage(Collections.singletonList(spec("dt", "3")), null));
+ Predicate> numbered =
+ partition -> Character.isDigit(partition.get("dt").charAt(0));
+
+ List> matches = new ArrayList<>();
+ List pageRowCounts = new ArrayList<>();
+ String token = null;
+ do {
+ FormatTablePartitionPage page =
+ ListFormatTablePartitionsProcedure.listPage(
+ catalog, Collections.emptyMap(), token, 2, numbered);
+ pageRowCounts.add(page.partitions().size());
+ matches.addAll(page.partitions());
+ token = page.nextPageToken();
+ } while (token != null);
+
+ assertThat(matches).containsExactly(spec("dt", "1"), spec("dt", "3"));
+ // The next-page token travels on result rows, so a raw page whose rows are all filtered
+ // out is skipped inside the same call instead of surfacing an empty page that would lose
+ // the cursor: the second call transparently advances from 'p2' to 'tail'.
+ assertThat(pageRowCounts).containsExactly(1, 1);
+ assertThat(catalog.requestedTokens).containsExactly(null, "p2", "tail");
+ assertThat(catalog.requestedPageSizes).containsExactly(2, 2, 2);
+ }
+
+ @Test
+ void listPageRejectsOversizedCatalogPage() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Arrays.asList(spec("dt", "1"), spec("dt", "2")), null));
+
+ assertThatThrownBy(
+ () ->
+ ListFormatTablePartitionsProcedure.listPage(
+ catalog, Collections.emptyMap(), null, 1, null))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("returned 2 partitions for a page of 1");
+ }
+
+ @Test
+ void listPageForwardsPrefixStartingTokenAndLimit() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ "start",
+ new FormatTablePartitionPage(
+ Collections.singletonList(spec("dt", "20260701")), "next"));
+
+ FormatTablePartitionPage result =
+ ListFormatTablePartitionsProcedure.listPage(
+ catalog, spec("dt", "202607"), "start", 1, null);
+
+ assertThat(result.partitions()).containsExactly(spec("dt", "20260701"));
+ assertThat(result.nextPageToken()).isEqualTo("next");
+ assertThat(catalog.requestedPrefixes).containsExactly(spec("dt", "202607"));
+ assertThat(catalog.requestedTokens).containsExactly("start");
+ assertThat(catalog.requestedPageSizes).containsExactly(1);
+ }
+
+ @Test
+ void listPageTreatsEmptyNextPageTokenAsTerminal() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Collections.singletonList(spec("dt", "20260701")), ""));
+
+ FormatTablePartitionPage result =
+ ListFormatTablePartitionsProcedure.listPage(
+ catalog, Collections.emptyMap(), null, 2, ignored -> true);
+
+ assertThat(result.partitions()).containsExactly(spec("dt", "20260701"));
+ assertThat(result.nextPageToken()).isNull();
+ assertThat(catalog.requestedTokens).containsExactly((String) null);
+ }
+
+ @Test
+ void syncDryRunReturnsFilesystemMinusCatalogWithoutCreatingPartitions() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Collections.singletonList(spec("dt", "20260701")), null));
+
+ List, String>> actions =
+ SyncFormatTableMetadataProcedure.syncPartitions(
+ catalog,
+ Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")),
+ Collections.singletonList("dt"),
+ true,
+ "ADD");
+
+ assertThat(actions).containsExactly(action(spec("dt", "20260702"), "ADD"));
+ assertThat(catalog.createdPartitions).isEmpty();
+ }
+
+ @Test
+ void syncApplyCreatesTheWholeDiffAsAnIdempotentBatch() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(null, new FormatTablePartitionPage(Collections.emptyList(), null));
+
+ List, String>> actions =
+ SyncFormatTableMetadataProcedure.syncPartitions(
+ catalog,
+ Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")),
+ Collections.singletonList("dt"),
+ false,
+ "ADD");
+
+ assertThat(actions)
+ .containsExactly(
+ action(spec("dt", "20260701"), "ADD"),
+ action(spec("dt", "20260702"), "ADD"));
+ assertThat(catalog.createdPartitions)
+ .containsExactly(Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")));
+ assertThat(catalog.createIgnoreFlags).containsExactly(true);
+ }
+
+ @Test
+ void syncApplyPropagatesCreatePermissionFailureAndRefreshesCache() throws Exception {
+ java.nio.file.Path partitionDirectory =
+ Files.createDirectories(tempDir.resolve("dt=20260701"));
+ Files.write(
+ partitionDirectory.resolve("data.csv"),
+ Collections.singletonList("1"),
+ StandardCharsets.UTF_8);
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("sync-format-table-metadata-permission-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ SecurityException failure = new SecurityException("partition create denied");
+ RecordingCatalog partitionCatalog =
+ new RecordingCatalog() {
+ @Override
+ public void createPartitions(
+ List> partitions, boolean ignoreIfExists) {
+ throw failure;
+ }
+ };
+ partitionCatalog.addPage(
+ null, new FormatTablePartitionPage(Collections.emptyList(), null));
+ AtomicInteger refreshCount = new AtomicInteger();
+ Procedure procedure =
+ new SyncFormatTableMetadataProcedure(
+ tableCatalog(
+ new PaimonFormatTable(
+ formatTable(tempDir.toUri().toString()),
+ partitionCatalog))) {
+ @Override
+ protected void refreshSparkCache(
+ org.apache.spark.sql.connector.catalog.Identifier ident,
+ org.apache.spark.sql.connector.catalog.Table table) {
+ refreshCount.incrementAndGet();
+ }
+ };
+
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"), null, false
+ })))
+ .isSameAs(failure);
+ assertThat(refreshCount).hasValue(1);
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void syncApplyRefreshesAfterAddSucceedsAndDropFails() throws Exception {
+ java.nio.file.Path partitionDirectory =
+ Files.createDirectories(tempDir.resolve("dt=20260715"));
+ Files.write(
+ partitionDirectory.resolve("data.csv"),
+ Collections.singletonList("15"),
+ StandardCharsets.UTF_8);
+
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("sync-format-table-metadata-partial-success-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ IllegalStateException failure =
+ new IllegalStateException("injected partition drop failure");
+ RecordingCatalog partitionCatalog =
+ new RecordingCatalog() {
+ @Override
+ public void dropPartitions(List> partitions) {
+ throw failure;
+ }
+ };
+ partitionCatalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Collections.singletonList(spec("dt", "20260714")), null));
+ AtomicInteger refreshCount = new AtomicInteger();
+ Procedure procedure =
+ new SyncFormatTableMetadataProcedure(
+ tableCatalog(
+ new PaimonFormatTable(
+ formatTable(tempDir.toUri().toString()),
+ partitionCatalog))) {
+ @Override
+ protected void refreshSparkCache(
+ org.apache.spark.sql.connector.catalog.Identifier ident,
+ org.apache.spark.sql.connector.catalog.Table table) {
+ refreshCount.incrementAndGet();
+ }
+ };
+
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"),
+ UTF8String.fromString("SYNC"),
+ false
+ })))
+ .isSameAs(failure);
+ assertThat(partitionCatalog.createdPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260715")));
+ assertThat(refreshCount).hasValue(1);
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void syncProcedureCallFailsClosedWhenSecondCatalogPageFails() throws Exception {
+ java.nio.file.Path partitionDirectory =
+ Files.createDirectories(tempDir.resolve("dt=20260715"));
+ Files.write(
+ partitionDirectory.resolve("data.csv"),
+ Collections.singletonList("15"),
+ StandardCharsets.UTF_8);
+
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("sync-format-table-metadata-second-page-failure-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ IllegalStateException listFailure =
+ new IllegalStateException("injected second catalog page failure");
+ AtomicInteger listCount = new AtomicInteger();
+ RecordingCatalog partitionCatalog =
+ new RecordingCatalog() {
+ @Override
+ public FormatTablePartitionPage listPartitions(
+ Map prefix, String pageToken, int pageSize) {
+ listCount.incrementAndGet();
+ if ("page-2".equals(pageToken)) {
+ throw listFailure;
+ }
+ return super.listPartitions(prefix, pageToken, pageSize);
+ }
+ };
+ partitionCatalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Collections.singletonList(spec("dt", "20260714")), "page-2"));
+ AtomicInteger refreshCount = new AtomicInteger();
+ Procedure procedure =
+ new SyncFormatTableMetadataProcedure(
+ tableCatalog(
+ new PaimonFormatTable(
+ formatTable(tempDir.toUri().toString()),
+ partitionCatalog))) {
+ @Override
+ protected void refreshSparkCache(
+ org.apache.spark.sql.connector.catalog.Identifier ident,
+ org.apache.spark.sql.connector.catalog.Table table) {
+ refreshCount.incrementAndGet();
+ }
+ };
+
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"),
+ UTF8String.fromString("SYNC"),
+ false
+ })))
+ .isSameAs(listFailure);
+ assertThat(listCount).hasValue(2);
+ assertThat(partitionCatalog.createdPartitions).isEmpty();
+ assertThat(partitionCatalog.droppedPartitions).isEmpty();
+ // Every non-preview attempt refreshes the Spark-side caches, even a failed one.
+ assertThat(refreshCount).hasValue(1);
+ assertThat(partitionDirectory).exists();
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void syncProcedureCallFailsClosedWhenFilesystemDiscoveryFailsPartway() throws Exception {
+ Files.createDirectories(tempDir.resolve("dt=20260715/hh=00"));
+ Files.createDirectories(tempDir.resolve("dt=20260716/hh=00"));
+
+ IOException listFailure = new IOException("injected nested filesystem LIST failure");
+ AtomicInteger completedPartitionListings = new AtomicInteger();
+ FileIO fileIO =
+ new LocalFileIO() {
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ if ("dt=20260716".equals(path.getName())) {
+ throw listFailure;
+ }
+ FileStatus[] statuses = super.listStatus(path);
+ Arrays.sort(
+ statuses,
+ java.util.Comparator.comparing(
+ status -> status.getPath().toString()));
+ if ("dt=20260715".equals(path.getName())) {
+ completedPartitionListings.incrementAndGet();
+ }
+ return statuses;
+ }
+ };
+ Map options = new LinkedHashMap<>();
+ options.put(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true");
+ RowType rowType =
+ RowType.builder()
+ .field("id", DataTypes.INT())
+ .field("dt", DataTypes.STRING())
+ .field("hh", DataTypes.STRING())
+ .build();
+ FormatTable partiallyListedTable =
+ FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("db", "t"))
+ .rowType(rowType)
+ .partitionKeys(Arrays.asList("dt", "hh"))
+ .location(new Path(tempDir.toUri().toString()).toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .build();
+
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("sync-format-table-metadata-filesystem-failure-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ AtomicInteger catalogListCount = new AtomicInteger();
+ RecordingCatalog partitionCatalog =
+ new RecordingCatalog() {
+ @Override
+ public FormatTablePartitionPage listPartitions(
+ Map prefix, String pageToken, int pageSize) {
+ catalogListCount.incrementAndGet();
+ return super.listPartitions(prefix, pageToken, pageSize);
+ }
+ };
+ AtomicInteger refreshCount = new AtomicInteger();
+ Procedure procedure =
+ new SyncFormatTableMetadataProcedure(
+ tableCatalog(
+ new PaimonFormatTable(
+ partiallyListedTable, partitionCatalog))) {
+ @Override
+ protected void refreshSparkCache(
+ org.apache.spark.sql.connector.catalog.Identifier ident,
+ org.apache.spark.sql.connector.catalog.Table table) {
+ refreshCount.incrementAndGet();
+ }
+ };
+
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"),
+ UTF8String.fromString("SYNC"),
+ false
+ })))
+ .isInstanceOf(RuntimeException.class)
+ .hasCause(listFailure);
+ assertThat(completedPartitionListings).hasValue(1);
+ assertThat(catalogListCount).hasValue(0);
+ assertThat(partitionCatalog.createdPartitions).isEmpty();
+ assertThat(partitionCatalog.droppedPartitions).isEmpty();
+ // Every non-preview attempt refreshes the Spark-side caches, even a failed one.
+ assertThat(refreshCount).hasValue(1);
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void syncRejectsUnsafeValueOnlyDirectoryBeforeCatalogMutation() throws Exception {
+ Files.createDirectories(tempDir.resolve("%2E%2E"));
+ SparkSession spark =
+ SparkSession.builder()
+ .master("local[1]")
+ .appName("sync-format-table-metadata-value-only-safety-test")
+ .config("spark.ui.enabled", "false")
+ .getOrCreate();
+ try {
+ AtomicInteger catalogListCount = new AtomicInteger();
+ RecordingCatalog partitionCatalog =
+ new RecordingCatalog() {
+ @Override
+ public FormatTablePartitionPage listPartitions(
+ Map prefix, String pageToken, int pageSize) {
+ catalogListCount.incrementAndGet();
+ return super.listPartitions(prefix, pageToken, pageSize);
+ }
+ };
+ Procedure procedure =
+ new SyncFormatTableMetadataProcedure(
+ tableCatalog(
+ new PaimonFormatTable(
+ formatTable(tempDir.toUri().toString(), true),
+ partitionCatalog))) {
+ @Override
+ protected void refreshSparkCache(
+ org.apache.spark.sql.connector.catalog.Identifier ident,
+ org.apache.spark.sql.connector.catalog.Table table) {}
+ };
+
+ assertThatThrownBy(
+ () ->
+ procedure.call(
+ new GenericInternalRow(
+ new Object[] {
+ UTF8String.fromString("db.t"),
+ UTF8String.fromString("SYNC"),
+ false
+ })))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("..");
+ assertThat(catalogListCount).hasValue(0);
+ assertThat(partitionCatalog.createdPartitions).isEmpty();
+ assertThat(partitionCatalog.droppedPartitions).isEmpty();
+ } finally {
+ spark.stop();
+ SparkSession.clearActiveSession();
+ SparkSession.clearDefaultSession();
+ }
+ }
+
+ @Test
+ void syncTreatsEmptyNextPageTokenAsTerminal() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Collections.singletonList(spec("dt", "20260701")), ""));
+
+ List, String>> actions =
+ SyncFormatTableMetadataProcedure.syncPartitions(
+ catalog,
+ Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")),
+ Collections.singletonList("dt"),
+ true,
+ "ADD");
+
+ assertThat(actions).containsExactly(action(spec("dt", "20260702"), "ADD"));
+ assertThat(catalog.requestedTokens).containsExactly((String) null);
+ }
+
+ @Test
+ void syncWithNoDiffDoesNotIssueAnEmptyCreate() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Collections.singletonList(spec("dt", "20260701")), null));
+
+ List, String>> actions =
+ SyncFormatTableMetadataProcedure.syncPartitions(
+ catalog,
+ Collections.singletonList(spec("dt", "20260701")),
+ Collections.singletonList("dt"),
+ false,
+ "SYNC");
+
+ assertThat(actions).isEmpty();
+ assertThat(catalog.createdPartitions).isEmpty();
+ assertThat(catalog.droppedPartitions).isEmpty();
+ }
+
+ @Test
+ void syncRejectsUnsupportedMode() {
+ assertThatThrownBy(() -> SyncFormatTableMetadataProcedure.validateMode("RECOVER"))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("ADD, DROP and SYNC");
+ SyncFormatTableMetadataProcedure.validateMode("ADD");
+ SyncFormatTableMetadataProcedure.validateMode("drop");
+ SyncFormatTableMetadataProcedure.validateMode("Sync");
+ }
+
+ @Test
+ void syncDropModeUnregistersOnlyMissingDirectories() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Arrays.asList(spec("dt", "20260714"), spec("dt", "20260715")), null));
+
+ // dt=20260715 exists on the filesystem, dt=20260714 does not: only the latter is dropped.
+ List, String>> dryRun =
+ SyncFormatTableMetadataProcedure.syncPartitions(
+ catalog,
+ Collections.singletonList(spec("dt", "20260715")),
+ Collections.singletonList("dt"),
+ true,
+ "DROP");
+ assertThat(dryRun).containsExactly(action(spec("dt", "20260714"), "DROP"));
+ assertThat(catalog.droppedPartitions).isEmpty();
+
+ List, String>> applied =
+ SyncFormatTableMetadataProcedure.syncPartitions(
+ catalog,
+ Collections.singletonList(spec("dt", "20260715")),
+ Collections.singletonList("dt"),
+ false,
+ "DROP");
+ assertThat(applied).containsExactly(action(spec("dt", "20260714"), "DROP"));
+ assertThat(catalog.droppedPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260714")));
+ assertThat(catalog.createdPartitions).isEmpty();
+ }
+
+ @Test
+ void syncDropModeAppliesLargeDiffInBoundedBatches() {
+ List> firstPage = new ArrayList<>();
+ for (int index = 0; index < 1000; index++) {
+ firstPage.add(spec("dt", String.format("%04d", index)));
+ }
+
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(null, new FormatTablePartitionPage(firstPage, "next"));
+ catalog.addPage(
+ "next",
+ new FormatTablePartitionPage(Collections.singletonList(spec("dt", "1000")), null));
+
+ List, String>> applied =
+ SyncFormatTableMetadataProcedure.syncPartitions(
+ catalog,
+ Collections.emptyList(),
+ Collections.singletonList("dt"),
+ false,
+ "DROP");
+
+ assertThat(applied).hasSize(1001);
+ assertThat(catalog.droppedPartitions).hasSize(2);
+ assertThat(catalog.droppedPartitions.get(0)).hasSize(1000);
+ assertThat(catalog.droppedPartitions.get(1)).hasSize(1);
+ assertThat(catalog.droppedPartitions.stream().mapToInt(List::size).sum()).isEqualTo(1001);
+ }
+
+ @Test
+ void syncSyncModeAddsAndDropsInOneCall() {
+ RecordingCatalog catalog = new RecordingCatalog();
+ catalog.addPage(
+ null,
+ new FormatTablePartitionPage(
+ Collections.singletonList(spec("dt", "20260714")), null));
+
+ List, String>> applied =
+ SyncFormatTableMetadataProcedure.syncPartitions(
+ catalog,
+ Collections.singletonList(spec("dt", "20260715")),
+ Collections.singletonList("dt"),
+ false,
+ "SYNC");
+ assertThat(applied)
+ .containsExactly(
+ action(spec("dt", "20260715"), "ADD"),
+ action(spec("dt", "20260714"), "DROP"));
+ assertThat(catalog.createdPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260715")));
+ assertThat(catalog.droppedPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260714")));
+ }
+
+ private static Map.Entry, String> action(
+ Map spec, String action) {
+ return new java.util.AbstractMap.SimpleEntry<>(spec, action);
+ }
+
+ private static Map spec(String key, String value) {
+ Map spec = new LinkedHashMap<>();
+ spec.put(key, value);
+ return spec;
+ }
+
+ private static FormatTable formatTable() {
+ return formatTable("file:/tmp/format-table-procedure-test");
+ }
+
+ private static FormatTable formatTable(String location) {
+ return formatTable(location, false);
+ }
+
+ private static FormatTable formatTable(String location, boolean onlyValueInPath) {
+ Map options = new LinkedHashMap<>();
+ options.put(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true");
+ options.put(
+ CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.toString(onlyValueInPath));
+ RowType rowType =
+ RowType.builder()
+ .field("id", DataTypes.INT())
+ .field("dt", DataTypes.STRING())
+ .build();
+ return FormatTable.builder()
+ .fileIO(LocalFileIO.create())
+ .identifier(Identifier.create("db", "t"))
+ .rowType(rowType)
+ .partitionKeys(Collections.singletonList("dt"))
+ .location(new Path(location).toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .build();
+ }
+
+ private static TableCatalog tableCatalog(PaimonFormatTable table) {
+ return (TableCatalog)
+ Proxy.newProxyInstance(
+ FormatTableMetadataProcedureTest.class.getClassLoader(),
+ new Class[] {TableCatalog.class},
+ (proxy, method, args) -> {
+ switch (method.getName()) {
+ case "name":
+ return "test_catalog";
+ case "defaultNamespace":
+ return new String[] {"db"};
+ case "loadTable":
+ return table;
+ case "equals":
+ return proxy == args[0];
+ case "hashCode":
+ return System.identityHashCode(proxy);
+ case "toString":
+ return "test_catalog";
+ default:
+ throw new UnsupportedOperationException(method.getName());
+ }
+ });
+ }
+
+ private static class RecordingCatalog implements FormatTablePartitionCatalog {
+
+ private final Map pages = new LinkedHashMap<>();
+ private final List requestedTokens = new ArrayList<>();
+ private final List requestedPageSizes = new ArrayList<>();
+ private final List> requestedPrefixes = new ArrayList<>();
+ private final List>> createdPartitions = new ArrayList<>();
+ private final List createIgnoreFlags = new ArrayList<>();
+ private final List>> droppedPartitions = new ArrayList<>();
+
+ private void addPage(String token, FormatTablePartitionPage page) {
+ pages.put(token, page);
+ }
+
+ @Override
+ public void createPartitions(List> partitions, boolean ignoreIfExists) {
+ createdPartitions.add(new ArrayList<>(partitions));
+ createIgnoreFlags.add(ignoreIfExists);
+ }
+
+ @Override
+ public void dropPartitions(List> partitions) {
+ droppedPartitions.add(new ArrayList<>(partitions));
+ }
+
+ @Override
+ public List> listPartitionsByNames(
+ List> partitions) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public FormatTablePartitionPage listPartitions(
+ Map prefix, String pageToken, int pageSize) {
+ requestedPrefixes.add(prefix == null ? null : new LinkedHashMap<>(prefix));
+ requestedTokens.add(pageToken);
+ requestedPageSizes.add(pageSize);
+ return pages.get(pageToken);
+ }
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala
new file mode 100644
index 000000000000..4f8506c07bfd
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.util
+
+import org.apache.paimon.CoreOptions.{FORMAT_TABLE_IMPLEMENTATION, METASTORE_PARTITIONED_TABLE}
+import org.apache.paimon.catalog.Identifier
+import org.apache.paimon.fs.local.LocalFileIO
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.table.FormatTable.Format
+import org.apache.paimon.types.{DataTypes, RowType}
+
+import org.apache.spark.sql.internal.SQLConf
+import org.scalatest.funsuite.AnyFunSuite
+
+import java.util.Collections
+
+import scala.collection.JavaConverters._
+
+/** Tests for [[OptionUtils]]. */
+class OptionUtilsTest extends AnyFunSuite {
+
+ test("reject engine implementation from SQL conf for managed format table") {
+ val exception = intercept[IllegalArgumentException] {
+ SQLConf.withExistingConf(engineSQLConf) {
+ OptionUtils.copyWithSQLConf(formatTable(managed = true))
+ }
+ }
+
+ assert(exception.getMessage.contains(METASTORE_PARTITIONED_TABLE.key()))
+ assert(exception.getMessage.contains(FORMAT_TABLE_IMPLEMENTATION.key()))
+ }
+
+ test("allow engine implementation from SQL conf for unmanaged format table") {
+ val copied = SQLConf.withExistingConf(engineSQLConf) {
+ OptionUtils.copyWithSQLConf(formatTable(managed = false))
+ }
+
+ assert(copied.options().get(FORMAT_TABLE_IMPLEMENTATION.key()) == "engine")
+ }
+
+ test("reject invalid managed format table option from SQL conf with option context") {
+ val sqlConf = new SQLConf
+ sqlConf.setConfString(s"spark.paimon.${METASTORE_PARTITIONED_TABLE.key()}", "yes")
+
+ val exception = intercept[IllegalArgumentException] {
+ SQLConf.withExistingConf(sqlConf) {
+ OptionUtils.copyWithSQLConf(formatTable(managed = false))
+ }
+ }
+
+ assert(exception.getMessage.contains("yes"))
+ assert(exception.getMessage.contains(METASTORE_PARTITIONED_TABLE.key()))
+ }
+
+ test("session-level metastore.partitioned-table is ignored, not failed, on a managed table") {
+ val sqlConf = new SQLConf
+ sqlConf.setConfString(s"spark.paimon.${METASTORE_PARTITIONED_TABLE.key()}", "false")
+
+ val copied = SQLConf.withExistingConf(sqlConf) {
+ OptionUtils.copyWithSQLConf(formatTable(managed = true))
+ }
+
+ // The persisted managed flag wins; the session-global override is dropped with a warning
+ // instead of failing every managed format table load in the session.
+ assert(copied.options().get(METASTORE_PARTITIONED_TABLE.key()) == "true")
+ }
+
+ test("session-level metastore.partitioned-table is ignored, not failed, on an unmanaged table") {
+ val sqlConf = new SQLConf
+ sqlConf.setConfString(s"spark.paimon.${METASTORE_PARTITIONED_TABLE.key()}", "true")
+
+ val copied = SQLConf.withExistingConf(sqlConf) {
+ OptionUtils.copyWithSQLConf(formatTable(managed = false))
+ }
+
+ assert(copied.options().get(METASTORE_PARTITIONED_TABLE.key()) == "false")
+ }
+
+ private def engineSQLConf: SQLConf = {
+ val sqlConf = new SQLConf
+ sqlConf.setConfString(
+ s"spark.paimon.${FORMAT_TABLE_IMPLEMENTATION.key()}",
+ "engine"
+ )
+ sqlConf
+ }
+
+ private def formatTable(managed: Boolean): FormatTable = {
+ FormatTable
+ .builder()
+ .fileIO(new LocalFileIO)
+ .identifier(Identifier.create("test_db", "format_table"))
+ .rowType(RowType.of(DataTypes.INT(), DataTypes.STRING()))
+ .partitionKeys(Collections.singletonList("dt"))
+ .location("file:///tmp/test_db.db/format_table")
+ .format(Format.PARQUET)
+ .options(Map(
+ METASTORE_PARTITIONED_TABLE.key() -> managed.toString,
+ FORMAT_TABLE_IMPLEMENTATION.key() -> "paimon").asJava)
+ .build()
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExecTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExecTest.scala
new file mode 100644
index 000000000000..707f051e4f87
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/commands/PaimonShowFormatTablePartitionsExecTest.scala
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.commands
+
+import org.apache.paimon.catalog.{CatalogContext, Identifier}
+import org.apache.paimon.fs.Path
+import org.apache.paimon.fs.local.LocalFileIO
+import org.apache.paimon.options.Options
+import org.apache.paimon.spark.PaimonSparkTestBase
+import org.apache.paimon.spark.format.{FormatTablePartitionCatalog, FormatTablePartitionPage, PaimonFormatTable}
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.types.DataTypes
+
+import org.apache.spark.sql.catalyst.analysis.ResolvedPartitionSpec
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GenericInternalRow}
+import org.apache.spark.sql.execution.SparkPlan
+import org.apache.spark.sql.types.StringType
+import org.apache.spark.unsafe.types.UTF8String
+
+import java.nio.file.Files
+import java.util.{Collections, LinkedHashMap, List => JList, Map => JMap}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
+
+class PaimonShowFormatTablePartitionsExecTest extends PaimonSparkTestBase {
+
+ test("managed Format Table SHOW PARTITIONS has a bounded catalog executor") {
+ Class.forName("org.apache.paimon.spark.commands.PaimonShowFormatTablePartitionsExec")
+ }
+
+ test("managed Format Table SHOW PARTITIONS exposes its default Driver cap") {
+ assert(PaimonShowFormatTablePartitionsExec.DEFAULT_MAX_RESULTS == 10000)
+ }
+
+ test("managed Format Table SHOW PARTITIONS pages catalog results and uses Spark formatting") {
+ val calls = ArrayBuffer.empty[(Map[String, String], String, Int)]
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = unsupported()
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = unsupported()
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = unsupported()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ calls += ((prefix.asScala.toMap, pageToken, pageSize))
+ pageToken match {
+ case null => FormatTablePartitionPage(List(partition("z")).asJava, "page-2")
+ case "page-2" => FormatTablePartitionPage(List(partition("a=1")).asJava, null)
+ case token => fail(s"Unexpected page token $token")
+ }
+ }
+ }
+ val table = new PaimonFormatTable(managedFormatTable(), gateway)
+ val spec = ResolvedPartitionSpec(
+ Seq("dt"),
+ new GenericInternalRow(Array[Any](UTF8String.fromString("2026/07/15"))))
+ val output = Seq(AttributeReference("partition", StringType, nullable = false)())
+ val constructor = classOf[PaimonShowFormatTablePartitionsExec].getConstructors
+ .find(_.getParameterCount == 4)
+ .getOrElse(fail("The catalog SHOW executor constructor is missing"))
+ val exec = constructor
+ .newInstance(output, table, Some(spec), Int.box(3))
+ .asInstanceOf[SparkPlan]
+
+ val actual = exec.executeCollect().map(_.getUTF8String(0).toString).toSeq
+
+ assert(actual == Seq("dt=2026%2F07%2F15/region=a%3D1", "dt=2026%2F07%2F15/region=z"))
+ assert(
+ calls.toSeq == Seq(
+ (Map("dt" -> "2026/07/15"), null, 4),
+ (Map("dt" -> "2026/07/15"), "page-2", 3)))
+ }
+
+ test("managed Format Table SHOW PARTITIONS rejects results above the Driver cap") {
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = unsupported()
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = unsupported()
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = unsupported()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ assert(pageSize == 3)
+ FormatTablePartitionPage(List(partition("a"), partition("b"), partition("c")).asJava, null)
+ }
+ }
+ val exec = PaimonShowFormatTablePartitionsExec(
+ Seq(AttributeReference("partition", StringType, nullable = false)()),
+ new PaimonFormatTable(managedFormatTable(), gateway),
+ None,
+ maxResults = 2)
+
+ val error = intercept[RuntimeException] {
+ exec.executeCollect()
+ }
+
+ assert(error.getMessage.contains("2"))
+ assert(error.getMessage.contains("CALL sys.list_format_table_partitions"))
+ }
+
+ test("managed Format Table SHOW PARTITIONS continues after an empty catalog page") {
+ val tokens = ArrayBuffer.empty[String]
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = unsupported()
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = unsupported()
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = unsupported()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ tokens += pageToken
+ pageToken match {
+ case null => FormatTablePartitionPage(Collections.emptyList(), "page-2")
+ case "page-2" => FormatTablePartitionPage(List(partition("a")).asJava, null)
+ case token => fail(s"Unexpected page token $token")
+ }
+ }
+ }
+ val exec = PaimonShowFormatTablePartitionsExec(
+ Seq(AttributeReference("partition", StringType, nullable = false)()),
+ new PaimonFormatTable(managedFormatTable(), gateway),
+ None,
+ maxResults = 2)
+
+ val actual = exec.executeCollect().map(_.getUTF8String(0).toString).toSeq
+
+ assert(actual == Seq("dt=2026%2F07%2F15/region=a"))
+ assert(tokens.toSeq == Seq(null, "page-2"))
+ }
+
+ test("managed Format Table SHOW PARTITIONS treats an empty next-page token as terminal") {
+ var calls = 0
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = unsupported()
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = unsupported()
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = unsupported()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ calls += 1
+ if (calls > 1) {
+ fail("An empty page token must not be fetched")
+ }
+ FormatTablePartitionPage(List(partition("a")).asJava, "")
+ }
+ }
+ val exec = PaimonShowFormatTablePartitionsExec(
+ Seq(AttributeReference("partition", StringType, nullable = false)()),
+ new PaimonFormatTable(managedFormatTable(), gateway),
+ None,
+ maxResults = 2)
+
+ val actual = exec.executeCollect().map(_.getUTF8String(0).toString).toSeq
+
+ assert(actual == Seq("dt=2026%2F07%2F15/region=a"))
+ assert(calls == 1)
+ }
+
+ private def partition(region: String): JMap[String, String] = {
+ val result = new LinkedHashMap[String, String]()
+ result.put("region", region)
+ result.put("dt", "2026/07/15")
+ result
+ }
+
+ private def managedFormatTable(): FormatTable = {
+ FormatTable
+ .builder()
+ .fileIO(LocalFileIO.create)
+ .identifier(Identifier.create("test_db", "format_table"))
+ .rowType(
+ DataTypes.ROW(
+ DataTypes.FIELD(0, "id", DataTypes.INT()),
+ DataTypes.FIELD(1, "dt", DataTypes.STRING()),
+ DataTypes.FIELD(2, "region", DataTypes.STRING())))
+ .partitionKeys(Seq("dt", "region").asJava)
+ .location(new Path(Files.createTempDirectory("format_table_show").toUri).toString)
+ .format(FormatTable.Format.CSV)
+ .options(Map("metastore.partitioned-table" -> "true").asJava)
+ .catalogContext(CatalogContext.create(new Options))
+ .build()
+ }
+
+ private def unsupported[T](): T = {
+ throw new UnsupportedOperationException("Not used by SHOW PARTITIONS")
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala
new file mode 100644
index 000000000000..71a14cec72db
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala
@@ -0,0 +1,1019 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.execution
+
+import org.apache.paimon.catalog.{CatalogContext, Identifier}
+import org.apache.paimon.fs.{FileIO, Path}
+import org.apache.paimon.fs.local.LocalFileIO
+import org.apache.paimon.options.Options
+import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase
+import org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions
+import org.apache.paimon.spark.commands.PaimonShowFormatTablePartitionsExec
+import org.apache.paimon.spark.format.{FormatTablePartitionCatalog, FormatTablePartitionPage, PaimonFormatTable}
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.types.DataTypes
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec, ResolvedTable}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GenericInternalRow}
+import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, DropPartitions, RepairTable, ShowPartitions}
+import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier, TableCatalog}
+import org.apache.spark.sql.types.StringType
+
+import java.io.IOException
+import java.lang.reflect.InvocationTargetException
+import java.nio.file.Files
+import java.util.{Collections, List => JList, Map => JMap}
+import java.util.concurrent.{Callable, Executors, TimeUnit}
+
+import scala.collection.JavaConverters._
+
+class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalogBase {
+
+ test("strategy preserves the complete ADD batch for the managed Format Table command") {
+ val (table, _) = formatTable(managed = true)
+ val resolved = ResolvedTable.create(
+ spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog],
+ SparkIdentifier.of(Array("test"), "format_table"),
+ table)
+ val parts = Seq(partition(20260715, 10), partition(20260716, 11))
+
+ val plans = PaimonStrategy(spark).apply(AddPartitions(resolved, parts, ifNotExists = true))
+
+ assert(plans.size == 1)
+ val add = plans.head.asInstanceOf[PaimonAddFormatTablePartitionsExec]
+ assert(add.partSpecs == parts)
+ assert(add.ignoreIfExists)
+ }
+
+ test("managed ADD performs no client-side existence lookup and forwards one atomic batch") {
+ val (table, gateway) = formatTable(managed = true)
+ var refreshCalls = 0
+ val parts = Seq(partition(20260715, 10), partition(20260716, 11))
+ val command = PaimonAddFormatTablePartitionsExec(
+ table,
+ parts,
+ ignoreIfExists = true,
+ () => refreshCalls += 1)
+
+ runCommand(command)
+
+ assert(gateway.createCalls == 1)
+ assert(gateway.lookupCalls == 0)
+ assert(gateway.ignoreIfExists)
+ assert(
+ gateway.created.map(_.asScala.toMap) == Seq(
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260716", "hh" -> "11")))
+ assert(refreshCalls == 1)
+ }
+
+ test("mock service owns partial repeats, all repeats, atomic failure, and concurrent ADD") {
+ val gateway = new AtomicGateway(Set(Map("dt" -> "20260715", "hh" -> "10")))
+ val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway)
+ val existing = partition(20260715, 10)
+ val added = partition(20260716, 11)
+
+ runCommand(
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ Seq(existing, added),
+ ignoreIfExists = true,
+ () => ()))
+ runCommand(
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ Seq(existing, added),
+ ignoreIfExists = true,
+ () => ()))
+
+ assert(
+ gateway.state == Set(
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260716", "hh" -> "11")))
+ assert(gateway.batches.take(2).forall(_.size == 2))
+
+ val ordinaryError = intercept[IllegalStateException] {
+ runCommand(
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ Seq(existing, partition(20260717, 12)),
+ ignoreIfExists = false,
+ () => ()))
+ }
+ assert(ordinaryError.getMessage.contains("already exists"))
+ assert(!gateway.state.exists(_("dt") == "20260717"))
+
+ val concurrent = partition(20260718, 13)
+ val executor = Executors.newFixedThreadPool(8)
+ try {
+ val tasks = (1 to 20).map {
+ _ =>
+ executor.submit(new Callable[Unit] {
+ override def call(): Unit = runCommand(
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ Seq(concurrent),
+ ignoreIfExists = true,
+ () => ()))
+ })
+ }
+ tasks.foreach(_.get(30, TimeUnit.SECONDS))
+ } finally {
+ executor.shutdownNow()
+ }
+ assert(gateway.state.count(_("dt") == "20260718") == 1)
+ }
+
+ test("unmanaged ADD and DROP always fail with an explicit unsupported error") {
+ val (table, gateway) = formatTable(managed = false)
+ val part = partition(20260715, 10)
+
+ val addError = intercept[UnsupportedOperationException] {
+ runCommand(
+ PaimonAddFormatTablePartitionsExec(table, Seq(part), ignoreIfExists = false, () => ()))
+ }
+ assert(addError.getMessage.contains("ADD PARTITION is not supported"))
+ assert(addError.getMessage.contains("unmanaged Format Table"))
+
+ val dropError = intercept[UnsupportedOperationException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(part),
+ ifExists = true,
+ purge = false,
+ () => ()))
+ }
+ assert(dropError.getMessage.contains("DROP PARTITION is not supported"))
+ assert(dropError.getMessage.contains("unmanaged Format Table"))
+ assert(gateway.createCalls == 0)
+ assert(gateway.dropCalls == 0)
+ assert(gateway.lookupCalls == 0)
+ }
+
+ test("SQL unmanaged Format Table partition DDL always fails with a clear error") {
+ val tableName = "unmanaged_format_partition_ddl"
+ withTable(tableName) {
+ sql(
+ s"CREATE TABLE $tableName (id INT, dt INT, hh INT) USING CSV " +
+ "TBLPROPERTIES ('format-table.implementation'='paimon') PARTITIONED BY (dt, hh)")
+
+ val addError = intercept[Exception] {
+ sql(s"ALTER TABLE $tableName ADD PARTITION (dt=20260715, hh=10)").collect()
+ }
+ assert(causeMessages(addError).contains("ADD PARTITION is not supported"))
+
+ val dropError = intercept[Exception] {
+ sql(s"ALTER TABLE $tableName DROP IF EXISTS PARTITION (dt=20260715, hh=10)").collect()
+ }
+ assert(causeMessages(dropError).contains("DROP PARTITION is not supported"))
+
+ // Partial specs keep Spark's strict partition-spec resolution for unmanaged Format Tables.
+ intercept[AnalysisException] {
+ sql(s"ALTER TABLE $tableName DROP PARTITION (dt=20260715)").collect()
+ }
+ }
+ }
+
+ test("rewrite routes managed Format Table DROP PARTITION through PaimonDropPartitions") {
+ val managedName = "managed_format_drop_rewrite"
+ val unmanagedName = "unmanaged_format_drop_rewrite"
+ withTable(managedName, unmanagedName) {
+ sql(s"""CREATE TABLE $managedName (id INT, dt STRING, hh STRING)
+ |USING CSV
+ |PARTITIONED BY (dt, hh)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'metastore.partitioned-table' = 'true')
+ |""".stripMargin)
+ sql(s"""CREATE TABLE $unmanagedName (id INT, dt STRING, hh STRING)
+ |USING CSV
+ |PARTITIONED BY (dt, hh)
+ |TBLPROPERTIES ('format-table.implementation' = 'paimon')
+ |""".stripMargin)
+
+ // RewriteSparkDDLCommands runs inside the extension parser, so parsePlan already returns
+ // the rewritten plan for catalog-managed Format Tables.
+ val managedPlan = spark.sessionState.sqlParser.parsePlan(
+ s"ALTER TABLE $managedName DROP IF EXISTS PARTITION (dt='20260715')")
+ assert(managedPlan.isInstanceOf[PaimonDropPartitions])
+ val paimonDrop = managedPlan.asInstanceOf[PaimonDropPartitions]
+ assert(paimonDrop.ifExists)
+ assert(!paimonDrop.purge)
+
+ // Unmanaged Format Tables keep Spark's own DropPartitions and strict spec resolution.
+ val unmanagedPlan = spark.sessionState.sqlParser.parsePlan(
+ s"ALTER TABLE $unmanagedName DROP IF EXISTS PARTITION (dt='20260715')")
+ assert(unmanagedPlan.isInstanceOf[DropPartitions])
+ }
+ }
+
+ test("SQL partial DROP on a managed Format Table unregisters the matching partitions") {
+ val tableName = "managed_format_partial_drop_sql"
+ withTable(tableName) {
+ sql(s"""CREATE TABLE $tableName (id INT, dt STRING, hh STRING)
+ |USING CSV
+ |PARTITIONED BY (dt, hh)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'metastore.partitioned-table' = 'true')
+ |""".stripMargin)
+ registerPartitions(
+ tableName,
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260715", "hh" -> "11"),
+ Map("dt" -> "20260716", "hh" -> "10"))
+
+ // Leading partial spec expands to every registered leaf partition below it.
+ sql(s"ALTER TABLE $tableName DROP PARTITION (dt='20260715')")
+ assert(shownPartitions(tableName) == Set("dt=20260716/hh=10"))
+
+ // Complete specs honor IF EXISTS and fail loudly for unregistered partitions.
+ sql(s"ALTER TABLE $tableName DROP IF EXISTS PARTITION (dt='20990101', hh='00')")
+ val missingError = intercept[Exception] {
+ sql(s"ALTER TABLE $tableName DROP PARTITION (dt='20990101', hh='00')").collect()
+ }
+ assert(
+ Iterator
+ .iterate(missingError: Throwable)(_.getCause)
+ .takeWhile(_ != null)
+ .exists(_.isInstanceOf[NoSuchPartitionsException]))
+ assert(shownPartitions(tableName) == Set("dt=20260716/hh=10"))
+
+ // Non-leading partial specs resolve by partition name through the gateway.
+ sql(s"ALTER TABLE $tableName DROP PARTITION (hh='10')")
+ assert(shownPartitions(tableName) == Set.empty)
+ }
+ }
+
+ test("strategy preserves the DROP batch and managed DROP unregisters through the gateway") {
+ val (table, gateway) = formatTable(managed = true)
+ val resolved = ResolvedTable.create(
+ spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog],
+ SparkIdentifier.of(Array("test"), "format_table"),
+ table)
+ val parts = Seq(partition(20260715, 10))
+ val logical = PaimonDropPartitions(resolved, parts, true, false)
+
+ val plans = PaimonStrategy(spark).apply(logical)
+ assert(plans.size == 1)
+ val command = plans.head.asInstanceOf[PaimonDropFormatTablePartitionsExec]
+ assert(command.partSpecs == parts)
+ assert(command.ifExists)
+ assert(!command.purge)
+
+ var refreshCalls = 0
+ runCommand(command.copy(refreshCache = () => refreshCalls += 1))
+ assert(gateway.lookupCalls == 1)
+ assert(gateway.dropCalls == 1)
+ assert(gateway.dropped.map(_.asScala.toMap) == Seq(Map("dt" -> "20260715", "hh" -> "10")))
+ assert(refreshCalls == 1)
+ }
+
+ test("strategy threads IF EXISTS and PURGE flags into the format-table DROP command") {
+ val (table, _) = formatTable(managed = true)
+ val resolved = ResolvedTable.create(
+ spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog],
+ SparkIdentifier.of(Array("test"), "format_table"),
+ table)
+ val parts = Seq(partition(20260715, 10))
+
+ val plans =
+ PaimonStrategy(spark).apply(DropPartitions(resolved, parts, ifExists = false, purge = true))
+
+ assert(plans.size == 1)
+ val command = plans.head.asInstanceOf[PaimonDropFormatTablePartitionsExec]
+ assert(command.partSpecs == parts)
+ assert(!command.ifExists)
+ assert(command.purge)
+ }
+
+ test("managed DROP PARTITION PURGE is rejected before any catalog access") {
+ val (table, gateway) = formatTable(managed = true)
+
+ val error = intercept[UnsupportedOperationException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(partition(20260715, 10)),
+ ifExists = false,
+ purge = true,
+ () => fail("PURGE must not refresh the cache")))
+ }
+
+ assert(error.getMessage.contains("PURGE"))
+ assert(gateway.lookupCalls == 0)
+ assert(gateway.dropCalls == 0)
+ }
+
+ test("managed complete DROP without IF EXISTS fails for unregistered partitions") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("format-table-drop-unregistered").toUri)
+ val pendingDir = new Path(tablePath, "dt=20260715/hh=10")
+ var dropCalls = 0
+ var refreshCalls = 0
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ throw new AssertionError("Complete specs must resolve through list-by-names")
+ }
+
+ try {
+ fileIO.mkdirs(pendingDir)
+ val table = new PaimonFormatTable(rawFormatTable(managed = true, tablePath.toString), gateway)
+
+ intercept[NoSuchPartitionsException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(partition(20260715, 10)),
+ ifExists = false,
+ purge = false,
+ () => refreshCalls += 1))
+ }
+
+ // The unregistered directory (e.g. awaiting MSCK registration) must survive untouched.
+ assert(dropCalls == 0)
+ assert(refreshCalls == 0)
+ assert(fileIO.exists(pendingDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed DROP IF EXISTS drops only registered partitions and preserves pending data") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("format-table-drop-if-exists").toUri)
+ val registeredSpec = Map("dt" -> "20260716", "hh" -> "11")
+ val registeredDir = new Path(tablePath, "dt=20260716/hh=11")
+ val pendingDir = new Path(tablePath, "dt=20260715/hh=10")
+ var lookedUp = Seq.empty[Map[String, String]]
+ var dropped = Seq.empty[Map[String, String]]
+ var refreshCalls = 0
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = {
+ lookedUp = partitions.asScala.map(_.asScala.toMap).toSeq
+ partitions.asScala.filter(_.asScala.toMap == registeredSpec).asJava
+ }
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ throw new AssertionError("Complete specs must resolve through list-by-names")
+ }
+
+ try {
+ fileIO.mkdirs(pendingDir)
+ fileIO.mkdirs(registeredDir)
+ val table = new PaimonFormatTable(rawFormatTable(managed = true, tablePath.toString), gateway)
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(partition(20260715, 10), partition(20260716, 11)),
+ ifExists = true,
+ purge = false,
+ () => refreshCalls += 1))
+
+ assert(
+ lookedUp == Seq(
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260716", "hh" -> "11")))
+ assert(dropped == Seq(registeredSpec))
+ assert(fileIO.exists(pendingDir))
+ assert(!fileIO.exists(registeredDir))
+ assert(refreshCalls == 1)
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed failed directory deletion stays unregistered and preserves pending data") {
+ val tablePath = new Path(Files.createTempDirectory("format-table-drop-retry-safe").toUri)
+ val failedSpec = Map("dt" -> "20260715", "hh" -> "10")
+ val pendingSpec = Map("dt" -> "20260716", "hh" -> "11")
+ val failedDir = new Path(tablePath, "dt=20260715/hh=10")
+ val pendingDir = new Path(tablePath, "dt=20260716/hh=11")
+ var failFirstDelete = true
+ val fileIO = new LocalFileIO {
+ override def delete(path: Path, recursive: Boolean): Boolean = {
+ if (path == failedDir && failFirstDelete) {
+ failFirstDelete = false
+ throw new IOException("injected exact partition delete failure")
+ }
+ super.delete(path, recursive)
+ }
+ }
+ var registered = Set(failedSpec)
+ var compensationCreates = Seq.empty[(Seq[Map[String, String]], Boolean)]
+ def newGateway(): FormatTablePartitionCatalog = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ val specs = partitions.asScala.map(_.asScala.toMap).toSeq
+ compensationCreates :+= ((specs, ignoreIfExists))
+ registered ++= specs
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ registered --= partitions.asScala.map(_.asScala.toMap)
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ partitions.asScala
+ .map(_.asScala.toMap)
+ .filter(registered.contains)
+ .map(_.asJava)
+ .asJava
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ throw new AssertionError("Exact specs must resolve through list-by-names")
+ }
+ var refreshCalls = 0
+
+ try {
+ fileIO.mkdirs(failedDir)
+ fileIO.mkdirs(pendingDir)
+ val firstTable =
+ new PaimonFormatTable(
+ rawFormatTable(managed = true, tablePath.toString, fileIO),
+ newGateway())
+
+ val firstError = intercept[IOException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ firstTable,
+ Seq(partition(20260715, 10)),
+ ifExists = true,
+ purge = false,
+ () => refreshCalls += 1))
+ }
+ assert(firstError.getMessage.contains("injected exact partition delete failure"))
+ // The failed partition stays unregistered and is never recreated automatically.
+ assert(registered.isEmpty)
+ assert(compensationCreates.isEmpty)
+ assert(fileIO.exists(failedDir))
+ assert(fileIO.exists(pendingDir))
+ assert(refreshCalls == 1)
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed ambiguous catalog response does not recreate registration") {
+ val tablePath = new Path(Files.createTempDirectory("format-table-drop-ambiguous").toUri)
+ val droppedSpec = Map("dt" -> "20260715", "hh" -> "10")
+ val droppedDir = new Path(tablePath, "dt=20260715/hh=10")
+ val pendingDir = new Path(tablePath, "dt=20260716/hh=11")
+ val lostResponse = new IOException("injected lost catalog DROP response")
+ var failAfterCommit = true
+ var registered = Set(droppedSpec)
+ var compensationCreates = Seq.empty[(Seq[Map[String, String]], Boolean)]
+ def newGateway(): FormatTablePartitionCatalog = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ val specs = partitions.asScala.map(_.asScala.toMap).toSeq
+ compensationCreates :+= ((specs, ignoreIfExists))
+ registered ++= specs
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ registered --= partitions.asScala.map(_.asScala.toMap)
+ if (failAfterCommit) {
+ failAfterCommit = false
+ throw lostResponse
+ }
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ partitions.asScala
+ .map(_.asScala.toMap)
+ .filter(registered.contains)
+ .map(_.asJava)
+ .asJava
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ throw new AssertionError("Exact specs must resolve through list-by-names")
+ }
+ val firstFileIO = LocalFileIO.create
+ var refreshCalls = 0
+
+ try {
+ firstFileIO.mkdirs(droppedDir)
+ firstFileIO.mkdirs(pendingDir)
+ val firstTable = new PaimonFormatTable(
+ rawFormatTable(managed = true, tablePath.toString, firstFileIO),
+ newGateway())
+
+ val firstError = intercept[IOException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ firstTable,
+ Seq(partition(20260715, 10)),
+ ifExists = true,
+ purge = false,
+ () => refreshCalls += 1))
+ }
+
+ assert(firstError eq lostResponse)
+ assert(registered.isEmpty)
+ assert(compensationCreates.isEmpty)
+ assert(firstFileIO.exists(droppedDir))
+ assert(firstFileIO.exists(pendingDir))
+ assert(refreshCalls == 1)
+ } finally {
+ firstFileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed DROP resolves a non-leading partial spec by partition name") {
+ var listedPrefixes = Seq.empty[Map[String, String]]
+ var dropped = Seq.empty[Map[String, String]]
+ val matching = Map("dt" -> "20260715", "hh" -> "10")
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ listedPrefixes :+= prefix.asScala.toMap
+ FormatTablePartitionPage(Seq(matching.asJava).asJava, null)
+ }
+ }
+ val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway)
+ val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10)))
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(hhOnly),
+ ifExists = false,
+ purge = false,
+ () => ()))
+
+ assert(listedPrefixes == Seq(Map.empty))
+ assert(dropped == Seq(matching))
+ }
+
+ test("managed multiple non-leading partial DROP specs share one catalog traversal") {
+ val first = Map("dt" -> "20260715", "hh" -> "10")
+ val second = Map("dt" -> "20260716", "hh" -> "11")
+ val third = Map("dt" -> "20260717", "hh" -> "10")
+ val unrelated = Map("dt" -> "20260718", "hh" -> "12")
+ var listedPrefixes = Seq.empty[Map[String, String]]
+ var listedTokens = Seq.empty[String]
+ var listedPageSizes = Seq.empty[Int]
+ var dropped = Seq.empty[Map[String, String]]
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ val requested = prefix.asScala.toMap
+ listedPrefixes :+= requested
+ listedTokens :+= pageToken
+ listedPageSizes :+= pageSize
+ pageToken match {
+ case null =>
+ FormatTablePartitionPage(Seq(first, unrelated).map(_.asJava).asJava, "page-2")
+ case "page-2" =>
+ FormatTablePartitionPage(Seq(second, third).map(_.asJava).asJava, null)
+ case other =>
+ throw new AssertionError(s"Unexpected page token $other")
+ }
+ }
+ }
+ val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway)
+ val hhTen = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10)))
+ val hhEleven = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](11)))
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(hhTen, hhEleven),
+ ifExists = false,
+ purge = false,
+ () => ()))
+
+ assert(listedPrefixes == Seq(Map.empty, Map.empty))
+ assert(listedTokens == Seq(null, "page-2"))
+ assert(listedPageSizes == Seq(1000, 1000))
+ assert(dropped == Seq(first, second, third))
+ assert(dropped.distinct == dropped)
+ }
+
+ test("managed DROP with an empty batch does not refresh cache") {
+ val (table, gateway) = formatTable(managed = true)
+ var refreshCalls = 0
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq.empty,
+ ifExists = false,
+ purge = false,
+ () => refreshCalls += 1))
+
+ assert(gateway.dropCalls == 0)
+ assert(refreshCalls == 0)
+ }
+
+ test("managed partial DROP consumes every catalog page before unregistering") {
+ var pageTokens = Seq.empty[Option[String]]
+ var dropped = Seq.empty[Map[String, String]]
+ val first = Map("dt" -> "20260715", "hh" -> "10")
+ val second = Map("dt" -> "20260716", "hh" -> "10")
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ pageTokens :+= Option(pageToken)
+ Option(pageToken) match {
+ case None => FormatTablePartitionPage(Seq(first.asJava).asJava, "page-2")
+ case Some("page-2") => FormatTablePartitionPage(Seq(second.asJava).asJava, null)
+ case other => throw new AssertionError(s"Unexpected page token $other")
+ }
+ }
+ }
+ val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway)
+ val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10)))
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(hhOnly),
+ ifExists = false,
+ purge = false,
+ () => ()))
+
+ assert(pageTokens == Seq(None, Some("page-2")))
+ assert(dropped == Seq(first, second))
+ }
+
+ test("managed partial DROP rejects an incomplete catalog result before mutation") {
+ var dropCalls = 0
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Seq(Map("dt" -> "20260715").asJava).asJava, null)
+ }
+ val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway)
+ val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10)))
+
+ val error = intercept[IllegalStateException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(hhOnly),
+ ifExists = false,
+ purge = false,
+ () => ()))
+ }
+
+ assert(error.getMessage.contains("complete partition spec"))
+ assert(dropCalls == 0)
+ }
+
+ test("managed partial DROP treats an empty next-page token as terminal") {
+ var listCalls = 0
+ var dropCalls = 0
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ listCalls += 1
+ if (listCalls > 1) {
+ throw new AssertionError("Empty page token must not be fetched")
+ }
+ FormatTablePartitionPage(Collections.emptyList(), "")
+ }
+ }
+ val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway)
+ val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10)))
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(hhOnly),
+ ifExists = false,
+ purge = false,
+ () => ()))
+
+ assert(listCalls == 1)
+ assert(dropCalls == 0)
+ }
+
+ test("MSCK strategy intercepts only managed Format Tables and maps the mode flags") {
+ val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog]
+ val identifier = SparkIdentifier.of(Array("test"), "format_table")
+ val (managed, _) = formatTable(managed = true)
+ val (unmanaged, _) = formatTable(managed = false)
+ val managedResolved = ResolvedTable.create(catalog, identifier, managed)
+
+ // (plain MSCK) -> ADD; DROP PARTITIONS -> DROP; SYNC PARTITIONS -> both.
+ Seq((true, false), (false, true), (true, true)).foreach {
+ case (add, drop) =>
+ val plans = PaimonStrategy(spark).apply(RepairTable(managedResolved, add, drop))
+ assert(plans.size == 1)
+ val repair = plans.head.asInstanceOf[PaimonRepairFormatTablePartitionsExec]
+ assert(repair.addPartitions == add)
+ assert(repair.dropPartitions == drop)
+ }
+
+ // Unmanaged tables keep Spark's own v2 rejection: no interception.
+ val unmanagedPlans = PaimonStrategy(spark)
+ .apply(RepairTable(ResolvedTable.create(catalog, identifier, unmanaged), true, false))
+ assert(unmanagedPlans.isEmpty)
+ }
+
+ test("MSCK repair reuses the sync engine: DROP unregisters catalog-only partitions") {
+ val gateway = new AtomicGateway(Set(Map("dt" -> "20260715", "hh" -> "10")))
+ val table = new PaimonFormatTable(rawFormatTable(managed = true), gateway)
+ var refreshCalls = 0
+
+ // The table location has no partition directories, so the registered partition is
+ // catalog-only; MSCK DROP PARTITIONS must unregister it (metadata-only).
+ runCommand(
+ PaimonRepairFormatTablePartitionsExec(
+ table,
+ addPartitions = false,
+ dropPartitions = true,
+ () => refreshCalls += 1))
+
+ assert(gateway.state.isEmpty)
+ assert(refreshCalls == 1)
+ }
+
+ test("strategy routes only managed Format Table SHOW PARTITIONS through the bounded executor") {
+ val output = Seq(AttributeReference("partition", StringType, nullable = false)())
+ val (managed, _) = formatTable(managed = true)
+ val (unmanaged, _) = formatTable(managed = false)
+ val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog]
+ val identifier = SparkIdentifier.of(Array("test"), "format_table")
+ val spec = Some(partition(20260715, 10))
+
+ val managedPlans = PaimonStrategy(spark).apply(
+ ShowPartitions(ResolvedTable.create(catalog, identifier, managed), spec, output))
+ val unmanagedPlans = PaimonStrategy(spark).apply(
+ ShowPartitions(ResolvedTable.create(catalog, identifier, unmanaged), spec, output))
+
+ assert(managedPlans.size == 1)
+ assert(
+ managedPlans.head.asInstanceOf[PaimonShowFormatTablePartitionsExec].maxResults ==
+ PaimonShowFormatTablePartitionsExec.DEFAULT_MAX_RESULTS)
+ assert(unmanagedPlans.isEmpty)
+ }
+
+ private def partition(dt: Int, hh: Int): ResolvedPartitionSpec =
+ ResolvedPartitionSpec(Seq("dt", "hh"), new GenericInternalRow(Array[Any](dt, hh)))
+
+ private def formatTable(managed: Boolean): (PaimonFormatTable, RecordingGateway) = {
+ val gateway = new RecordingGateway
+ (new PaimonFormatTable(rawFormatTable(managed), gateway), gateway)
+ }
+
+ private def rawFormatTable(managed: Boolean): FormatTable =
+ rawFormatTable(
+ managed,
+ new Path(Files.createTempDirectory("format-table-ddl-planning-test").toUri).toString)
+
+ private def rawFormatTable(managed: Boolean, location: String): FormatTable = {
+ rawFormatTable(managed, location, LocalFileIO.create)
+ }
+
+ private def rawFormatTable(managed: Boolean, location: String, fileIO: FileIO): FormatTable = {
+ val options = if (managed) {
+ Map("metastore.partitioned-table" -> "true")
+ } else {
+ Map.empty[String, String]
+ }
+ FormatTable
+ .builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("test", "format_table"))
+ .rowType(
+ DataTypes.ROW(
+ DataTypes.FIELD(0, "id", DataTypes.INT()),
+ DataTypes.FIELD(1, "dt", DataTypes.INT()),
+ DataTypes.FIELD(2, "hh", DataTypes.INT())))
+ .partitionKeys(Seq("dt", "hh").asJava)
+ .location(location)
+ .format(FormatTable.Format.CSV)
+ .options(options.asJava)
+ .catalogContext(CatalogContext.create(new Options))
+ .build()
+ }
+
+ private def registerPartitions(tableName: String, specs: Map[String, String]*): Unit =
+ paimonCatalog.createPartitions(
+ Identifier.create(dbName0, tableName),
+ specs.map(_.asJava).asJava,
+ true)
+
+ private def shownPartitions(tableName: String): Set[String] =
+ sql(s"SHOW PARTITIONS $tableName").collect().map(_.getString(0)).toSet
+
+ private def runCommand(command: AnyRef): Unit = {
+ try {
+ command.getClass.getMethod("run").invoke(command)
+ } catch {
+ case e: InvocationTargetException => throw e.getCause
+ }
+ }
+
+ private def causeMessages(error: Throwable): String = {
+ Iterator
+ .iterate(error)(_.getCause)
+ .takeWhile(_ != null)
+ .flatMap(e => Option(e.getMessage))
+ .mkString(" | ")
+ }
+
+ private class RecordingGateway extends FormatTablePartitionCatalog {
+ var createCalls = 0
+ var lookupCalls = 0
+ var created = Seq.empty[JMap[String, String]]
+ var ignoreIfExists = false
+ var dropCalls = 0
+ var dropped = Seq.empty[JMap[String, String]]
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ createCalls += 1
+ created = partitions.asScala.toSeq
+ this.ignoreIfExists = ignoreIfExists
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropCalls += 1
+ dropped = partitions.asScala.toSeq
+ }
+
+ // Reports every requested spec as registered. ADD tests assert lookupCalls == 0 because
+ // managed ADD must never perform a client-side existence lookup.
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = {
+ lookupCalls += 1
+ partitions
+ }
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Collections.emptyList(), null)
+ }
+
+ private class AtomicGateway(initial: Set[Map[String, String]])
+ extends FormatTablePartitionCatalog {
+
+ private var partitions = initial
+ var batches = Seq.empty[Seq[Map[String, String]]]
+
+ def state: Set[Map[String, String]] = synchronized(partitions)
+
+ override def createPartitions(
+ partitionsToCreate: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = synchronized {
+ val batch = partitionsToCreate.asScala.map(_.asScala.toMap).toSeq
+ batches :+= batch
+ val duplicates = batch.filter(partitions.contains)
+ if (duplicates.nonEmpty && !ignoreIfExists) {
+ throw new IllegalStateException(s"Partition already exists: ${duplicates.head}")
+ }
+ partitions ++= batch
+ }
+
+ override def dropPartitions(partitionsToDrop: JList[JMap[String, String]]): Unit =
+ synchronized {
+ partitions --= partitionsToDrop.asScala.map(_.asScala.toMap)
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ throw new AssertionError("ADD must not perform a client-side existence lookup")
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = synchronized {
+ val specs = new java.util.ArrayList[JMap[String, String]]()
+ partitions.foreach {
+ spec =>
+ val ordered = new java.util.LinkedHashMap[String, String]()
+ spec.foreach { case (key, value) => ordered.put(key, value) }
+ specs.add(ordered)
+ }
+ FormatTablePartitionPage(specs, null)
+ }
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalogTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalogTest.scala
new file mode 100644
index 000000000000..07887266083b
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogFormatTablePartitionCatalogTest.scala
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format
+
+import org.apache.paimon.PagedList
+import org.apache.paimon.catalog.{Catalog, Identifier}
+import org.apache.paimon.partition.Partition
+
+import org.apache.spark.SparkFunSuite
+
+import java.lang.reflect.{InvocationHandler, Method, Proxy}
+import java.util.{Arrays, LinkedHashMap, List => JList, Map => JMap}
+
+import scala.collection.JavaConverters._
+
+class CatalogFormatTablePartitionCatalogTest extends SparkFunSuite {
+
+ private val identifier = Identifier.create("db", "format_table")
+
+ test("ADD forwards the complete batch and ignoreIfExists to the catalog") {
+ var forwardedIdentifier: Identifier = null
+ var forwardedPartitions = Seq.empty[JMap[String, String]]
+ var forwardedFlags = Seq.empty[Boolean]
+ val catalog = proxyCatalog {
+ (_, method, args) =>
+ method.getName match {
+ case "createPartitions" if args.length == 3 =>
+ forwardedIdentifier = args(0).asInstanceOf[Identifier]
+ forwardedPartitions = args(1).asInstanceOf[JList[JMap[String, String]]].asScala.toSeq
+ forwardedFlags :+= args(2).asInstanceOf[java.lang.Boolean].booleanValue()
+ null
+ case _ => null
+ }
+ }
+ val partitions = Arrays.asList(spec("dt", "20260715"), spec("dt", "20260716"))
+
+ // Strict-ADD atomicity is the catalog service's job (the managed ADD command skips Spark's
+ // client-side existence precheck), so the flag must reach the catalog unchanged.
+ adapter(catalog).createPartitions(partitions, ignoreIfExists = true)
+ adapter(catalog).createPartitions(partitions, ignoreIfExists = false)
+
+ assert(forwardedFlags == Seq(true, false))
+ assert(forwardedIdentifier == identifier)
+ assert(forwardedPartitions == partitions.asScala)
+ }
+
+ test("DROP unregisters oversized expansions in bounded batches") {
+ var batchSizes = Seq.empty[Int]
+ var dropped = Seq.empty[JMap[String, String]]
+ val catalog = proxyCatalog {
+ (_, method, args) =>
+ method.getName match {
+ case "dropPartitions" =>
+ val batch = args(1).asInstanceOf[JList[JMap[String, String]]]
+ batchSizes :+= batch.size()
+ dropped ++= batch.asScala
+ null
+ case _ => null
+ }
+ }
+ val partitions = new java.util.ArrayList[JMap[String, String]]()
+ (0 until 2500).foreach(i => partitions.add(spec("dt", f"d$i%04d")))
+
+ adapter(catalog).dropPartitions(partitions)
+
+ assert(batchSizes == Seq(1000, 1000, 500))
+ assert(dropped == partitions.asScala)
+ }
+
+ test("map catalog list-by-names and paged prefix results to partition specs") {
+ var namesRequest = Seq.empty[JMap[String, String]]
+ var pageToken: String = null
+ var pageSize = 0
+ var namePattern: String = null
+ val catalog = proxyCatalog {
+ (_, method, args) =>
+ method.getName match {
+ case "listPartitionsByNames" =>
+ namesRequest = args(1).asInstanceOf[JList[JMap[String, String]]].asScala.toSeq
+ Arrays.asList(partition(spec("dt", "20260715")))
+ case "listPartitionsPaged" =>
+ pageSize = args(1).asInstanceOf[Integer]
+ pageToken = args(2).asInstanceOf[String]
+ namePattern = args(3).asInstanceOf[String]
+ new PagedList[Partition](
+ Arrays.asList(partition(spec("dt", "20260715")), partition(spec("dt", "20260716"))),
+ "next")
+ case _ => null
+ }
+ }
+ val gateway = adapter(catalog)
+ val requested = Arrays.asList(spec("dt", "20260715"), spec("dt", "missing"))
+
+ assert(
+ gateway.listPartitionsByNames(requested).asScala.map(_.asScala.toMap) ==
+ Seq(Map("dt" -> "20260715")))
+ assert(namesRequest == requested.asScala)
+
+ val page = gateway.listPartitions(spec("dt", "20260715"), "from", 17)
+ assert(page.partitions.asScala.map(_.asScala.toMap) == Seq(Map("dt" -> "20260715")))
+ assert(page.nextPageToken == "next")
+ assert(pageToken == "from")
+ assert(pageSize == 17)
+ assert(namePattern == null)
+ }
+
+ test("partial partition specs are enforced client-side, not as server patterns") {
+ var serverPattern: String = "not-called"
+ val catalog = proxyCatalog {
+ (_, method, args) =>
+ method.getName match {
+ case "listPartitionsPaged" =>
+ serverPattern = args(3).asInstanceOf[String]
+ new PagedList[Partition](
+ Arrays.asList(
+ partition(multiSpec("dt" -> "20260715", "hh" -> "10")),
+ partition(multiSpec("dt" -> "20260715", "hh" -> "11"))),
+ null)
+ case _ => null
+ }
+ }
+
+ val page = adapter(catalog).listPartitions(spec("hh", "10"), null, 17)
+
+ assert(serverPattern == null)
+ assert(
+ page.partitions.asScala.map(_.asScala.toMap) ==
+ Seq(Map("dt" -> "20260715", "hh" -> "10")))
+ }
+
+ test("managed reads propagate catalog listing failures") {
+ val catalog = proxyCatalog {
+ (_, method, _) =>
+ method.getName match {
+ case "listPartitionsByNames" | "listPartitionsPaged" =>
+ throw new UnsupportedOperationException("managed listing unavailable")
+ case _ => null
+ }
+ }
+ val gateway = adapter(catalog)
+
+ val pointError = intercept[UnsupportedOperationException] {
+ gateway.listPartitionsByNames(Arrays.asList(spec("dt", "20260715")))
+ }
+ val pageError = intercept[UnsupportedOperationException] {
+ gateway.listPartitions(new LinkedHashMap[String, String](), null, 10)
+ }
+
+ assert(pointError.getMessage.contains("managed listing unavailable"))
+ assert(pageError.getMessage.contains("managed listing unavailable"))
+ }
+
+ private def adapter(catalog: Catalog): FormatTablePartitionCatalog =
+ CatalogFormatTablePartitionCatalog(catalog, identifier)
+
+ private def proxyCatalog(handler: (Any, Method, Array[AnyRef]) => AnyRef): Catalog = {
+ Proxy
+ .newProxyInstance(
+ getClass.getClassLoader,
+ Array(classOf[Catalog]),
+ new InvocationHandler {
+ override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = {
+ handler(proxy, method, Option(args).getOrElse(Array.empty[AnyRef]))
+ }
+ }
+ )
+ .asInstanceOf[Catalog]
+ }
+
+ private def spec(key: String, value: String): JMap[String, String] = {
+ val result = new LinkedHashMap[String, String]()
+ result.put(key, value)
+ result
+ }
+
+ private def multiSpec(entries: (String, String)*): JMap[String, String] = {
+ val result = new LinkedHashMap[String, String]()
+ entries.foreach { case (key, value) => result.put(key, value) }
+ result
+ }
+
+ private def partition(spec: JMap[String, String]): Partition =
+ new Partition(spec, 0L, 0L, 0L, 0L, 0, false)
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTableBatchWriteTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTableBatchWriteTest.scala
new file mode 100644
index 000000000000..ce4d1951e1b4
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTableBatchWriteTest.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format
+
+import org.apache.paimon.spark.write.FormatTableWriteTaskResult
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.table.sink.{BatchTableCommit, BatchWriteBuilder}
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.connector.write.WriterCommitMessage
+import org.apache.spark.sql.types.StructType
+
+import java.lang.reflect.{InvocationHandler, Method, Proxy}
+import java.util.concurrent.atomic.AtomicInteger
+
+class FormatTableBatchWriteTest extends SparkFunSuite {
+
+ test("abort after commit starts does not clean up potentially committed files") {
+ val commitCalls = new AtomicInteger
+ val abortCalls = new AtomicInteger
+ val batchWrite =
+ new FormatTableBatchWrite(formatTable(commitCalls, abortCalls), None, None, StructType(Nil))
+ val messages: Array[WriterCommitMessage] = Array(FormatTableWriteTaskResult(Seq.empty))
+
+ val error = intercept[RuntimeException] {
+ batchWrite.commit(messages)
+ }
+ assert(error.getMessage == "partition registration response lost")
+
+ // Spark invokes abort after commit throws. The commit outcome may be ambiguous, so cleanup
+ // could delete files which were already committed and registered remotely.
+ batchWrite.abort(messages)
+
+ assert(commitCalls.get == 1)
+ assert(abortCalls.get == 0)
+ }
+
+ test("abort before commit delegates cleanup") {
+ val commitCalls = new AtomicInteger
+ val abortCalls = new AtomicInteger
+ val batchWrite =
+ new FormatTableBatchWrite(formatTable(commitCalls, abortCalls), None, None, StructType(Nil))
+
+ batchWrite.abort(Array(FormatTableWriteTaskResult(Seq.empty)))
+
+ assert(commitCalls.get == 0)
+ assert(abortCalls.get == 1)
+ }
+
+ private def formatTable(commitCalls: AtomicInteger, abortCalls: AtomicInteger): FormatTable = {
+ val tableCommit = proxy(classOf[BatchTableCommit]) {
+ case "commit" =>
+ commitCalls.incrementAndGet()
+ throw new RuntimeException("partition registration response lost")
+ case "abort" =>
+ abortCalls.incrementAndGet()
+ null
+ }
+ val writeBuilder = proxy(classOf[BatchWriteBuilder]) { case "newCommit" => tableCommit }
+ proxy(classOf[FormatTable]) {
+ case "newBatchWriteBuilder" => writeBuilder
+ case "name" => "test_db.format_table"
+ }
+ }
+
+ private def proxy[T](clazz: Class[T])(responses: PartialFunction[String, AnyRef]): T = {
+ Proxy
+ .newProxyInstance(
+ clazz.getClassLoader,
+ Array(clazz),
+ new InvocationHandler {
+ override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = {
+ responses.applyOrElse(
+ method.getName,
+ (name: String) =>
+ throw new UnsupportedOperationException(s"Unexpected $name invocation"))
+ }
+ }
+ )
+ .asInstanceOf[T]
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala
new file mode 100644
index 000000000000..f62734a55ca0
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala
@@ -0,0 +1,738 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format
+
+import org.apache.paimon.catalog.{CatalogContext, Identifier}
+import org.apache.paimon.fs.{FileIO, Path}
+import org.apache.paimon.fs.local.LocalFileIO
+import org.apache.paimon.options.Options
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.types.DataTypes
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.unsafe.types.UTF8String
+
+import java.lang.reflect.{InvocationHandler, InvocationTargetException, Method, Proxy}
+import java.nio.file.Files
+import java.util.{ArrayList, Collections, List => JList, Map => JMap}
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+class FormatTablePartitionManagementTest extends SparkFunSuite {
+
+ test("managed ADD forwards the complete batch and ignoreIfExists to the partition catalog") {
+ var forwardedPartitions = Seq.empty[JMap[String, String]]
+ var forwardedIgnoreIfExists = false
+
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ forwardedPartitions = partitions.asScala.toSeq
+ forwardedIgnoreIfExists = ignoreIfExists
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {}
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Collections.emptyList(), null)
+ }
+ val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway)
+
+ val rows = Array[InternalRow](
+ new GenericInternalRow(Array[Any](20260715, 10)),
+ new GenericInternalRow(Array[Any](20260716, 11)))
+ val properties = Array.fill[JMap[String, String]](2)(Collections.emptyMap())
+ sparkTable.createFormatTablePartitions(rows, properties, ignoreIfExists = true)
+
+ assert(forwardedIgnoreIfExists)
+ assert(
+ forwardedPartitions.map(_.asScala.toMap) == Seq(
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260716", "hh" -> "11")))
+ }
+
+ test("managed ADD creates the partition directory") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("managed-format-add-mkdirs").toUri)
+ val table = managedFormatTable(fileIO, tablePath.toString)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+ try {
+ val sparkTable = new PaimonFormatTable(table, emptyGateway)
+ sparkTable.createFormatTablePartitions(
+ Array(partitionRow(20260715, 10)),
+ Array[JMap[String, String]](Collections.emptyMap()),
+ ignoreIfExists = true)
+
+ // ADD PARTITION creates the directory client-side, so a scan sees an empty partition
+ // rather than failing on a missing directory (Hive ADD PARTITION semantics).
+ assert(fileIO.exists(partitionDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed DROP rejects a partition value that would escape the table location") {
+ var dropCalls = 0
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Collections.emptyList(), null)
+ }
+
+ val sparkTable =
+ new PaimonFormatTable(managedStringFormatTable(onlyValueInPath = true), gateway)
+ val error = intercept[IllegalArgumentException] {
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt")),
+ Array(new GenericInternalRow(Array[Any](UTF8String.fromString("..")))))
+ }
+
+ // Traversal values are rejected before any catalog mutation, so no partition is unregistered.
+ assert(error.getMessage.contains(".."))
+ assert(dropCalls == 0)
+ }
+
+ test("managed ADD with LOCATION is rejected before any catalog RPC") {
+ var createCalls = 0
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = createCalls += 1
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {}
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Collections.emptyList(), null)
+ }
+
+ val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway)
+ val error = intercept[UnsupportedOperationException] {
+ sparkTable.createFormatTablePartitions(
+ Array(new GenericInternalRow(Array[Any](20260715, 10))),
+ Array(Map("location" -> "file:/tmp/custom").asJava),
+ ignoreIfExists = true)
+ }
+
+ assert(error.getMessage.contains("LOCATION"))
+ assert(createCalls == 0)
+ }
+
+ test("managed DROP PARTITION unregisters first and then deletes the partition directory") {
+ var dropped = Seq.empty[JMap[String, String]]
+ var directoryExistedAtUnregister = false
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("managed-format-drop-ordering").toUri)
+ val table = managedFormatTable(fileIO, tablePath.toString)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ // Ordering contract: the directory must still exist when the catalog unregisters.
+ directoryExistedAtUnregister = fileIO.exists(partitionDir)
+ dropped = partitions.asScala.toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Collections.emptyList(), null)
+ }
+
+ try {
+ fileIO.mkdirs(partitionDir)
+ assert(fileIO.exists(partitionDir))
+
+ val sparkTable = new PaimonFormatTable(table, gateway)
+ assert(
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt", "hh")),
+ Array(new GenericInternalRow(Array[Any](20260715, 10)))))
+
+ assert(dropped.map(_.asScala.toMap) == Seq(Map("dt" -> "20260715", "hh" -> "10")))
+ assert(directoryExistedAtUnregister)
+ assert(!fileIO.exists(partitionDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed direct partial DROP expands leading values to exact leaf partitions") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("managed-format-direct-partial-drop").toUri)
+ val dtDir = new Path(tablePath, "dt=20260715")
+ val firstDir = new Path(dtDir, "hh=10")
+ val secondDir = new Path(dtDir, "hh=11")
+ val first = partitionSpec(20260715, 10)
+ val second = partitionSpec(20260715, 11)
+ var listedPrefixes = Seq.empty[Map[String, String]]
+ var dropped = Seq.empty[Map[String, String]]
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ listedPrefixes :+= prefix.asScala.toMap
+ FormatTablePartitionPage(Seq(first.asJava, second.asJava).asJava, null)
+ }
+ }
+
+ try {
+ fileIO.mkdirs(firstDir)
+ fileIO.mkdirs(secondDir)
+ val sparkTable =
+ new PaimonFormatTable(managedFormatTable(fileIO, tablePath.toString), gateway)
+
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt")),
+ Array(new GenericInternalRow(Array[Any](20260715))))
+
+ // The expansion always runs one unfiltered traversal; constraints apply client-side.
+ assert(listedPrefixes == Seq(Map.empty))
+ assert(dropped == Seq(first, second))
+ assert(!fileIO.exists(firstDir))
+ assert(!fileIO.exists(secondDir))
+ assert(fileIO.exists(dtDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed DROP fails when FileIO reports an undeleted partition directory") {
+ val delegate = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("managed-format-drop").toUri)
+ val fileIO = falseDeleteFileIO(delegate)
+ val table = managedFormatTable(fileIO, tablePath.toString)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+ delegate.mkdirs(partitionDir)
+ val gateway = new RecordingDropCatalog
+
+ try {
+ val error = intercept[java.io.IOException] {
+ new PaimonFormatTable(table, gateway)
+ .dropFormatTablePartitions(
+ Array(Array("dt", "hh")),
+ Array(new GenericInternalRow(Array[Any](20260715, 10))))
+ }
+
+ assert(error.getMessage.contains("was not deleted"))
+ assert(error.getMessage.contains("dt=20260715"))
+ assert(error.getMessage.contains("hh=10"))
+ assert(gateway.dropCalls == 1)
+ assert(delegate.exists(partitionDir))
+ } finally {
+ delegate.delete(tablePath, true)
+ }
+ }
+
+ test("managed DROP treats an already missing partition directory as deleted") {
+ val delegate = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("managed-format-drop-missing").toUri)
+ val fileIO = falseDeleteFileIO(delegate)
+ val table = managedFormatTable(fileIO, tablePath.toString)
+ val gateway = new RecordingDropCatalog
+
+ try {
+ assert(
+ new PaimonFormatTable(table, gateway)
+ .dropFormatTablePartitions(
+ Array(Array("dt", "hh")),
+ Array(new GenericInternalRow(Array[Any](20260715, 10)))))
+ assert(gateway.dropCalls == 1)
+ } finally {
+ delegate.delete(tablePath, true)
+ }
+ }
+
+ test("managed DROP deduplicates overlapping full and partial specs before deleting") {
+ val delegate = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("managed-format-drop-overlap").toUri)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+ var deleteCalls = 0
+ val fileIO = Proxy
+ .newProxyInstance(
+ classOf[FileIO].getClassLoader,
+ Array(classOf[FileIO]),
+ new InvocationHandler {
+ override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = {
+ if (method.getName == "delete" && args(0) == partitionDir) {
+ deleteCalls += 1
+ }
+ try {
+ method.invoke(delegate, Option(args).getOrElse(Array.empty[AnyRef]): _*)
+ } catch {
+ case error: InvocationTargetException => throw error.getCause
+ }
+ }
+ }
+ )
+ .asInstanceOf[FileIO]
+ val table = managedFormatTable(fileIO, tablePath.toString)
+ val matching = partitionSpec(20260715, 10)
+ var dropped = Seq.empty[Map[String, String]]
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Seq(matching.asJava).asJava, null)
+ }
+
+ try {
+ delegate.mkdirs(partitionDir)
+ val sparkTable = new PaimonFormatTable(table, gateway)
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt", "hh"), Array("hh")),
+ Array(partitionRow(20260715, 10), new GenericInternalRow(Array[Any](10))))
+
+ assert(dropped == Seq(matching))
+ assert(deleteCalls == 1)
+ assert(!delegate.exists(partitionDir))
+ assert(delegate.exists(new Path(tablePath, "dt=20260715")))
+ } finally {
+ delegate.delete(tablePath, true)
+ }
+ }
+
+ test("managed partial DROP leaves catalog and data untouched when a later page fails") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("managed-format-drop-page-failure").toUri)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+ val matching = partitionSpec(20260715, 10)
+ var dropCalls = 0
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ Option(pageToken) match {
+ case None => FormatTablePartitionPage(Seq(matching.asJava).asJava, "page-2")
+ case Some("page-2") => throw new TestPartitionPageException
+ case other => throw new AssertionError(s"Unexpected page token $other")
+ }
+ }
+ }
+
+ try {
+ fileIO.mkdirs(partitionDir)
+ val sparkTable =
+ new PaimonFormatTable(managedFormatTable(fileIO, tablePath.toString), gateway)
+
+ intercept[TestPartitionPageException] {
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("hh")),
+ Array(new GenericInternalRow(Array[Any](10))))
+ }
+
+ assert(dropCalls == 0)
+ assert(fileIO.exists(partitionDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed partial DROP keeps data untouched after an ambiguous catalog response") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("managed-format-drop-response-loss").toUri)
+ val firstDir = new Path(tablePath, "dt=20260715/hh=10")
+ val secondDir = new Path(tablePath, "dt=20260715/hh=11")
+ val first = partitionSpec(20260715, 10)
+ val second = partitionSpec(20260715, 11)
+ val registered = mutable.LinkedHashSet(first, second)
+ val responseFailure = new TestCatalogDropResponseException
+ var dropCalls = 0
+ var failDropResponse = true
+ val gateway = new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit =
+ throw new AssertionError("An ambiguous DROP must not recreate catalog partitions")
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropCalls += 1
+ registered --= partitions.asScala.map(_.asScala.toMap)
+ if (failDropResponse) {
+ failDropResponse = false
+ throw responseFailure
+ }
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(registered.toSeq.map(_.asJava).asJava, null)
+ }
+
+ try {
+ fileIO.mkdirs(firstDir)
+ fileIO.mkdirs(secondDir)
+ val sparkTable =
+ new PaimonFormatTable(managedFormatTable(fileIO, tablePath.toString), gateway)
+
+ val error = intercept[TestCatalogDropResponseException] {
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt")),
+ Array(new GenericInternalRow(Array[Any](20260715))))
+ }
+
+ assert(error eq responseFailure)
+ assert(dropCalls == 1)
+ assert(registered.isEmpty)
+ assert(fileIO.exists(firstDir))
+ assert(fileIO.exists(secondDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("managed IF NOT EXISTS delegates all and partial duplicate batches to the catalog") {
+ val first = partitionSpec(20260715, 10)
+ val second = partitionSpec(20260716, 11)
+ val third = partitionSpec(20260717, 12)
+ val gateway = new InMemoryPartitionCatalog(Seq(first, second))
+ val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway)
+
+ sparkTable.createFormatTablePartitions(
+ Array(partitionRow(20260715, 10), partitionRow(20260716, 11)),
+ Array.fill[JMap[String, String]](2)(Collections.emptyMap()),
+ ignoreIfExists = true)
+ sparkTable.createFormatTablePartitions(
+ Array(partitionRow(20260716, 11), partitionRow(20260717, 12)),
+ Array.fill[JMap[String, String]](2)(Collections.emptyMap()),
+ ignoreIfExists = true)
+
+ assert(gateway.createRequests == Seq((Seq(first, second), true), (Seq(second, third), true)))
+ assert(gateway.partitions == Seq(first, second, third))
+ assert(gateway.lookupCalls == 0)
+ }
+
+ test("managed ordinary ADD propagates the catalog duplicate error") {
+ val existing = partitionSpec(20260715, 10)
+ val gateway = new InMemoryPartitionCatalog(Seq(existing))
+ val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway)
+
+ val error = intercept[TestPartitionAlreadyExistsException] {
+ sparkTable.createFormatTablePartitions(
+ Array(partitionRow(20260715, 10)),
+ Array[JMap[String, String]](Collections.emptyMap()),
+ ignoreIfExists = false)
+ }
+
+ assert(error.partition == existing)
+ assert(gateway.lookupCalls == 0)
+ }
+
+ test("concurrent managed IF NOT EXISTS ADD relies on catalog atomicity") {
+ val first = partitionSpec(20260715, 10)
+ val second = partitionSpec(20260716, 11)
+ val third = partitionSpec(20260717, 12)
+ val gateway = new InMemoryPartitionCatalog
+ val sparkTable = new PaimonFormatTable(managedFormatTable(), gateway)
+ val start = new CountDownLatch(1)
+ val done = new CountDownLatch(2)
+ val failures = Collections.synchronizedList(new ArrayList[Throwable]())
+
+ def add(rows: Array[InternalRow]): Thread =
+ new Thread(new Runnable {
+ override def run(): Unit = {
+ try {
+ start.await()
+ sparkTable.createFormatTablePartitions(
+ rows,
+ Array.fill[JMap[String, String]](rows.length)(Collections.emptyMap()),
+ ignoreIfExists = true)
+ } catch {
+ case error: Throwable => failures.add(error)
+ } finally {
+ done.countDown()
+ }
+ }
+ })
+
+ val left = add(Array(partitionRow(20260715, 10), partitionRow(20260716, 11)))
+ val right = add(Array(partitionRow(20260716, 11), partitionRow(20260717, 12)))
+ left.start()
+ right.start()
+ start.countDown()
+
+ assert(done.await(10, TimeUnit.SECONDS))
+ assert(failures.isEmpty)
+ assert(gateway.partitions.toSet == Set(first, second, third))
+ assert(gateway.createRequests.size == 2)
+ assert(gateway.createRequests.forall(_._2))
+ assert(gateway.lookupCalls == 0)
+ }
+
+ test("managed ADD preserves typed special partition values") {
+ val gateway = new InMemoryPartitionCatalog
+ val sparkTable = new PaimonFormatTable(managedStringFormatTable(), gateway)
+ val values = Seq("a/b", "a=b", "a%", "中文 value")
+ val rows: Seq[InternalRow] = values
+ .map(
+ value => new GenericInternalRow(Array[Any](UTF8String.fromString(value))): InternalRow) :+
+ new GenericInternalRow(Array[Any](null))
+
+ sparkTable.createFormatTablePartitions(
+ rows.toArray,
+ Array.fill[JMap[String, String]](rows.size)(Collections.emptyMap()),
+ ignoreIfExists = true)
+
+ assert(
+ gateway.createRequests == Seq(
+ (values.map(value => Map("dt" -> value)) :+ Map("dt" -> "__DEFAULT_PARTITION__"), true)))
+ }
+
+ private def emptyGateway: FormatTablePartitionCatalog =
+ new FormatTablePartitionCatalog {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {}
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Collections.emptyList(), null)
+ }
+
+ private class InMemoryPartitionCatalog(initialPartitions: Seq[Map[String, String]] = Seq.empty)
+ extends FormatTablePartitionCatalog {
+
+ private val storedPartitions = mutable.LinkedHashSet(initialPartitions: _*)
+ private var requests = Seq.empty[(Seq[Map[String, String]], Boolean)]
+ var lookupCalls = 0
+
+ def createRequests: Seq[(Seq[Map[String, String]], Boolean)] = synchronized(requests)
+
+ def partitions: Seq[Map[String, String]] = synchronized(storedPartitions.toSeq)
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = synchronized {
+ val requested = partitions.asScala.map(_.asScala.toMap).toSeq
+ requests :+= ((requested, ignoreIfExists))
+ if (!ignoreIfExists) {
+ requested.find(storedPartitions.contains).foreach {
+ duplicate => throw new TestPartitionAlreadyExistsException(duplicate)
+ }
+ }
+ storedPartitions ++= requested
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = synchronized {
+ storedPartitions --= partitions.asScala.map(_.asScala.toMap)
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] = synchronized {
+ lookupCalls += 1
+ partitions.asScala
+ .map(_.asScala.toMap)
+ .filter(storedPartitions.contains)
+ .map(_.asJava)
+ .asJava
+ }
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ throw new AssertionError("ADD must not enumerate catalog or filesystem partitions")
+ }
+
+ private class TestPartitionAlreadyExistsException(val partition: Map[String, String])
+ extends RuntimeException
+
+ private class TestPartitionPageException extends RuntimeException
+
+ private class TestCatalogDropResponseException extends RuntimeException
+
+ private class RecordingDropCatalog extends FormatTablePartitionCatalog {
+ var dropCalls = 0
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ Collections.emptyList()
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage =
+ FormatTablePartitionPage(Collections.emptyList(), null)
+ }
+
+ private def partitionRow(dt: Int, hh: Int): InternalRow =
+ new GenericInternalRow(Array[Any](dt, hh))
+
+ private def partitionSpec(dt: Int, hh: Int): Map[String, String] =
+ Map("dt" -> dt.toString, "hh" -> hh.toString)
+
+ private def uniqueTableLocation(prefix: String): String =
+ new Path(Files.createTempDirectory(prefix).toUri).toString
+
+ private def managedFormatTable(
+ fileIO: FileIO = LocalFileIO.create,
+ location: String = uniqueTableLocation("format_table")): FormatTable = {
+ FormatTable
+ .builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("test_db", "format_table"))
+ .rowType(
+ DataTypes.ROW(
+ DataTypes.FIELD(0, "id", DataTypes.INT()),
+ DataTypes.FIELD(1, "dt", DataTypes.INT()),
+ DataTypes.FIELD(2, "hh", DataTypes.INT())))
+ .partitionKeys(Seq("dt", "hh").asJava)
+ .location(location)
+ .format(FormatTable.Format.CSV)
+ .options(Map("metastore.partitioned-table" -> "true").asJava)
+ .catalogContext(CatalogContext.create(new Options))
+ .build()
+ }
+
+ private def falseDeleteFileIO(delegate: FileIO): FileIO = {
+ Proxy
+ .newProxyInstance(
+ classOf[FileIO].getClassLoader,
+ Array(classOf[FileIO]),
+ new InvocationHandler {
+ override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = {
+ if (method.getName == "delete" && method.getParameterCount == 2) {
+ java.lang.Boolean.FALSE
+ } else {
+ try {
+ method.invoke(delegate, Option(args).getOrElse(Array.empty[AnyRef]): _*)
+ } catch {
+ case error: InvocationTargetException => throw error.getCause
+ }
+ }
+ }
+ }
+ )
+ .asInstanceOf[FileIO]
+ }
+
+ private def managedStringFormatTable(onlyValueInPath: Boolean = false): FormatTable = {
+ FormatTable
+ .builder()
+ .fileIO(LocalFileIO.create)
+ .identifier(Identifier.create("test_db", "format_table_special"))
+ .rowType(DataTypes.ROW(
+ DataTypes.FIELD(0, "id", DataTypes.INT()),
+ DataTypes.FIELD(1, "dt", DataTypes.STRING())))
+ .partitionKeys(Seq("dt").asJava)
+ .location(uniqueTableLocation("format_table_special"))
+ .format(FormatTable.Format.CSV)
+ .options(Map(
+ "metastore.partitioned-table" -> "true",
+ "format-table.partition-path-only-value" -> onlyValueInPath.toString).asJava)
+ .catalogContext(CatalogContext.create(new Options))
+ .build()
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/ManagedFormatTableCatalogTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/ManagedFormatTableCatalogTest.scala
new file mode 100644
index 000000000000..d699f9fa61b0
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/ManagedFormatTableCatalogTest.scala
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format
+
+import org.apache.paimon.spark.{PaimonSparkTestWithRestCatalogBase, SparkCatalog}
+
+import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier}
+
+class ManagedFormatTableCatalogTest extends PaimonSparkTestWithRestCatalogBase {
+
+ test("SparkCatalog injects a branch-aware partition catalog into a managed Format Table") {
+ val tableName = "managed_format_table_catalog_gateway"
+ withTable(tableName) {
+ sql(
+ s"CREATE TABLE $tableName (id INT, dt STRING) USING CSV " +
+ "TBLPROPERTIES ('format-table.implementation'='paimon', " +
+ "'metastore.partitioned-table'='true') PARTITIONED BY (dt)")
+
+ val sparkCatalog = spark.sessionState.catalogManager
+ .catalog("paimon")
+ .asInstanceOf[SparkCatalog]
+ val sparkTable = sparkCatalog
+ .loadTable(SparkIdentifier.of(Array(dbName0), tableName))
+ .asInstanceOf[PaimonFormatTable]
+
+ assert(sparkTable.partitionCatalog != null)
+ }
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ManagedFormatTableMsckRepairTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ManagedFormatTableMsckRepairTest.scala
new file mode 100644
index 000000000000..ddcefa8820a3
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/ManagedFormatTableMsckRepairTest.scala
@@ -0,0 +1,602 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.sql
+
+import org.apache.paimon.catalog.Identifier
+import org.apache.paimon.fs.Path
+import org.apache.paimon.spark.{PaimonSparkTestWithRestCatalogBase, SparkCatalog}
+import org.apache.paimon.spark.execution.PaimonRepairFormatTablePartitionsExec
+import org.apache.paimon.spark.format.{FormatTablePartitionCatalog, FormatTablePartitionPage, PaimonFormatTable}
+import org.apache.paimon.table.FormatTable
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier, Table => SparkTable}
+import org.apache.spark.sql.execution.CommandResultExec
+
+import java.lang.reflect.InvocationTargetException
+import java.util.{List => JList, Map => JMap}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+class ManagedFormatTableMsckRepairTest extends PaimonSparkTestWithRestCatalogBase {
+
+ override protected def sparkConf: SparkConf =
+ super.sparkConf
+ .set("spark.sql.catalog.paimon", classOf[FaultInjectingFormatTableSparkCatalog].getName)
+
+ override protected def beforeEach(): Unit = {
+ MsckFaultInjection.reset()
+ super.beforeEach()
+ }
+
+ override protected def afterEach(): Unit = {
+ try {
+ super.afterEach()
+ } finally {
+ MsckFaultInjection.reset()
+ }
+ }
+
+ test("bare MSCK, explicit ADD, and REPAIR alias register filesystem-only partitions") {
+ val tableName = "msck_add_forms"
+ val first = "20260715"
+ val second = "20260716"
+ val third = "20260717"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+
+ writeCsvPartition(tableName, first, 15, "bare-msck")
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName")
+ assertPartitionState(tableName, Set(first))
+
+ writeCsvPartition(tableName, second, 16, "explicit-add")
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS")
+ assertPartitionState(tableName, Set(first, second))
+
+ writeCsvPartition(tableName, third, 17, "repair-alias")
+ executeManagedRepair(s"REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS")
+ assertPartitionState(tableName, Set(first, second, third))
+
+ executeManagedRepair(s"REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS")
+
+ assertPartitionState(tableName, Set(first, second, third))
+ assert(MsckFaultInjection.createCalls == 3)
+ assertRowsAndChecksum(
+ tableName,
+ Seq(
+ Row(15, "bare-msck", first),
+ Row(16, "explicit-add", second),
+ Row(17, "repair-alias", third)),
+ 48L)
+ }
+ }
+
+ test("DROP unregisters only catalog-only partitions and preserves valid files") {
+ val tableName = "msck_drop_catalog_only"
+ val catalogOnly = "20260714"
+ val common = "20260715"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+ val commonPath = writeCsvPartition(tableName, common, 15, "common")
+ registerPartitions(tableName, catalogOnly, common)
+
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName DROP PARTITIONS")
+
+ assertPartitionState(tableName, Set(common))
+ assert(formatTable(tableName).fileIO().exists(commonPath))
+ assertRowsAndChecksum(tableName, Seq(Row(15, "common", common)), 15L)
+
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName DROP PARTITIONS")
+ assertPartitionState(tableName, Set(common))
+ assert(MsckFaultInjection.createCalls == 0)
+ assert(MsckFaultInjection.dropCalls == 1)
+ }
+ }
+
+ test("SYNC converges both directions and is idempotent") {
+ val tableName = "msck_sync_both_directions"
+ val catalogOnly = "20260714"
+ val filesystemOnly = "20260715"
+ val common = "20260716"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+ val filesystemPath =
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ val commonPath = writeCsvPartition(tableName, common, 16, "common")
+ registerPartitions(tableName, catalogOnly, common)
+
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+
+ assertPartitionState(tableName, Set(filesystemOnly, common))
+ assert(formatTable(tableName).fileIO().exists(filesystemPath))
+ assert(formatTable(tableName).fileIO().exists(commonPath))
+ assertRowsAndChecksum(
+ tableName,
+ Seq(Row(15, "filesystem-only", filesystemOnly), Row(16, "common", common)),
+ 31L)
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 1)
+
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+ assertPartitionState(tableName, Set(filesystemOnly, common))
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 1)
+ }
+ }
+
+ test("MSCK rejects unmanaged Format Tables and native Paimon tables") {
+ val unmanaged = "msck_unmanaged_format_table"
+ val native = "msck_native_paimon_table"
+ val partition = "20260715"
+
+ withTable(unmanaged, native) {
+ createUnmanagedFormatTable(unmanaged)
+ writeCsvPartition(unmanaged, partition, 15, "unmanaged")
+ checkAnswer(
+ sql(s"SELECT id, payload, dt FROM $unmanaged"),
+ Seq(Row(15, "unmanaged", partition)))
+
+ assertMsckRejected(s"MSCK REPAIR TABLE paimon.$dbName0.$unmanaged")
+ checkAnswer(
+ sql(s"SELECT id, payload, dt FROM $unmanaged"),
+ Seq(Row(15, "unmanaged", partition)))
+
+ sql(
+ s"CREATE TABLE $native (id INT, payload STRING, dt STRING) " +
+ "USING paimon PARTITIONED BY (dt)")
+ sql(s"INSERT INTO $native VALUES (16, 'native', '$partition')")
+
+ assertMsckRejected(s"MSCK REPAIR TABLE paimon.$dbName0.$native")
+ checkAnswer(sql(s"SELECT id, payload, dt FROM $native"), Seq(Row(16, "native", partition)))
+ assert(MsckFaultInjection.createCalls == 0)
+ assert(MsckFaultInjection.dropCalls == 0)
+ }
+ }
+
+ test("SYNC propagates catalog LIST failure without mutating either side") {
+ val tableName = "msck_sync_list_failure"
+ val catalogOnly = "20260714"
+ val filesystemOnly = "20260715"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+ val filesystemPath =
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ registerPartitions(tableName, catalogOnly)
+ MsckFaultInjection.failList = true
+
+ val error = intercept[Exception] {
+ sql(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS").collect()
+ }
+
+ assert(causeMessages(error).contains(MsckFaultInjection.LIST_FAILURE))
+ assert(MsckFaultInjection.listCalls == 1)
+ assert(MsckFaultInjection.createCalls == 0)
+ assert(MsckFaultInjection.dropCalls == 0)
+ assert(registeredPartitions(tableName) == Set(catalogOnly))
+ assert(formatTable(tableName).fileIO().exists(filesystemPath))
+
+ MsckFaultInjection.failList = false
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+ assertPartitionState(tableName, Set(filesystemOnly))
+ assertRowsAndChecksum(tableName, Seq(Row(15, "filesystem-only", filesystemOnly)), 15L)
+ }
+ }
+
+ test("direct repair mutates nothing but still refreshes when catalog LIST fails") {
+ val tableName = "msck_direct_list_failure"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+ val gateway = new StatefulFaultCatalog
+ gateway.failList = true
+ var refreshCalls = 0
+ val command = PaimonRepairFormatTablePartitionsExec(
+ new PaimonFormatTable(formatTable(tableName), gateway),
+ addPartitions = true,
+ dropPartitions = true,
+ () => refreshCalls += 1)
+
+ val error = intercept[IllegalStateException] {
+ runCommand(command)
+ }
+
+ assert(error.getMessage == MsckFaultInjection.LIST_FAILURE)
+ assert(gateway.createCalls == 0)
+ assert(gateway.dropCalls == 0)
+ // A failed response cannot prove the service did nothing, so every repair attempt
+ // refreshes the Spark-side caches.
+ assert(refreshCalls == 1)
+ }
+ }
+
+ test("direct repair refreshes after create applies and loses its response") {
+ val tableName = "msck_direct_create_lost_response"
+ val filesystemOnly = "20260715"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ val gateway = new StatefulFaultCatalog
+ gateway.failCreateAfterApply = true
+ var refreshCalls = 0
+ val command = PaimonRepairFormatTablePartitionsExec(
+ new PaimonFormatTable(formatTable(tableName), gateway),
+ addPartitions = true,
+ dropPartitions = true,
+ () => refreshCalls += 1)
+
+ val error = intercept[IllegalStateException] {
+ runCommand(command)
+ }
+
+ assert(error.getMessage == MsckFaultInjection.CREATE_LOST_RESPONSE)
+ assert(gateway.partitions == Set(Map("dt" -> filesystemOnly)))
+ assert(gateway.createCalls == 1)
+ assert(gateway.dropCalls == 0)
+ assert(refreshCalls == 1)
+ }
+ }
+
+ test("direct repair still attempts refresh when DROP fails after a durable ADD") {
+ val tableName = "msck_direct_drop_and_refresh_failure"
+ val catalogOnly = "20260714"
+ val filesystemOnly = "20260715"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ val gateway = new StatefulFaultCatalog(Set(Map("dt" -> catalogOnly)))
+ gateway.failDrop = true
+ var refreshCalls = 0
+ val command = PaimonRepairFormatTablePartitionsExec(
+ new PaimonFormatTable(formatTable(tableName), gateway),
+ addPartitions = true,
+ dropPartitions = true,
+ () => {
+ refreshCalls += 1
+ throw new IllegalStateException(MsckFaultInjection.REFRESH_FAILURE)
+ }
+ )
+
+ val error = intercept[IllegalStateException] {
+ runCommand(command)
+ }
+
+ // The refresh sits in a plain finally, so its failure is the one that propagates; the
+ // point is that the ADD stayed durable and the refresh was still attempted after the
+ // DROP failure.
+ assert(error.getMessage == MsckFaultInjection.REFRESH_FAILURE)
+ assert(gateway.partitions == Set(Map("dt" -> catalogOnly), Map("dt" -> filesystemOnly)))
+ assert(gateway.createCalls == 1)
+ assert(gateway.dropCalls == 1)
+ assert(refreshCalls == 1)
+ }
+ }
+
+ test("direct repair preserves a DROP throwable rethrown by refresh") {
+ val tableName = "msck_direct_same_drop_refresh_failure"
+ val catalogOnly = "20260714"
+ val filesystemOnly = "20260715"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ val gateway = new StatefulFaultCatalog(Set(Map("dt" -> catalogOnly)))
+ val dropFailure = new IllegalStateException(MsckFaultInjection.DROP_FAILURE)
+ gateway.dropFailure = dropFailure
+ var refreshCalls = 0
+ val command = PaimonRepairFormatTablePartitionsExec(
+ new PaimonFormatTable(formatTable(tableName), gateway),
+ addPartitions = true,
+ dropPartitions = true,
+ () => {
+ refreshCalls += 1
+ throw dropFailure
+ }
+ )
+
+ val error = intercept[Throwable] {
+ runCommand(command)
+ }
+
+ assert(error eq dropFailure)
+ assert(!error.isInstanceOf[MatchError])
+ assert(error.getSuppressed.isEmpty)
+ assert(gateway.partitions == Set(Map("dt" -> catalogOnly), Map("dt" -> filesystemOnly)))
+ assert(gateway.createCalls == 1)
+ assert(gateway.dropCalls == 1)
+ assert(refreshCalls == 1)
+ }
+ }
+
+ test("SYNC invalidates cached data and converges after ADD succeeds but DROP fails") {
+ val tableName = "msck_partial_sync_failure"
+ val filesystemOnly = "20260715"
+ val catalogOnly = "20260714"
+
+ withTable(tableName) {
+ createManagedFormatTable(tableName)
+ sql(s"CACHE TABLE $tableName").collect()
+ assert(spark.catalog.isCached(tableName))
+ checkAnswer(sql(s"SELECT * FROM $tableName"), Seq.empty[Row])
+
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ registerPartitions(tableName, catalogOnly)
+ MsckFaultInjection.failDrop = true
+
+ val error = intercept[Exception] {
+ sql(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS").collect()
+ }
+
+ assert(causeMessages(error).contains(MsckFaultInjection.DROP_FAILURE))
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 1)
+ assert(registeredPartitions(tableName) == Set(filesystemOnly, catalogOnly))
+ assert(spark.catalog.isCached(tableName))
+
+ // ADD is already durable even though the command failed. A stale CACHE TABLE must not hide it.
+ checkAnswer(
+ sql(s"SELECT id, payload, dt FROM $tableName WHERE dt = '$filesystemOnly'"),
+ Seq(Row(15, "filesystem-only", filesystemOnly)))
+
+ MsckFaultInjection.failDrop = false
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+
+ assertPartitionState(tableName, Set(filesystemOnly))
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 2)
+ assertRowsAndChecksum(tableName, Seq(Row(15, "filesystem-only", filesystemOnly)), 15L)
+
+ executeManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+ assertPartitionState(tableName, Set(filesystemOnly))
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 2)
+ }
+ }
+
+ private def createManagedFormatTable(tableName: String): Unit =
+ sql(s"""CREATE TABLE $tableName (id INT, payload STRING, dt STRING)
+ |USING CSV
+ |PARTITIONED BY (dt)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'metastore.partitioned-table' = 'true')
+ |""".stripMargin)
+
+ private def createUnmanagedFormatTable(tableName: String): Unit =
+ sql(s"""CREATE TABLE $tableName (id INT, payload STRING, dt STRING)
+ |USING CSV
+ |PARTITIONED BY (dt)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'metastore.partitioned-table' = 'false')
+ |""".stripMargin)
+
+ private def executeManagedRepair(statement: String): Unit = {
+ val result = sql(statement)
+ val resultPlan = result.queryExecution.executedPlan
+ assert(resultPlan.isInstanceOf[CommandResultExec], resultPlan.treeString)
+ val commandPlan = resultPlan.asInstanceOf[CommandResultExec].commandPhysicalPlan
+ assert(commandPlan.isInstanceOf[PaimonRepairFormatTablePartitionsExec], commandPlan.treeString)
+ result.collect()
+ }
+
+ private def assertMsckRejected(statement: String): Unit = {
+ val error = intercept[Exception] {
+ sql(statement).collect()
+ }
+ val messages = causeMessages(error)
+ assert(messages.contains("MSCK REPAIR TABLE"), messages)
+ assert(messages.toLowerCase(java.util.Locale.ROOT).contains("not supported"), messages)
+ }
+
+ private def assertPartitionState(tableName: String, expected: Set[String]): Unit = {
+ assert(registeredPartitions(tableName) == expected)
+ assert(
+ sql(s"SHOW PARTITIONS $tableName").collect().map(_.getString(0)).toSet ==
+ expected.map(value => s"dt=$value"))
+ }
+
+ private def assertRowsAndChecksum(
+ tableName: String,
+ expectedRows: Seq[Row],
+ expectedIdSum: Long): Unit = {
+ checkAnswer(sql(s"SELECT id, payload, dt FROM $tableName ORDER BY id"), expectedRows)
+ checkAnswer(
+ sql(s"SELECT COUNT(*), COALESCE(SUM(id), 0) FROM $tableName"),
+ Seq(Row(expectedRows.size.toLong, expectedIdSum)))
+ }
+
+ private def runCommand(command: AnyRef): Unit = {
+ try {
+ command.getClass.getMethod("run").invoke(command)
+ } catch {
+ case error: InvocationTargetException => throw error.getCause
+ }
+ }
+
+ private def tableIdentifier(tableName: String): Identifier =
+ Identifier.create(dbName0, tableName)
+
+ private def formatTable(tableName: String): FormatTable =
+ paimonCatalog.getTable(tableIdentifier(tableName)).asInstanceOf[FormatTable]
+
+ private def writeCsvPartition(
+ tableName: String,
+ partition: String,
+ id: Int,
+ payload: String): Path = {
+ val table = formatTable(tableName)
+ val partitionPath = new Path(table.location(), s"dt=$partition")
+ table.fileIO().mkdirs(partitionPath)
+ table.fileIO().writeFile(new Path(partitionPath, f"part-$id%05d.csv"), s"$id,$payload\n", false)
+ partitionPath
+ }
+
+ private def registerPartitions(tableName: String, partitions: String*): Unit =
+ paimonCatalog.createPartitions(
+ tableIdentifier(tableName),
+ partitions.map(value => Map("dt" -> value).asJava).asJava,
+ true)
+
+ private def registeredPartitions(tableName: String): Set[String] =
+ paimonCatalog
+ .listPartitions(tableIdentifier(tableName))
+ .asScala
+ .map(_.spec().get("dt"))
+ .toSet
+
+ private def causeMessages(error: Throwable): String =
+ Iterator
+ .iterate(error)(_.getCause)
+ .takeWhile(_ != null)
+ .flatMap(cause => Option(cause.getMessage))
+ .mkString(" | ")
+}
+
+private[sql] object MsckFaultInjection {
+ val DROP_FAILURE = "injected MSCK partition drop failure"
+ val LIST_FAILURE = "injected MSCK partition list failure"
+ val CREATE_LOST_RESPONSE = "injected MSCK create lost response"
+ val REFRESH_FAILURE = "injected MSCK cache refresh failure"
+
+ @volatile var failDrop = false
+ @volatile var failList = false
+ @volatile var createCalls = 0
+ @volatile var dropCalls = 0
+ @volatile var listCalls = 0
+
+ def reset(): Unit = {
+ failDrop = false
+ failList = false
+ createCalls = 0
+ dropCalls = 0
+ listCalls = 0
+ }
+}
+
+private[sql] class StatefulFaultCatalog(initial: Set[Map[String, String]] = Set.empty)
+ extends FormatTablePartitionCatalog {
+
+ private val state = mutable.LinkedHashSet(initial.toSeq: _*)
+ var failList = false
+ var failCreateAfterApply = false
+ var failDrop = false
+ var dropFailure: RuntimeException = null
+ var createCalls = 0
+ var dropCalls = 0
+
+ def partitions: Set[Map[String, String]] = state.toSet
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ createCalls += 1
+ state ++= partitions.asScala.map(_.asScala.toMap)
+ if (failCreateAfterApply) {
+ throw new IllegalStateException(MsckFaultInjection.CREATE_LOST_RESPONSE)
+ }
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropCalls += 1
+ if (dropFailure != null) {
+ throw dropFailure
+ }
+ if (failDrop) {
+ throw new IllegalStateException(MsckFaultInjection.DROP_FAILURE)
+ }
+ state --= partitions.asScala.map(_.asScala.toMap)
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ partitions.asScala
+ .map(_.asScala.toMap)
+ .filter(state.contains)
+ .map(_.asJava)
+ .asJava
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ if (failList) {
+ throw new IllegalStateException(MsckFaultInjection.LIST_FAILURE)
+ }
+ if (pageToken != null) {
+ throw new AssertionError(s"Unexpected page token $pageToken")
+ }
+ FormatTablePartitionPage(state.toSeq.map(_.asJava).asJava, null)
+ }
+}
+
+/** Spark-loadable test catalog which delegates all state to the embedded Paimon REST catalog. */
+private[sql] class FaultInjectingFormatTableSparkCatalog extends SparkCatalog {
+
+ override def loadTable(ident: SparkIdentifier): SparkTable = {
+ super.loadTable(ident) match {
+ case table: PaimonFormatTable if table.partitionCatalog != null =>
+ new PaimonFormatTable(
+ table.table,
+ new FaultInjectingFormatTablePartitionCatalog(table.partitionCatalog))
+ case table => table
+ }
+ }
+}
+
+private[sql] class FaultInjectingFormatTablePartitionCatalog(delegate: FormatTablePartitionCatalog)
+ extends FormatTablePartitionCatalog {
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ delegate.createPartitions(partitions, ignoreIfExists)
+ MsckFaultInjection.createCalls += 1
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ MsckFaultInjection.dropCalls += 1
+ if (MsckFaultInjection.failDrop) {
+ throw new IllegalStateException(MsckFaultInjection.DROP_FAILURE)
+ }
+ delegate.dropPartitions(partitions)
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[JMap[String, String]] =
+ delegate.listPartitionsByNames(partitions)
+
+ override def listPartitions(
+ prefix: JMap[String, String],
+ pageToken: String,
+ pageSize: Int): FormatTablePartitionPage = {
+ MsckFaultInjection.listCalls += 1
+ if (MsckFaultInjection.failList) {
+ throw new IllegalStateException(MsckFaultInjection.LIST_FAILURE)
+ }
+ delegate.listPartitions(prefix, pageToken, pageSize)
+ }
+}