listPartitionsByNames(
throw new TableNoPermissionException(identifier, e);
} catch (NotImplementedException e) {
Table table = getTable(identifier);
- if (isCatalogManagedFormatTable(table)) {
+ if (hasCatalogManagedPartitions(table)) {
throw e;
}
return listPartitionsFromFileSystem(table, partitions);
@@ -1323,9 +1334,13 @@ RESTApi api() {
return api;
}
- private static boolean isCatalogManagedFormatTable(Table table) {
- return table instanceof FormatTable
- && new CoreOptions(table.options()).partitionedTableInMetastore();
+ /**
+ * Whether the catalog owns this table's partitions, which is exactly whether loading it
+ * produced a partition manager. Reading the option instead would be a second answer to the same
+ * question, and the two can disagree on a table built outside the catalog.
+ */
+ private static boolean hasCatalogManagedPartitions(Table table) {
+ return table instanceof FormatTable && ((FormatTable) table).partitionManager() != null;
}
private FileIO fileIOForData(Path path, Identifier identifier) {
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
index 9a832176caaf..f15278a7fd36 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FormatTable.java
@@ -18,9 +18,12 @@
package org.apache.paimon.table;
+import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
+import org.apache.paimon.annotation.Experimental;
import org.apache.paimon.annotation.Public;
import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogUtils;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.manifest.IndexManifestEntry;
@@ -29,6 +32,7 @@
import org.apache.paimon.stats.Statistics;
import org.apache.paimon.table.format.FormatBatchWriteBuilder;
import org.apache.paimon.table.format.FormatReadBuilder;
+import org.apache.paimon.table.format.FormatTablePartitionManager;
import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.StreamWriteBuilder;
import org.apache.paimon.table.source.BatchVectorSearchBuilder;
@@ -56,8 +60,9 @@
* operations on this table allow for reading or writing to these files, facilitating the retrieval
* of existing data and the addition of new files.
*
- * Partitioned file format table just like the standard hive format. Partitions are discovered
- * and inferred based on directory structure.
+ *
A partitioned file format table uses the standard Hive directory layout. By default,
+ * partitions are discovered from that layout. Format Tables with catalog-managed partitions use
+ * catalog metadata for partition visibility while retaining the same physical layout.
*
* @since 0.9.0
*/
@@ -75,6 +80,17 @@ public interface FormatTable extends Table {
CatalogContext catalogContext();
+ /**
+ * The catalog partition registrations of this table, or null when its partitions are discovered
+ * from the filesystem. When non-null, catalog registration decides partition visibility: a
+ * partition directory that is not registered is not visible to scans.
+ */
+ @Experimental
+ @Nullable
+ default FormatTablePartitionManager partitionManager() {
+ return null;
+ }
+
/** Currently supported formats. */
enum Format {
ORC,
@@ -115,6 +131,7 @@ class Builder {
private Map options;
@Nullable private String comment;
private CatalogContext catalogContext;
+ @Nullable private FormatTablePartitionManager partitionManager;
public Builder fileIO(FileIO fileIO) {
this.fileIO = fileIO;
@@ -161,6 +178,12 @@ public Builder catalogContext(CatalogContext catalogContext) {
return this;
}
+ @Experimental
+ public Builder partitionManager(@Nullable FormatTablePartitionManager partitionManager) {
+ this.partitionManager = partitionManager;
+ return this;
+ }
+
public FormatTable build() {
return new FormatTableImpl(
fileIO,
@@ -171,7 +194,8 @@ public FormatTable build() {
format,
options,
comment,
- catalogContext);
+ catalogContext,
+ partitionManager);
}
}
@@ -189,6 +213,7 @@ class FormatTableImpl implements FormatTable {
private final Map options;
@Nullable private final String comment;
private CatalogContext catalogContext;
+ @Nullable private final FormatTablePartitionManager partitionManager;
public FormatTableImpl(
FileIO fileIO,
@@ -199,7 +224,8 @@ public FormatTableImpl(
Format format,
Map options,
@Nullable String comment,
- CatalogContext catalogContext) {
+ CatalogContext catalogContext,
+ @Nullable FormatTablePartitionManager partitionManager) {
this.fileIO = fileIO;
this.identifier = identifier;
this.rowType = rowType;
@@ -209,6 +235,7 @@ public FormatTableImpl(
this.options = options;
this.comment = comment;
this.catalogContext = catalogContext;
+ this.partitionManager = partitionManager;
}
@Override
@@ -265,6 +292,31 @@ public FileIO fileIO() {
public FormatTable copy(Map dynamicOptions) {
Map newOptions = new HashMap<>(options);
newOptions.putAll(dynamicOptions);
+
+ CoreOptions coreOptions = CoreOptions.fromMap(options);
+ CoreOptions copiedCoreOptions = CoreOptions.fromMap(newOptions);
+ boolean hasCatalogManagedPartitions = partitionManager != null;
+ boolean persistedPartitionsFromCatalog = coreOptions.formatTablePartitionsFromCatalog();
+ boolean dynamicPartitionsFromCatalog =
+ copiedCoreOptions.formatTablePartitionsFromCatalog();
+ if (persistedPartitionsFromCatalog != dynamicPartitionsFromCatalog) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Dynamic option '%s' cannot change where a Format Table's partitions come from.",
+ CoreOptions.FORMAT_TABLE_PARTITION_SOURCE.key()));
+ }
+ if (hasCatalogManagedPartitions
+ && coreOptions.formatTablePartitionOnlyValueInPath()
+ != copiedCoreOptions.formatTablePartitionOnlyValueInPath()) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Dynamic option '%s' cannot change the physical partition layout of a Format Table with catalog-managed partitions.",
+ CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key()));
+ }
+ if (hasCatalogManagedPartitions) {
+ CatalogUtils.validateCatalogManagedPartitionOptions(newOptions);
+ }
+
return new FormatTableImpl(
fileIO,
identifier,
@@ -274,7 +326,8 @@ public FormatTable copy(Map dynamicOptions) {
format,
newOptions,
comment,
- catalogContext);
+ catalogContext,
+ partitionManager);
}
@Override
@@ -302,6 +355,12 @@ public FullTextSearchBuilder newFullTextSearchBuilder() {
public CatalogContext catalogContext() {
return this.catalogContext;
}
+
+ @Override
+ @Nullable
+ public FormatTablePartitionManager partitionManager() {
+ return partitionManager;
+ }
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java
new file mode 100644
index 000000000000..7085135f17ff
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManager.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.PagedList;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogLoader;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.utils.FunctionWithException;
+import org.apache.paimon.utils.PartitionPathUtils;
+import org.apache.paimon.utils.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/**
+ * A {@link FormatTablePartitionManager} reading and writing partitions through a {@link Catalog}.
+ * One catalog is created per operation and closed when it ends, so the table stays serializable and
+ * no client outlives the call that needed it.
+ */
+class CatalogFormatTablePartitionManager implements FormatTablePartitionManager {
+
+ private static final long serialVersionUID = 1L;
+
+ /** Bounds one request: catalog services cap the partitions a single call may carry. */
+ private static final int REQUEST_SIZE = 1000;
+
+ private final Identifier identifier;
+ private final List partitionKeys;
+ private final CatalogLoader catalogLoader;
+
+ CatalogFormatTablePartitionManager(
+ Identifier identifier, List partitionKeys, CatalogLoader catalogLoader) {
+ this.identifier = identifier;
+ // Copied: the caller's list must not be able to change what counts as a leading prefix.
+ this.partitionKeys = Collections.unmodifiableList(new ArrayList<>(partitionKeys));
+ this.catalogLoader = catalogLoader;
+ }
+
+ @Override
+ public List listPartitions(Map prefix) {
+ LinkedHashMap ordered = orderedPrefix(partitionKeys, prefix);
+ checkArgument(
+ ordered.size() == prefix.size(),
+ "Partition prefix %s is not a leading prefix of partition keys %s of table %s.",
+ prefix,
+ partitionKeys,
+ identifier.getFullName());
+ String pattern = PartitionPathUtils.buildPartitionNamePrefixPattern(partitionKeys, ordered);
+ return execute(
+ catalog -> {
+ List partitions = new ArrayList<>();
+ Set seenPageTokens = new HashSet<>();
+ String pageToken = null;
+ do {
+ PagedList page =
+ catalog.listPartitionsPaged(
+ identifier, REQUEST_SIZE, pageToken, pattern);
+ if (page.getElements() != null) {
+ partitions.addAll(page.getElements());
+ }
+ pageToken = page.getNextPageToken();
+ if (StringUtils.isNotEmpty(pageToken) && !seenPageTokens.add(pageToken)) {
+ throw new IllegalStateException(
+ String.format(
+ "Catalog returned repeated partition page token '%s' for format table %s.",
+ pageToken, identifier.getFullName()));
+ }
+ } while (StringUtils.isNotEmpty(pageToken));
+ // A catalog may ignore the pattern, so the prefix is enforced here as well.
+ partitions.removeIf(partition -> !matchesPrefix(partition, prefix));
+ return Collections.unmodifiableList(partitions);
+ },
+ "list partitions");
+ }
+
+ @Override
+ public List listPartitionsByNames(List> partitions) {
+ if (partitions.isEmpty()) {
+ return Collections.emptyList();
+ }
+ return execute(
+ catalog -> {
+ List found = new ArrayList<>();
+ for (List> batch : batches(partitions)) {
+ found.addAll(catalog.listPartitionsByNames(identifier, batch));
+ }
+ return Collections.unmodifiableList(found);
+ },
+ "list partitions by names");
+ }
+
+ @Override
+ public void createPartitions(List> partitions, boolean ignoreIfExists) {
+ if (partitions.isEmpty()) {
+ return;
+ }
+ execute(
+ catalog -> {
+ if (!ignoreIfExists) {
+ // Rejecting the whole batch when any partition exists is only meaningful
+ // if the batch stays one request, so a strict create is never split.
+ catalog.createPartitions(identifier, partitions, false);
+ return null;
+ }
+ // An idempotent create is safe to split: a rerun converges from a partially
+ // applied batch.
+ for (List> batch : batches(partitions)) {
+ catalog.createPartitions(identifier, batch, true);
+ }
+ return null;
+ },
+ "create partitions");
+ }
+
+ @Override
+ public void dropPartitions(List> partitions) {
+ if (partitions.isEmpty()) {
+ return;
+ }
+ execute(
+ catalog -> {
+ // Dropping ignores missing partitions, so a partially applied batch converges
+ // on a rerun and may be split.
+ for (List> batch : batches(partitions)) {
+ catalog.dropPartitions(identifier, batch);
+ }
+ return null;
+ },
+ "drop partitions");
+ }
+
+ private static boolean matchesPrefix(Partition partition, Map prefix) {
+ for (Map.Entry entry : prefix.entrySet()) {
+ if (!entry.getValue().equals(partition.spec().get(entry.getKey()))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static List>> batches(List> partitions) {
+ List>> batches = new ArrayList<>();
+ for (int start = 0; start < partitions.size(); start += REQUEST_SIZE) {
+ batches.add(
+ partitions.subList(start, Math.min(start + REQUEST_SIZE, partitions.size())));
+ }
+ return batches;
+ }
+
+ private T execute(FunctionWithException operation, String what) {
+ try (Catalog catalog = catalogLoader.load()) {
+ return operation.apply(catalog);
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new RuntimeException(
+ String.format(
+ "Failed to %s of format table %s.", what, identifier.getFullName()),
+ e);
+ }
+ }
+
+ /** Order a prefix by the table partition keys, stopping at the first key it does not cover. */
+ private static LinkedHashMap orderedPrefix(
+ List partitionKeys, Map prefix) {
+ LinkedHashMap ordered = new LinkedHashMap<>();
+ for (String partitionKey : partitionKeys) {
+ if (!prefix.containsKey(partitionKey)) {
+ break;
+ }
+ ordered.put(partitionKey, prefix.get(partitionKey));
+ }
+ return ordered;
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
index 924e67dbdfdc..0b510c594051 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatBatchWriteBuilder.java
@@ -87,7 +87,8 @@ public BatchTableCommit newCommit() {
Identifier.fromString(table.fullName()),
staticPartition,
syncHiveUri,
- table.catalogContext());
+ table.catalogContext(),
+ table.partitionManager());
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
index ed98c6ba50ff..c0f356fe226d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableCommit.java
@@ -48,7 +48,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.StringJoiner;
import static org.apache.paimon.table.format.FormatBatchWriteBuilder.validateStaticPartition;
@@ -63,6 +62,7 @@ public class FormatTableCommit implements BatchTableCommit {
protected boolean overwrite = false;
private Catalog hiveCatalog;
private Identifier tableIdentifier;
+ @Nullable private final FormatTablePartitionManager partitionManager;
public FormatTableCommit(
String location,
@@ -73,7 +73,8 @@ public FormatTableCommit(
Identifier tableIdentifier,
@Nullable Map staticPartitions,
@Nullable String syncHiveUri,
- CatalogContext catalogContext) {
+ CatalogContext catalogContext,
+ @Nullable FormatTablePartitionManager partitionManager) {
this.location = location;
this.fileIO = fileIO;
this.formatTablePartitionOnlyValueInPath = formatTablePartitionOnlyValueInPath;
@@ -82,6 +83,7 @@ public FormatTableCommit(
this.overwrite = overwrite;
this.partitionKeys = partitionKeys;
this.tableIdentifier = tableIdentifier;
+ this.partitionManager = partitionManager;
if (syncHiveUri != null) {
try {
Options options = new Options();
@@ -143,7 +145,9 @@ public void commit(List commitMessages) {
for (TwoPhaseOutputStream.Committer committer : committers) {
committer.commit(this.fileIO);
- if (partitionKeys != null && !partitionKeys.isEmpty() && hiveCatalog != null) {
+ if (partitionKeys != null
+ && !partitionKeys.isEmpty()
+ && (hiveCatalog != null || partitionManager != null)) {
partitionSpecs.add(
extractPartitionSpecFromPath(
committer.targetPath().getParent(), partitionKeys));
@@ -152,6 +156,11 @@ public void commit(List commitMessages) {
for (TwoPhaseOutputStream.Committer committer : committers) {
committer.clean(this.fileIO);
}
+ if (partitionManager != null && !partitionSpecs.isEmpty()) {
+ // Concurrent writers may touch the same partition, so registration is an
+ // idempotent ADD rather than a strict create.
+ partitionManager.createPartitions(new ArrayList<>(partitionSpecs), true);
+ }
for (Map partitionSpec : partitionSpecs) {
if (hiveCatalog != null) {
try {
@@ -192,12 +201,25 @@ private Method getHiveCreatePartitionsInHmsMethod() throws NoSuchMethodException
private LinkedHashMap extractPartitionSpecFromPath(
Path partitionPath, List partitionKeys) {
- if (formatTablePartitionOnlyValueInPath) {
- return PartitionPathUtils.extractPartitionSpecFromPathOnlyValue(
- partitionPath, partitionKeys);
- } else {
- return PartitionPathUtils.extractPartitionSpecFromPath(partitionPath);
+ // Only the trailing partitionKeys.size() components are the spec: the table location
+ // itself may contain foreign 'k=v' segments that must not leak into what is registered.
+ LinkedHashMap partitionSpec =
+ formatTablePartitionOnlyValueInPath
+ ? PartitionPathUtils.extractPartitionSpecFromPathOnlyValue(
+ partitionPath, partitionKeys)
+ : PartitionPathUtils.extractPartitionSpecFromPath(
+ partitionPath, partitionKeys);
+ if (partitionSpec == null) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Partition path '%s' does not end in the %s partition directories of "
+ + "table %s declared by partition keys %s.",
+ partitionPath,
+ partitionKeys.size(),
+ tableIdentifier.getFullName(),
+ partitionKeys));
}
+ return partitionSpec;
}
private static Path buildPartitionPath(
@@ -208,20 +230,19 @@ private static Path buildPartitionPath(
if (partitionSpec.isEmpty() || partitionKeys.isEmpty()) {
throw new IllegalArgumentException("partitionSpec or partitionKeys is empty.");
}
- StringJoiner joiner = new StringJoiner("/");
+ LinkedHashMap orderedSpec = new LinkedHashMap<>();
for (int i = 0; i < partitionSpec.size(); i++) {
String key = partitionKeys.get(i);
if (partitionSpec.containsKey(key)) {
- if (formatTablePartitionOnlyValueInPath) {
- joiner.add(partitionSpec.get(key));
- } else {
- joiner.add(key + "=" + partitionSpec.get(key));
- }
+ orderedSpec.put(key, partitionSpec.get(key));
} else {
throw new RuntimeException("partitionSpec does not contain key: " + key);
}
}
- return new Path(location, joiner.toString());
+ return new Path(
+ location,
+ PartitionPathUtils.generatePartitionPathUtil(
+ orderedSpec, formatTablePartitionOnlyValueInPath));
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionManager.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionManager.java
new file mode 100644
index 000000000000..64b643fba26e
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTablePartitionManager.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.annotation.Experimental;
+import org.apache.paimon.catalog.CatalogLoader;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.partition.Partition;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The catalog partition registrations of a single Format Table with catalog-managed partitions.
+ *
+ * This manages registration metadata only: it never creates, deletes or resolves a partition
+ * directory. Partition values are raw values, never escaped directory names, and building
+ * directories from them — including keeping them inside the table — is the caller's job.
+ *
+ *
Paging, page tokens, name patterns and per-request batching are implementation details; every
+ * method here takes or returns its complete argument.
+ *
+ * @since 1.5
+ */
+@Experimental
+public interface FormatTablePartitionManager extends Serializable {
+
+ /**
+ * List every partition whose leading partition values match {@code prefix}, or all partitions
+ * when it is empty. The prefix must be a contiguous leading subset of the table's partition
+ * keys.
+ */
+ List listPartitions(Map prefix);
+
+ /** Return those of the given complete partition specs that are registered. */
+ List listPartitionsByNames(List> partitions);
+
+ /**
+ * Register partitions. With {@code ignoreIfExists=false} the whole batch is rejected when any
+ * partition already exists, so such a request is never split.
+ */
+ void createPartitions(List> partitions, boolean ignoreIfExists);
+
+ /** Unregister partitions. Metadata only; missing partitions are ignored. */
+ void dropPartitions(List> partitions);
+
+ /** Create a manager reading and writing partitions through a {@link CatalogLoader}. */
+ static FormatTablePartitionManager create(
+ Identifier identifier, List partitionKeys, CatalogLoader catalogLoader) {
+ return new CatalogFormatTablePartitionManager(identifier, partitionKeys, catalogLoader);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
index cb0edefc82fc..44908749cad6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/format/FormatTableScan.java
@@ -32,6 +32,7 @@
import org.apache.paimon.fs.Path;
import org.apache.paimon.manifest.PartitionEntry;
import org.apache.paimon.options.Options;
+import org.apache.paimon.partition.Partition;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.partition.PartitionPredicate.AndPartitionPredicate;
import org.apache.paimon.partition.PartitionPredicate.DefaultPartitionPredicate;
@@ -59,12 +60,14 @@
import javax.annotation.Nullable;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -81,9 +84,10 @@ public class FormatTableScan implements InnerTableScan {
private static final Logger LOG = LoggerFactory.getLogger(FormatTableScan.class);
- private final FormatTable table;
- private final CoreOptions coreOptions;
+ final FormatTable table;
+ final CoreOptions coreOptions;
@Nullable private PartitionPredicate partitionFilter;
+ @Nullable private final FormatTablePartitionManager partitionManager;
@Nullable private final Integer limit;
private final long targetSplitSize;
private final long openFileCost;
@@ -97,6 +101,7 @@ public FormatTableScan(
this.coreOptions = new CoreOptions(table.options());
this.partitionFilter = partitionFilter;
this.limit = limit;
+ this.partitionManager = table.partitionManager();
this.targetSplitSize = coreOptions.splitTargetSize();
this.openFileCost = coreOptions.splitOpenFileCost();
this.format = table.format();
@@ -115,6 +120,29 @@ public Plan plan() {
@Override
public List listPartitionEntries() {
+ if (partitionManager != null) {
+ List partitions = partitionManager.listPartitions(Collections.emptyMap());
+ if (partitions.isEmpty()) {
+ warnIfFilesystemPartitionsExist();
+ }
+ boolean onlyValueInPath = coreOptions.formatTablePartitionOnlyValueInPath();
+ List entries = new ArrayList<>(partitions.size());
+ Set> seen = new HashSet<>(partitions.size());
+ for (Partition partition : partitions) {
+ if (!seen.add(partition.spec())) {
+ continue;
+ }
+ entries.add(
+ new PartitionEntry(
+ toPartitionRow(normalizeSpec(partition.spec(), onlyValueInPath)),
+ partition.recordCount(),
+ partition.fileSizeInBytes(),
+ partition.fileCount(),
+ partition.lastFileCreationTime(),
+ partition.totalBuckets()));
+ }
+ return entries;
+ }
List, Path>> partition2Paths =
searchPartSpecAndPaths(
table.fileIO(),
@@ -142,7 +170,7 @@ public static boolean isDataFileName(String fileName) {
return fileName != null && !fileName.startsWith(".") && !fileName.startsWith("_");
}
- private BinaryRow toPartitionRow(LinkedHashMap partitionSpec) {
+ BinaryRow toPartitionRow(LinkedHashMap partitionSpec) {
RowType partitionType = table.partitionType();
GenericRow row =
convertSpecToInternalRow(partitionSpec, partitionType, table.defaultPartName());
@@ -160,7 +188,28 @@ public List splits() {
LinkedHashMap partitionSpec = pair.getKey();
BinaryRow partitionRow = toPartitionRow(partitionSpec);
if (partitionFilter == null || partitionFilter.test(partitionRow)) {
- splits.addAll(createSplits(fileIO, pair.getValue(), partitionRow));
+ try {
+ splits.addAll(createSplits(fileIO, pair.getValue(), partitionRow));
+ } catch (FileNotFoundException e) {
+ if (partitionManager == null) {
+ throw e;
+ }
+ // A registered partition without a directory reads as empty,
+ // matching Hive (e.g. ADD PARTITION before the first INSERT).
+ // Warn so a directory removed behind the catalog's back stays
+ // discoverable.
+ LOG.warn(
+ "Partition '{}' of format table {} is registered in "
+ + "the catalog but its directory '{}' does not "
+ + "exist; treating the partition as empty. If the "
+ + "directory was removed on purpose, drop the "
+ + "partition or repair the metadata, e.g. with "
+ + "MSCK REPAIR TABLE.",
+ PartitionPathUtils.generatePartitionName(
+ partitionSpec, false),
+ table.fullName(),
+ pair.getValue());
+ }
}
}
} else {
@@ -178,6 +227,16 @@ public List splits() {
}
List, Path>> findPartitions() {
+ if (partitionManager != null) {
+ Map prefix = leadingEqualityPrefix();
+ List partitions = partitionManager.listPartitions(prefix);
+ if (partitions.isEmpty() && prefix.isEmpty()) {
+ warnIfFilesystemPartitionsExist();
+ }
+ // The prefix is a coarse pre-filter; the full predicate is applied per partition in
+ // the plan, as it is for filesystem discovery.
+ return toSpecsAndPaths(partitions, coreOptions.formatTablePartitionOnlyValueInPath());
+ }
LOG.debug(
"Find partitions for format table {}, partition filter: {}",
table.name(),
@@ -222,6 +281,101 @@ List, Path>> findPartitions() {
}
}
+ /**
+ * Turn the partitions a catalog reports into the specs and directories to read. Catalog
+ * metadata is not trusted for path construction: a spec that cannot form a partition directory
+ * of this table is rejected rather than resolved to some other directory.
+ */
+ private List, Path>> toSpecsAndPaths(
+ List partitions, boolean onlyValueInPath) {
+ List, Path>> result = new ArrayList<>(partitions.size());
+ Path tablePath = new Path(table.location());
+ // Do not trust the catalog to be duplicate-free: a repeated spec would double every split
+ // of that partition and silently duplicate query results.
+ Set seenPartitionPaths = new HashSet<>(partitions.size());
+ for (Partition partition : partitions) {
+ LinkedHashMap spec = normalizeSpec(partition.spec(), onlyValueInPath);
+ String partitionPath =
+ PartitionPathUtils.generatePartitionPathUtil(spec, onlyValueInPath);
+ if (seenPartitionPaths.add(partitionPath)) {
+ result.add(Pair.of(spec, new Path(tablePath, partitionPath)));
+ }
+ }
+ return result;
+ }
+
+ /** Order a catalog spec by the table partition keys and check it can form a directory. */
+ private LinkedHashMap normalizeSpec(
+ @Nullable Map spec, boolean onlyValueInPath) {
+ List partitionKeys = table.partitionKeys();
+ if (spec == null
+ || spec.size() != partitionKeys.size()
+ || !spec.keySet().containsAll(partitionKeys)) {
+ throw corruptPartitionSpec(spec);
+ }
+ LinkedHashMap normalized = new LinkedHashMap<>();
+ for (String partitionKey : partitionKeys) {
+ String value = spec.get(partitionKey);
+ // In a value-only layout, '.' and '..' are complete path components and would resolve
+ // to a directory outside the table.
+ try {
+ PartitionPathUtils.validatePartitionValueForPath(value, onlyValueInPath);
+ } catch (IllegalArgumentException e) {
+ throw corruptPartitionSpec(spec);
+ }
+ normalized.put(partitionKey, value);
+ }
+ return normalized;
+ }
+
+ private IllegalStateException corruptPartitionSpec(@Nullable Map spec) {
+ return new IllegalStateException(
+ String.format(
+ "Catalog returned corrupt partition metadata %s for format table %s; "
+ + "expected exactly the partition keys %s with values usable as "
+ + "path components.",
+ spec, table.fullName(), table.partitionKeys()));
+ }
+
+ /**
+ * The leading equality prefix of the partition predicate, in partition-key order, pushed down
+ * to the partition catalog. Empty when there is no predicate or it does not start with
+ * equalities on the leading partition keys.
+ */
+ private Map leadingEqualityPrefix() {
+ Optional predicate = extractPartitionPredicate(partitionFilter);
+ if (!predicate.isPresent()) {
+ return Collections.emptyMap();
+ }
+ return extractLeadingEqualityPartitionSpecWhenOnlyAnd(
+ table.partitionKeys(), predicate.get(), table.partitionType());
+ }
+
+ /**
+ * Warn when the catalog knows no partitions but the table directory contains subdirectories:
+ * typically a table that predates catalog-managed partitions (or was written by a client that
+ * does not register partitions) and needs a metadata sync before its data becomes visible.
+ */
+ private void warnIfFilesystemPartitionsExist() {
+ try {
+ for (FileStatus status : table.fileIO().listStatus(new Path(table.location()))) {
+ if (status.isDir() && !status.getPath().getName().startsWith(".")) {
+ LOG.warn(
+ "Format table {} has no partitions registered in the catalog "
+ + "but its location {} contains directories. Data written "
+ + "before enabling catalog-managed partitions (or by clients "
+ + "that do not register partitions) is invisible until the "
+ + "partition metadata is synced, e.g. with MSCK REPAIR TABLE.",
+ table.fullName(),
+ table.location());
+ return;
+ }
+ }
+ } catch (IOException ignored) {
+ // Best-effort hint only; never fail or slow down the scan because of it.
+ }
+ }
+
protected static List, Path>> generatePartitions(
List partitionKeys,
RowType partitionType,
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
index fb7baf5b06b2..44ded39b6ab5 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/PartitionPathUtils.java
@@ -109,13 +109,105 @@ public static String generatePartitionPathUtil(
suffixBuf.append(escapePathName(e.getKey()));
suffixBuf.append('=');
}
- suffixBuf.append(escapePathName(e.getValue()));
+ String value = e.getValue();
+ validatePartitionValueForPath(value, onlyValue);
+ suffixBuf.append(escapePathName(value));
i++;
}
suffixBuf.append(Path.SEPARATOR);
return suffixBuf.toString();
}
+ /**
+ * Generate a partition path without the trailing separator, e.g. {@code dt=20250101/hr=01}.
+ * This is the canonical partition name used when talking to a partition-managing catalog.
+ */
+ public static String generatePartitionName(
+ LinkedHashMap partitionSpec, boolean onlyValue) {
+ String path = generatePartitionPathUtil(partitionSpec, onlyValue);
+ return path.endsWith(Path.SEPARATOR)
+ ? path.substring(0, path.length() - Path.SEPARATOR.length())
+ : path;
+ }
+
+ /**
+ * Validate that a partition value is safe for the configured path layout. In a key-value
+ * layout, values such as {@code "."} are part of a component such as {@code "pt=."} and are
+ * safe. In a value-only layout, {@code "."} and {@code ".."} are complete path components and
+ * would resolve to a different directory.
+ */
+ public static void validatePartitionValueForPath(String value, boolean onlyValueInPath) {
+ if (value == null
+ || value.isEmpty()
+ || (onlyValueInPath && (".".equals(value) || "..".equals(value)))) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Partition value '%s' cannot be used as a partition path component.",
+ value));
+ }
+ }
+
+ /** Conservatively validate a value when the physical partition layout is unknown. */
+ public static void validatePartitionValueForPath(String value) {
+ validatePartitionValueForPath(value, true);
+ }
+
+ /** Validate every value of a partition spec for the configured path layout. */
+ public static void validatePartitionSpecForPath(
+ Map partitionSpec, boolean onlyValueInPath) {
+ for (String value : partitionSpec.values()) {
+ validatePartitionValueForPath(value, onlyValueInPath);
+ }
+ }
+
+ /** Conservatively validate a spec when the physical partition layout is unknown. */
+ public static void validatePartitionSpecForPath(Map partitionSpec) {
+ validatePartitionSpecForPath(partitionSpec, true);
+ }
+
+ /**
+ * Build the partition-name prefix pattern pushed down to a partition-managing catalog from the
+ * leading equality prefix of a partition predicate.
+ *
+ * Pattern contract (shared by every engine talking to the catalog): partition names are the
+ * escaped {@code key=value} form joined by {@code '/'}; {@code '%'} is the only wildcard and
+ * there is no escape sequence for it ({@code '_'} stays a literal). A complete spec matches the
+ * exact partition name; an incomplete prefix is suffixed with {@code '%'}.
+ *
+ *
Returns {@code null} whenever pushdown must be skipped and the caller should list all
+ * partitions instead: the equality prefix is empty, a prefix value is blank, or the escaped
+ * prefix contains a literal {@code '%'} that the contract cannot express.
+ */
+ @Nullable
+ public static String buildPartitionNamePrefixPattern(
+ List partitionKeys, Map equalityPrefix) {
+ if (equalityPrefix.isEmpty()) {
+ return null;
+ }
+ LinkedHashMap orderedPrefix = new LinkedHashMap<>();
+ for (String partitionKey : partitionKeys) {
+ if (!equalityPrefix.containsKey(partitionKey)) {
+ break;
+ }
+ String value = equalityPrefix.get(partitionKey);
+ if (StringUtils.isNullOrWhitespaceOnly(value)) {
+ return null;
+ }
+ orderedPrefix.put(partitionKey, value);
+ }
+ if (orderedPrefix.isEmpty()) {
+ return null;
+ }
+ String escapedPrefix = generatePartitionPath(orderedPrefix);
+ if (escapedPrefix.indexOf('%') >= 0) {
+ return null;
+ }
+ if (orderedPrefix.size() == partitionKeys.size()) {
+ return escapedPrefix.substring(0, escapedPrefix.length() - 1);
+ }
+ return escapedPrefix + '%';
+ }
+
public static List generatePartitionPaths(
List> partitions, RowType partitionType) {
return partitions.stream()
@@ -265,12 +357,53 @@ public static LinkedHashMap extractPartitionSpecFromPath(Path cu
return fullPartSpec;
}
+ /**
+ * Extract exactly the trailing {@code key=value} components for the declared partition keys, or
+ * null when the path does not end in them. Values are unescaped, so they are the raw partition
+ * values.
+ */
+ @Nullable
+ public static LinkedHashMap extractPartitionSpecFromPath(
+ Path currPath, List partitionKeys) {
+ String[] values = new String[partitionKeys.size()];
+ Path current = currPath;
+ for (int i = partitionKeys.size() - 1; i >= 0; i--) {
+ if (current == null) {
+ return null;
+ }
+ Matcher matcher = PARTITION_NAME_PATTERN.matcher(current.getName());
+ if (!matcher.matches()
+ || !partitionKeys.get(i).equals(unescapePathName(matcher.group(1)))) {
+ return null;
+ }
+ values[i] = unescapePathName(matcher.group(2));
+ current = current.getParent();
+ }
+
+ LinkedHashMap spec = new LinkedHashMap<>();
+ for (int i = 0; i < partitionKeys.size(); i++) {
+ spec.put(partitionKeys.get(i), values[i]);
+ }
+ return spec;
+ }
+
+ @Nullable
public static LinkedHashMap extractPartitionSpecFromPathOnlyValue(
Path currPath, List partitionKeys) {
LinkedHashMap fullPartSpec = new LinkedHashMap<>();
String[] split = currPath.toString().split(Path.SEPARATOR);
+ if (split.length < partitionKeys.size()) {
+ return null;
+ }
for (int i = 0; i < partitionKeys.size(); i++) {
- fullPartSpec.put(partitionKeys.get(i), split[split.length - partitionKeys.size() + i]);
+ // Unescape the directory component so the extracted value is the RAW partition value,
+ // consistent with the key=value branch (extractPartitionSpecFromPath) and with the
+ // values the write path registers into a partition-managing catalog. Without this,
+ // directories containing escaped characters (e.g. a%3Ab) would round-trip to a
+ // different value than the one registered (a:b).
+ fullPartSpec.put(
+ partitionKeys.get(i),
+ unescapePathName(split[split.length - partitionKeys.size() + i]));
}
return fullPartSpec;
}
@@ -330,8 +463,9 @@ public static List, Path>> searchPartSpecAndP
part.getPath(), partitionKeys),
part.getPath()));
} else {
- LinkedHashMap spec = extractPartitionSpecFromPath(part.getPath());
- if (spec.size() != partitionKeys.size()) {
+ LinkedHashMap spec =
+ extractPartitionSpecFromPath(part.getPath(), partitionKeys);
+ if (spec == null) {
// illegal path, for example: /path/to/table/tmp/unknown, path without "="
continue;
}
@@ -360,8 +494,17 @@ private static FileStatus[] getFileStatusRecurse(
GenericRow values =
partitionType == null ? null : new GenericRow(partitionType.getFieldCount());
+ FileStatus fileStatus;
+ try {
+ fileStatus = fileIO.getFileStatus(path);
+ } catch (FileNotFoundException e) {
+ // A missing root simply means the table has no partitions yet.
+ return new FileStatus[0];
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to list files in " + path, e);
+ }
+
try {
- FileStatus fileStatus = fileIO.getFileStatus(path);
// Skip partition levels already fixed by the scan-path prefix.
int levelOffset = partitionKeys.size() - expectLevel;
listStatusRecursively(
@@ -378,9 +521,10 @@ private static FileStatus[] getFileStatusRecurse(
defaultPartValue,
levelOffset,
values);
- } catch (FileNotFoundException e) {
- return new FileStatus[0];
} catch (IOException e) {
+ // Never degrade a mid-scan failure into an empty listing: callers diff this result
+ // against partition metadata and an incomplete listing would deregister partitions
+ // that still exist.
throw new RuntimeException("Failed to list files in " + path, e);
}
@@ -412,7 +556,15 @@ private static void listStatusRecursively(
}
if (fileStatus.isDir()) {
- for (FileStatus stat : fileIO.listStatus(fileStatus.getPath())) {
+ FileStatus[] children;
+ try {
+ children = fileIO.listStatus(fileStatus.getPath());
+ } catch (FileNotFoundException e) {
+ // The directory vanished after the parent listed it: the partitions beneath it
+ // are gone, skipping just this subtree keeps the rest of the listing complete.
+ return;
+ }
+ for (FileStatus stat : children) {
int partitionKeyIndex = levelOffset + level;
String partitionKey = partitionKeys.get(partitionKeyIndex);
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
index 0961050f110d..3e91251ae812 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CachingCatalogTest.java
@@ -368,6 +368,32 @@ public void testCreatePartitionsWithIgnoreIfExistsInvalidatesPartitionCache() th
assertThat(catalog.listPartitions(identifier)).containsExactly(created);
}
+ @Test
+ public void testCreatePartitionsFailureInvalidatesPartitionCache() throws Exception {
+ Catalog wrapped = Mockito.mock(Catalog.class);
+ TestableCachingCatalog catalog =
+ new TestableCachingCatalog(wrapped, EXPIRATION_TTL, ticker);
+ Identifier identifier = new Identifier("db", "tbl");
+ Map spec = singletonMap("dt", "20260717");
+ Partition created = new Partition(spec, 0, 0, 0, 0, -1, false);
+ List stored = new ArrayList<>();
+ when(wrapped.listPartitions(identifier)).thenAnswer(ignored -> new ArrayList<>(stored));
+ RuntimeException responseLost = new RuntimeException("response lost");
+ Mockito.doAnswer(
+ ignored -> {
+ stored.add(created);
+ throw responseLost;
+ })
+ .when(wrapped)
+ .createPartitions(identifier, singletonList(spec));
+
+ assertThat(catalog.listPartitions(identifier)).isEmpty();
+ assertThatThrownBy(() -> catalog.createPartitions(identifier, singletonList(spec)))
+ .isSameAs(responseLost);
+
+ assertThat(catalog.listPartitions(identifier)).containsExactly(created);
+ }
+
@Test
public void testDeadlock() throws Exception {
Catalog underlyCatalog = this.catalog;
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java
new file mode 100644
index 000000000000..28aa706c31ac
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/CatalogUtilsTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.catalog;
+
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import static org.apache.paimon.CoreOptions.FILE_FORMAT;
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_IMPLEMENTATION;
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_SOURCE;
+import static org.apache.paimon.CoreOptions.PATH;
+import static org.apache.paimon.CoreOptions.TYPE;
+import static org.apache.paimon.TableType.FORMAT_TABLE;
+import static org.apache.paimon.catalog.CatalogUtils.loadTable;
+import static org.apache.paimon.catalog.CatalogUtils.validateCreateTable;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/** Tests for {@link CatalogUtils}. */
+class CatalogUtilsTest {
+
+ @Test
+ void testRejectEngineImplementationForCatalogManagedPartitionsOnCreate() {
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .option(TYPE.key(), FORMAT_TABLE.toString())
+ .option(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
+ .option(FORMAT_TABLE_IMPLEMENTATION.key(), "engine")
+ .build();
+
+ // The combination is rejected where catalog-managed partitions are actually served.
+ assertThatThrownBy(
+ () ->
+ CatalogUtils.validateCatalogManagedFormatTablePartitions(
+ Identifier.create("db", "t"), schema.options(), false))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(FORMAT_TABLE_PARTITION_SOURCE.key())
+ .hasMessageContaining(FORMAT_TABLE_IMPLEMENTATION.key());
+
+ // Elsewhere it is not this validation's business: the option is inert there.
+ validateCreateTable(schema, false);
+ }
+
+ @Test
+ void testEngineImplementationCannotHaveCatalogManagedPartitions() {
+ // Asking for catalog-managed partitions where they cannot be served is an error, not a
+ // table that silently reads its partitions from somewhere else.
+ assertThatThrownBy(() -> loadFormatTableRequestingCatalogManagedPartitions("engine", true))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(FORMAT_TABLE_PARTITION_SOURCE.key())
+ .hasMessageContaining(FORMAT_TABLE_IMPLEMENTATION.key());
+ }
+
+ @Test
+ void testOutsideRestCatalogPartitionsFromCatalogAreRejected() {
+ // The option names one source only, so a catalog that cannot serve it says so rather than
+ // reading the table's partitions from somewhere the option did not ask for.
+ assertThatThrownBy(() -> loadFormatTableRequestingCatalogManagedPartitions("paimon", false))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(FORMAT_TABLE_PARTITION_SOURCE.key())
+ .hasMessageContaining("REST catalog");
+ }
+
+ @Test
+ void testLoadCatalogManagedPartitionsFromRestCatalog() throws Exception {
+ Table table = loadFormatTableRequestingCatalogManagedPartitions("paimon", true);
+ assertThat(table).isInstanceOf(FormatTable.class);
+ FormatTable formatTable = (FormatTable) table;
+ assertThat(
+ new org.apache.paimon.CoreOptions(formatTable.options())
+ .formatTablePartitionsFromCatalog())
+ .isTrue();
+ assertThat(formatTable.partitionManager()).isNotNull();
+ }
+
+ private static Table loadFormatTableRequestingCatalogManagedPartitions(
+ String formatTableImplementation, boolean isRestCatalog) throws Exception {
+ Identifier identifier =
+ Identifier.create("catalog_partition_db", "catalog_partition_table");
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .option(TYPE.key(), FORMAT_TABLE.toString())
+ .option(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
+ .option(FORMAT_TABLE_IMPLEMENTATION.key(), formatTableImplementation)
+ .option(FILE_FORMAT.key(), "parquet")
+ .option(
+ PATH.key(),
+ "file:///tmp/catalog_partition_db.db/catalog_partition_table")
+ .build();
+ TableMetadata metadata =
+ new TableMetadata(
+ TableSchema.create(0, schema), false, "catalog-partition-table-id");
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.catalogLoader()).thenReturn(() -> catalog);
+
+ return loadTable(
+ catalog,
+ identifier,
+ path -> new LocalFileIO(),
+ path -> new LocalFileIO(),
+ ignored -> metadata,
+ null,
+ null,
+ null,
+ isRestCatalog);
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
index 04f67c018d1b..36677de510a9 100644
--- a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java
@@ -18,9 +18,12 @@
package org.apache.paimon.catalog;
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.TableType;
import org.apache.paimon.fs.Path;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
@@ -77,4 +80,33 @@ public void testAlterDatabase() throws Exception {
false))
.isInstanceOf(UnsupportedOperationException.class);
}
+
+ @Test
+ public void testPartitionsFromCatalogAreRejectedOutsideRestCatalog() throws Exception {
+ String database = "rest_partition_source_db";
+ Identifier identifier = Identifier.create(database, "rest_partition_source_table");
+ catalog.createDatabase(database, false);
+ // Write the schema file directly to simulate a format table carrying the REST-only
+ // partition source in a filesystem catalog.
+ Schema schema =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
+ .option(CoreOptions.FILE_FORMAT.key(), "parquet")
+ .option(CoreOptions.FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
+ .build();
+ Path tablePath =
+ ((FileSystemCatalog) DelegateCatalog.rootCatalog(catalog))
+ .getTableLocation(identifier);
+ new SchemaManager(fileIO, tablePath).createTable(schema);
+
+ // The option names one thing only, so a catalog that cannot serve it says so rather than
+ // reading the table's partitions from somewhere the option did not ask for.
+ assertThatThrownBy(() -> catalog.getTable(identifier))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(CoreOptions.FORMAT_TABLE_PARTITION_SOURCE.key())
+ .hasMessageContaining("REST catalog");
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
index b4af3194b72f..90508d5ead99 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
@@ -26,6 +26,8 @@
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.catalog.TableQueryAuthResult;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.predicate.FieldRef;
@@ -46,8 +48,11 @@
import org.apache.paimon.rest.exceptions.NotImplementedException;
import org.apache.paimon.rest.responses.ConfigResponse;
import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.format.FormatTablePartitionManager;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.InstantiationUtil;
import org.apache.paimon.utils.JsonSerdeUtil;
import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap;
@@ -216,8 +221,8 @@ void testCreateTableDefaultOptions() throws Exception {
}
@Test
- void testManagedFormatTablePagedPartitionListingDoesNotFallback() throws Exception {
- Identifier identifier = createManagedFormatTable();
+ void testCatalogManagedPagedPartitionListingDoesNotFallback() throws Exception {
+ Identifier identifier = createFormatTableWithCatalogManagedPartitions();
restCatalogServer.setPartitionListingSupported(false);
assertThatThrownBy(() -> restCatalog.listPartitionsPaged(identifier, null, null, null))
@@ -225,8 +230,8 @@ void testManagedFormatTablePagedPartitionListingDoesNotFallback() throws Excepti
}
@Test
- void testManagedFormatTablePartitionListingByNamesDoesNotFallback() throws Exception {
- Identifier identifier = createManagedFormatTable();
+ void testCatalogManagedPartitionListingByNamesDoesNotFallback() throws Exception {
+ Identifier identifier = createFormatTableWithCatalogManagedPartitions();
restCatalogServer.setPartitionListingSupported(false);
assertThatThrownBy(
@@ -239,22 +244,104 @@ void testManagedFormatTablePartitionListingByNamesDoesNotFallback() throws Excep
}
@Test
- void testManagedFormatTablePartitionListingDoesNotFallback() throws Exception {
- Identifier identifier = createManagedFormatTable();
+ void testCatalogManagedPartitionListingDoesNotFallback() throws Exception {
+ Identifier identifier = createFormatTableWithCatalogManagedPartitions();
restCatalogServer.setPartitionListingSupported(false);
assertThatThrownBy(() -> restCatalog.listPartitions(identifier))
.isInstanceOf(NotImplementedException.class);
}
- private Identifier createManagedFormatTable() throws Exception {
+ @Test
+ void testCatalogManagedPartitionListingReflectsCatalogMutationsImmediately() throws Exception {
+ Identifier identifier = createFormatTableWithCatalogManagedPartitions();
+ FormatTable table = (FormatTable) restCatalog.getTable(identifier);
+ FormatTablePartitionManager partitionManager = table.partitionManager();
+ assertThat(partitionManager).isNotNull();
+ assertThat(partitionManager.listPartitions(Collections.emptyMap())).isEmpty();
+ Map partition = Collections.singletonMap("dt", "20260717");
+
+ restCatalog.createPartitions(identifier, Collections.singletonList(partition));
+
+ // Listings are not cached, so a mutation through the catalog is visible to the next read.
+ assertThat(partitionManager.listPartitions(Collections.emptyMap()))
+ .extracting(org.apache.paimon.partition.Partition::spec)
+ .containsExactly(partition);
+
+ restCatalog.dropPartitions(identifier, Collections.singletonList(partition));
+
+ assertThat(partitionManager.listPartitions(Collections.emptyMap())).isEmpty();
+ }
+
+ @Test
+ void testPartitionManagerSurvivesSerialization() throws Exception {
+ Identifier identifier = createFormatTableWithCatalogManagedPartitions();
+ FormatTable table = (FormatTable) restCatalog.getTable(identifier);
+ Map partition = Collections.singletonMap("dt", "20260717");
+ restCatalog.createPartitions(identifier, Collections.singletonList(partition));
+
+ // A table travels to task processes; its partition catalog must rebuild its client there.
+ FormatTablePartitionManager roundTripped =
+ InstantiationUtil.clone(table.partitionManager());
+
+ assertThat(roundTripped.listPartitions(Collections.emptyMap()))
+ .extracting(org.apache.paimon.partition.Partition::spec)
+ .containsExactly(partition);
+ }
+
+ @Test
+ void testRejectCatalogManagedPartitionsOnExternalTableBeforeCreate() throws Exception {
+ Identifier identifier = Identifier.create("db1", "external_partitioned_format_table");
+ restCatalog.createDatabase(identifier.getDatabaseName(), true);
+ String externalPath = dataPath + "/external-partitioned-format-table";
+ Schema schema =
+ Schema.newBuilder()
+ .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
+ .option(CoreOptions.FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
+ .option(CoreOptions.FILE_FORMAT.key(), "parquet")
+ .option(CoreOptions.PATH.key(), externalPath)
+ .column("id", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .build();
+
+ assertThatThrownBy(() -> restCatalog.createTable(identifier, schema, false))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("internal table");
+ assertThat(restCatalog.listTables(identifier.getDatabaseName()))
+ .doesNotContain(identifier.getTableName());
+ assertThat(LocalFileIO.create().exists(new Path(externalPath))).isFalse();
+ }
+
+ @Test
+ void testRoundTrippedFormatTableReplacePassesClientValidation() throws Exception {
+ Identifier identifier = createFormatTableWithCatalogManagedPartitions();
+ FormatTable existing = (FormatTable) restCatalog.getTable(identifier);
+ Schema replacement =
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("dt", DataTypes.STRING())
+ .partitionKeys("dt")
+ .options(existing.options())
+ .build();
+ assertThat(replacement.options())
+ .containsEntry(CoreOptions.PATH.key(), existing.location());
+
+ // The mock service does not implement Format Table replacement. Reaching that response
+ // proves the REST client accepted the unchanged synthetic path from the loaded table.
+ assertThatThrownBy(() -> restCatalog.replaceTable(identifier, replacement, false))
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining("replaceTable does not support format tables");
+ }
+
+ private Identifier createFormatTableWithCatalogManagedPartitions() throws Exception {
Identifier identifier = Identifier.create("db1", "managed_partition_table");
restCatalog.createDatabase(identifier.getDatabaseName(), true);
restCatalog.createTable(
identifier,
Schema.newBuilder()
.option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
- .option(CoreOptions.METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(CoreOptions.FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
.option(CoreOptions.FILE_FORMAT.key(), "parquet")
.column("id", DataTypes.INT())
.column("dt", DataTypes.STRING())
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
index 3a0e1539eea1..b5cd544db109 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
@@ -923,9 +923,10 @@ private MockResponse loadSnapshot(Identifier identifier, String version) throws
private Optional checkTablePartitioned(Identifier identifier) {
if (tableMetadataStore.containsKey(identifier.getFullName())) {
TableMetadata tableMetadata = tableMetadataStore.get(identifier.getFullName());
+ CoreOptions tableOptions = CoreOptions.fromMap(tableMetadata.schema().options());
boolean partitioned =
- CoreOptions.fromMap(tableMetadata.schema().options())
- .partitionedTableInMetastore();
+ tableOptions.partitionedTableInMetastore()
+ || tableOptions.formatTablePartitionsFromCatalog();
if (!partitioned) {
return Optional.of(mockResponse(new ErrorResponse(null, null, "", 501), 501));
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
index 504dc6fab6ec..15f565af7fa0 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java
@@ -76,6 +76,7 @@
import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.FormatTable;
import org.apache.paimon.table.Instant;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.TableSnapshot;
@@ -135,6 +136,7 @@
import static java.util.Collections.singletonMap;
import static org.apache.paimon.CoreOptions.COMMIT_USER_PREFIX;
import static org.apache.paimon.CoreOptions.END_INPUT_CHECK_PARTITION_EXPIRE;
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_SOURCE;
import static org.apache.paimon.CoreOptions.METASTORE_PARTITIONED_TABLE;
import static org.apache.paimon.CoreOptions.METASTORE_TAG_TO_PARTITION;
import static org.apache.paimon.CoreOptions.PARTITION_EXPIRATION_STRATEGY;
@@ -1535,14 +1537,14 @@ void testListPartitionsFromFile() throws Exception {
}
@Test
- void testCreatePartitionsForManagedFormatTable() throws Exception {
- Identifier identifier = Identifier.create("format_partition_db", "managed_table");
+ void testCreatePartitionsForCatalogManagedFormatTablePartitions() throws Exception {
+ Identifier identifier = Identifier.create("format_partition_db", "catalog_partition_table");
catalog.createDatabase(identifier.getDatabaseName(), true);
catalog.createTable(
identifier,
Schema.newBuilder()
.option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
- .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
.option(CoreOptions.FILE_FORMAT.key(), "parquet")
.column("id", DataTypes.INT())
.column("dt", DataTypes.STRING())
@@ -1576,14 +1578,15 @@ void testCreatePartitionsForManagedFormatTable() throws Exception {
}
@Test
- void testDropPartitionsForManagedFormatTable() throws Exception {
- Identifier identifier = Identifier.create("format_partition_db", "managed_drop_table");
+ void testDropPartitionsForCatalogManagedFormatTablePartitions() throws Exception {
+ Identifier identifier =
+ Identifier.create("format_partition_db", "catalog_partition_drop_table");
catalog.createDatabase(identifier.getDatabaseName(), true);
catalog.createTable(
identifier,
Schema.newBuilder()
.option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
- .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
.option(CoreOptions.FILE_FORMAT.key(), "parquet")
.column("id", DataTypes.INT())
.column("dt", DataTypes.STRING())
@@ -1639,6 +1642,70 @@ void testDropPartitionsLoadsNonManagedTableOnce() throws Exception {
Mockito.verify(catalogSpy, Mockito.times(1)).getTable(identifier);
}
+ @Test
+ void testCatalogManagedPartitionCommitAndScanMatchesFileSystemMode() throws Exception {
+ Identifier identifier =
+ Identifier.create("format_partition_db", "catalog_partition_scan_table");
+ catalog.createDatabase(identifier.getDatabaseName(), true);
+ catalog.createTable(
+ identifier,
+ Schema.newBuilder()
+ .option(CoreOptions.TYPE.key(), TableType.FORMAT_TABLE.toString())
+ .option(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
+ .option(CoreOptions.FILE_FORMAT.key(), "csv")
+ .column("id", DataTypes.INT())
+ .column("year", DataTypes.INT())
+ .column("month", DataTypes.INT())
+ .partitionKeys("year", "month")
+ .build(),
+ false);
+
+ FormatTable managedTable = (FormatTable) catalog.getTable(identifier);
+ BatchWriteBuilder writeBuilder = managedTable.newBatchWriteBuilder();
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ write.write(GenericRow.of(1, 2024, 10));
+ write.write(GenericRow.of(2, 2025, 10));
+ write.write(GenericRow.of(3, 2025, 11));
+ commit.commit(write.prepareCommit());
+ }
+
+ List> registeredPartitions =
+ Arrays.asList(
+ ImmutableMap.of("year", "2024", "month", "10"),
+ ImmutableMap.of("year", "2025", "month", "10"),
+ ImmutableMap.of("year", "2025", "month", "11"));
+ assertThat(catalog.listPartitions(identifier).stream().map(Partition::spec))
+ .containsExactlyInAnyOrderElementsOf(registeredPartitions);
+ // The table carries the catalog partition metadata its scan reads from.
+ assertThat(managedTable.partitionManager()).isNotNull();
+
+ Map fileSystemOptions = new HashMap<>(managedTable.options());
+ fileSystemOptions.put(FORMAT_TABLE_PARTITION_SOURCE.key(), "filesystem");
+ FormatTable fileSystemTable =
+ FormatTable.builder()
+ .fileIO(managedTable.fileIO())
+ .identifier(Identifier.create("format_partition_db", "filesystem_scan"))
+ .rowType(managedTable.rowType())
+ .partitionKeys(managedTable.partitionKeys())
+ .location(managedTable.location())
+ .format(managedTable.format())
+ .options(fileSystemOptions)
+ .catalogContext(managedTable.catalogContext())
+ .build();
+
+ Map partitionFilter = singletonMap("year", "2025");
+ List readFromCatalog = read(managedTable, null, null, partitionFilter, null);
+ // Assert the rows themselves first: comparing the two reads alone would also pass if
+ // both stopped returning anything.
+ assertThat(readFromCatalog)
+ .extracting(row -> row.getInt(0) + "," + row.getInt(1) + "," + row.getInt(2))
+ .containsExactlyInAnyOrder("2,2025,10", "3,2025,11");
+ assertThat(readFromCatalog)
+ .containsExactlyInAnyOrderElementsOf(
+ read(fileSystemTable, null, null, partitionFilter, null));
+ }
+
@Test
void testListPartitions() throws Exception {
innerTestListPartitions(true);
@@ -1725,7 +1792,7 @@ public void testListPartitionsPaged() throws Exception {
catalog.createTable(
identifier,
Schema.newBuilder()
- .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
.option(METASTORE_TAG_TO_PARTITION.key(), "dt")
.column("col", DataTypes.INT())
.column("dt", DataTypes.STRING())
@@ -1841,7 +1908,7 @@ public void testListPartitionsPagedWithMultiLevel() throws Exception {
catalog.createTable(
identifier,
Schema.newBuilder()
- .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
.option(METASTORE_TAG_TO_PARTITION.key(), "dt")
.column("col", DataTypes.INT())
.column("dt", DataTypes.STRING())
@@ -1879,7 +1946,7 @@ public void testListPartitionsByNamesExceedsLimit() throws Exception {
catalog.createTable(
identifier,
Schema.newBuilder()
- .option(METASTORE_PARTITIONED_TABLE.key(), "true")
+ .option(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")
.option(METASTORE_TAG_TO_PARTITION.key(), "dt")
.column("col", DataTypes.INT())
.column("dt", DataTypes.STRING())
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java b/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java
new file mode 100644
index 000000000000..46ce9763eb77
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/FormatTableCompatibilityTest.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.table.format.FormatTablePartitionManager;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH;
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_SOURCE;
+import static org.apache.paimon.CoreOptions.READ_BATCH_SIZE;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Compatibility tests for the public {@link FormatTable} API. */
+class FormatTableCompatibilityTest {
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testCopyRejectsCatalogManagedPartitionStateChange(boolean withCatalogManagedPartitions) {
+ FormatTable table = formatTable(partitionSource(withCatalogManagedPartitions));
+
+ assertThatThrownBy(
+ () ->
+ table.copy(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_SOURCE.key(),
+ partitionSource(!withCatalogManagedPartitions))))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(FORMAT_TABLE_PARTITION_SOURCE.key());
+ }
+
+ @Test
+ void testCopyAllowsSemanticallyEquivalentCatalogManagedPartitionOption() {
+ FormatTable table = formatTable("REST");
+
+ FormatTable copied =
+ table.copy(Collections.singletonMap(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest"));
+
+ assertThat(copied).isNotSameAs(table);
+ assertThat(copied.options()).containsEntry(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest");
+ }
+
+ @Test
+ void testCopyRejectsChangingAbsentCatalogManagedPartitionDefault() {
+ FormatTable table = formatTable(Collections.emptyMap(), null);
+
+ assertThat(table.options()).doesNotContainKey(FORMAT_TABLE_PARTITION_SOURCE.key());
+ assertThatThrownBy(
+ () ->
+ table.copy(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_SOURCE.key(), "rest")))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(FORMAT_TABLE_PARTITION_SOURCE.key());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testCatalogManagedPartitionCopyRejectsPhysicalPartitionLayoutChange(
+ boolean onlyValueInPath) {
+ Map options = new LinkedHashMap<>();
+ options.put(FORMAT_TABLE_PARTITION_SOURCE.key(), "rest");
+ options.put(
+ FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(), Boolean.toString(onlyValueInPath));
+ // The layout is only fixed for a table that actually has catalog-managed partitions.
+ FormatTable table =
+ formatTable(
+ options,
+ FormatTablePartitionManager.create(
+ Identifier.create(
+ "catalog_partition_db", "catalog_partition_table"),
+ Collections.singletonList("dt"),
+ () -> null));
+
+ assertThatThrownBy(
+ () ->
+ table.copy(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.toString(!onlyValueInPath))))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testCopyPreservesPartitionModeAndCatalogProviderForUnrelatedOption(
+ boolean withCatalogManagedPartitions) {
+ Identifier identifier =
+ Identifier.create("catalog_partition_db", "catalog_partition_table");
+ FormatTablePartitionManager partitionManager =
+ FormatTablePartitionManager.create(
+ identifier, Collections.singletonList("dt"), () -> null);
+ Map options = new LinkedHashMap<>();
+ options.put(
+ FORMAT_TABLE_PARTITION_SOURCE.key(), partitionSource(withCatalogManagedPartitions));
+ FormatTable table = formatTable(options, partitionManager);
+
+ FormatTable copied = table.copy(Collections.singletonMap(READ_BATCH_SIZE.key(), "128"));
+
+ assertThat(copied).isNotSameAs(table);
+ assertThat(copied.options())
+ .containsEntry(READ_BATCH_SIZE.key(), "128")
+ .containsEntry(
+ FORMAT_TABLE_PARTITION_SOURCE.key(),
+ partitionSource(withCatalogManagedPartitions));
+ assertThat(
+ new org.apache.paimon.CoreOptions(copied.options())
+ .formatTablePartitionsFromCatalog())
+ .isEqualTo(withCatalogManagedPartitions);
+ assertThat(copied.partitionManager()).isSameAs(partitionManager);
+ assertThat(table.options())
+ .containsExactlyEntriesOf(options)
+ .doesNotContainKey(READ_BATCH_SIZE.key());
+ }
+
+ @Test
+ void testPartitionManagerIsOptionalForOtherImplementations() throws Exception {
+ // Implementations outside this repository do not have to know about catalog-managed
+ // partitions.
+ assertThat(FormatTable.class.getMethod("partitionManager").isDefault()).isTrue();
+ }
+
+ private static String partitionSource(boolean fromCatalog) {
+ return fromCatalog ? "rest" : "filesystem";
+ }
+
+ private static FormatTable formatTable(String partitionedInMetastore) {
+ return formatTable(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_SOURCE.key(), partitionedInMetastore),
+ null);
+ }
+
+ private static FormatTable formatTable(
+ Map options, FormatTablePartitionManager partitionManager) {
+ return FormatTable.builder()
+ .fileIO(LocalFileIO.create())
+ .identifier(Identifier.create("catalog_partition_db", "catalog_partition_table"))
+ .rowType(
+ RowType.builder()
+ .field("id", DataTypes.INT())
+ .field("dt", DataTypes.STRING())
+ .build())
+ .partitionKeys(Collections.singletonList("dt"))
+ .location("file:///warehouse/catalog_partition_table")
+ .format(FormatTable.Format.PARQUET)
+ .options(options)
+ .partitionManager(partitionManager)
+ .build();
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java
new file mode 100644
index 000000000000..d7a5d97e691a
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogFormatTablePartitionManagerTest.java
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.PagedList;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogLoader;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.utils.InstantiationUtil;
+
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for {@link CatalogFormatTablePartitionManager}: paging, prefix pushdown, per-request
+ * batching and catalog lifecycle are transport details this adapter owns, and none of them are
+ * visible through {@link FormatTablePartitionManager}.
+ */
+class CatalogFormatTablePartitionManagerTest {
+
+ private static final Identifier IDENTIFIER =
+ Identifier.create("catalog_partition_db", "catalog_partition_table");
+
+ private static final List PARTITION_KEYS = Arrays.asList("year", "month");
+
+ /** One catalog request never carries more than this many partitions. */
+ private static final int REQUEST_SIZE = 1000;
+
+ /** Handed to the serializable loader, which cannot capture a test instance field. */
+ private static Catalog staticCatalog;
+
+ // ------------------------------------------------------------------------
+ // paging
+ // ------------------------------------------------------------------------
+
+ @Test
+ void testListPartitionsFollowsEveryPage() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition first = partition("2025", "01");
+ Partition second = partition("2025", "02");
+ when(catalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenReturn(
+ new PagedList<>(Collections.singletonList(first), "page-2"),
+ new PagedList<>(Collections.singletonList(second), null));
+
+ List partitions =
+ partitionManager(catalog).listPartitions(Collections.emptyMap());
+
+ assertThat(partitions).containsExactly(first, second);
+ ArgumentCaptor pageTokens = ArgumentCaptor.forClass(String.class);
+ verify(catalog, times(2))
+ .listPartitionsPaged(eq(IDENTIFIER), eq(REQUEST_SIZE), pageTokens.capture(), any());
+ assertThat(pageTokens.getAllValues()).containsExactly(null, "page-2");
+ }
+
+ @Test
+ void testEmptyNextPageTokenEndsPaging() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition only = partition("2025", "01");
+ when(catalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenReturn(new PagedList<>(Collections.singletonList(only), ""));
+
+ List partitions =
+ partitionManager(catalog).listPartitions(Collections.emptyMap());
+
+ assertThat(partitions).containsExactly(only);
+ verify(catalog, times(1)).listPartitionsPaged(any(), any(), any(), any());
+ }
+
+ @Test
+ void testRepeatedPageTokenFailsFast() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ // a -> b -> a: following the cycle would list the same pages forever.
+ when(catalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenReturn(
+ new PagedList<>(Collections.singletonList(partition("2025", "01")), "a"),
+ new PagedList<>(Collections.singletonList(partition("2025", "02")), "b"),
+ new PagedList<>(Collections.singletonList(partition("2025", "03")), "a"));
+
+ FormatTablePartitionManager partitionManager = partitionManager(catalog);
+
+ assertThatThrownBy(() -> partitionManager.listPartitions(Collections.emptyMap()))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("repeated partition page token")
+ .hasMessageContaining("catalog_partition_db.catalog_partition_table");
+ }
+
+ // ------------------------------------------------------------------------
+ // prefix pushdown and client side filtering
+ // ------------------------------------------------------------------------
+
+ @Test
+ void testLeadingPrefixIsPushedDownAsPattern() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenReturn(new PagedList<>(Collections.emptyList(), null));
+
+ partitionManager(catalog).listPartitions(Collections.singletonMap("year", "2025"));
+
+ verify(catalog)
+ .listPartitionsPaged(eq(IDENTIFIER), eq(REQUEST_SIZE), isNull(), eq("year=2025/%"));
+ }
+
+ @Test
+ void testEmptyPrefixPushesDownNoPattern() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenReturn(new PagedList<>(Collections.emptyList(), null));
+
+ partitionManager(catalog).listPartitions(Collections.emptyMap());
+
+ verify(catalog).listPartitionsPaged(eq(IDENTIFIER), eq(REQUEST_SIZE), isNull(), isNull());
+ }
+
+ @Test
+ void testPrefixIsEnforcedOnCatalogsThatIgnoreThePattern() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition matching = partition("2025", "01");
+ Partition other = partition("2024", "01");
+ when(catalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenReturn(new PagedList<>(Arrays.asList(matching, other), null));
+
+ List partitions =
+ partitionManager(catalog).listPartitions(Collections.singletonMap("year", "2025"));
+
+ assertThat(partitions).containsExactly(matching);
+ }
+
+ @Test
+ void testNonLeadingPrefixIsRejected() {
+ Catalog catalog = mock(Catalog.class);
+ FormatTablePartitionManager partitionManager = partitionManager(catalog);
+ // "month" without "year" cannot be expressed as a partition path prefix.
+ Map prefix = Collections.singletonMap("month", "10");
+
+ assertThatThrownBy(() -> partitionManager.listPartitions(prefix))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("leading prefix");
+ }
+
+ // ------------------------------------------------------------------------
+ // batching
+ // ------------------------------------------------------------------------
+
+ @Test
+ void testIdempotentCreateIsSplitIntoRequests() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ List> specs = specs(2500);
+
+ partitionManager(catalog).createPartitions(specs, true);
+
+ List>> batches = capturedCreates(catalog, true, 3);
+ assertThat(batches).extracting(List::size).containsExactly(1000, 1000, 500);
+ assertThat(flatten(batches)).isEqualTo(specs);
+ }
+
+ @Test
+ void testStrictCreateStaysOneRequest() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ List> specs = specs(2500);
+
+ partitionManager(catalog).createPartitions(specs, false);
+
+ // Rejecting the whole batch when any partition exists only holds if it is never split.
+ List>> batches = capturedCreates(catalog, false, 1);
+ assertThat(batches.get(0)).isEqualTo(specs);
+ }
+
+ @Test
+ void testDropIsSplitIntoRequests() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ List> specs = specs(2500);
+
+ partitionManager(catalog).dropPartitions(specs);
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor>> captor = ArgumentCaptor.forClass(List.class);
+ verify(catalog, times(3)).dropPartitions(eq(IDENTIFIER), captor.capture());
+ assertThat(captor.getAllValues()).extracting(List::size).containsExactly(1000, 1000, 500);
+ assertThat(flatten(captor.getAllValues())).isEqualTo(specs);
+ }
+
+ @Test
+ void testListByNamesIsSplitAndUnioned() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ List> specs = specs(2500);
+ Partition first = partition("2025", "0000");
+ Partition second = partition("2025", "1000");
+ Partition third = partition("2025", "2000");
+ when(catalog.listPartitionsByNames(any(), anyList()))
+ .thenReturn(
+ Collections.singletonList(first),
+ Collections.singletonList(second),
+ Collections.singletonList(third));
+
+ List found = partitionManager(catalog).listPartitionsByNames(specs);
+
+ assertThat(found).containsExactly(first, second, third);
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor>> captor = ArgumentCaptor.forClass(List.class);
+ verify(catalog, times(3)).listPartitionsByNames(eq(IDENTIFIER), captor.capture());
+ assertThat(captor.getAllValues()).extracting(List::size).containsExactly(1000, 1000, 500);
+ assertThat(flatten(captor.getAllValues())).isEqualTo(specs);
+ }
+
+ @Test
+ void testEmptyInputTouchesNoCatalog() {
+ Catalog createCatalog = mock(Catalog.class);
+ partitionManager(createCatalog).createPartitions(Collections.emptyList(), true);
+ verifyNoInteractions(createCatalog);
+
+ Catalog dropCatalog = mock(Catalog.class);
+ partitionManager(dropCatalog).dropPartitions(Collections.emptyList());
+ verifyNoInteractions(dropCatalog);
+
+ Catalog listCatalog = mock(Catalog.class);
+ assertThat(partitionManager(listCatalog).listPartitionsByNames(Collections.emptyList()))
+ .isEmpty();
+ verifyNoInteractions(listCatalog);
+ }
+
+ // ------------------------------------------------------------------------
+ // catalog lifecycle
+ // ------------------------------------------------------------------------
+
+ @Test
+ void testEveryOperationClosesTheCatalog() throws Exception {
+ Catalog listCatalog = mock(Catalog.class);
+ when(listCatalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenReturn(new PagedList<>(Collections.emptyList(), null));
+ partitionManager(listCatalog).listPartitions(Collections.emptyMap());
+ verify(listCatalog).close();
+
+ Catalog byNamesCatalog = mock(Catalog.class);
+ when(byNamesCatalog.listPartitionsByNames(any(), anyList()))
+ .thenReturn(Collections.emptyList());
+ partitionManager(byNamesCatalog).listPartitionsByNames(specs(1));
+ verify(byNamesCatalog).close();
+
+ Catalog createCatalog = mock(Catalog.class);
+ partitionManager(createCatalog).createPartitions(specs(1), true);
+ verify(createCatalog).close();
+
+ Catalog dropCatalog = mock(Catalog.class);
+ partitionManager(dropCatalog).dropPartitions(specs(1));
+ verify(dropCatalog).close();
+ }
+
+ @Test
+ void testFailingOperationStillClosesTheCatalog() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenThrow(new RuntimeException("catalog unavailable"));
+ FormatTablePartitionManager partitionManager = partitionManager(catalog);
+
+ assertThatThrownBy(() -> partitionManager.listPartitions(Collections.emptyMap()))
+ .isInstanceOf(RuntimeException.class);
+
+ verify(catalog).close();
+ }
+
+ @Test
+ void testRejectedPrefixNeverLoadsACatalog() {
+ Catalog catalog = mock(Catalog.class);
+ FormatTablePartitionManager partitionManager = partitionManager(catalog);
+ Map prefix = Collections.singletonMap("month", "10");
+
+ assertThatThrownBy(() -> partitionManager.listPartitions(prefix))
+ .isInstanceOf(IllegalArgumentException.class);
+
+ verifyNoInteractions(catalog);
+ }
+
+ // ------------------------------------------------------------------------
+ // error translation
+ // ------------------------------------------------------------------------
+
+ @Test
+ void testCheckedExceptionIsWrappedWithTableContext() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Catalog.TableNotExistException notExist = new Catalog.TableNotExistException(IDENTIFIER);
+ when(catalog.listPartitionsPaged(any(), any(), any(), any())).thenThrow(notExist);
+ FormatTablePartitionManager partitionManager = partitionManager(catalog);
+
+ Throwable thrown =
+ catchThrowable(() -> partitionManager.listPartitions(Collections.emptyMap()));
+
+ assertThat(thrown)
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining("list partitions")
+ .hasMessageContaining("catalog_partition_db.catalog_partition_table");
+ assertThat(thrown.getCause()).isSameAs(notExist);
+ }
+
+ @Test
+ void testRuntimeExceptionIsRethrownUnchanged() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ RuntimeException failure = new IllegalStateException("catalog unavailable");
+ doThrow(failure).when(catalog).createPartitions(any(), anyList(), anyBoolean());
+ FormatTablePartitionManager partitionManager = partitionManager(catalog);
+
+ Throwable thrown = catchThrowable(() -> partitionManager.createPartitions(specs(1), true));
+
+ assertThat(thrown).isSameAs(failure);
+ }
+
+ // ------------------------------------------------------------------------
+ // serializability
+ // ------------------------------------------------------------------------
+
+ @Test
+ void testSurvivesJavaSerialization() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition only = partition("2025", "01");
+ when(catalog.listPartitionsPaged(any(), any(), any(), any()))
+ .thenReturn(new PagedList<>(Collections.singletonList(only), null));
+ staticCatalog = catalog;
+
+ FormatTablePartitionManager partitionManager =
+ FormatTablePartitionManager.create(IDENTIFIER, PARTITION_KEYS, new TestLoader());
+ FormatTablePartitionManager cloned = InstantiationUtil.clone(partitionManager);
+
+ assertThat(cloned.listPartitions(Collections.emptyMap())).containsExactly(only);
+ verify(catalog).close();
+ }
+
+ // ------------------------------------------------------------------------
+ // helpers
+ // ------------------------------------------------------------------------
+
+ private static FormatTablePartitionManager partitionManager(Catalog catalog) {
+ return FormatTablePartitionManager.create(IDENTIFIER, PARTITION_KEYS, () -> catalog);
+ }
+
+ private static List>> capturedCreates(
+ Catalog catalog, boolean ignoreIfExists, int expectedRequests) throws Exception {
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor>> captor = ArgumentCaptor.forClass(List.class);
+ verify(catalog, times(expectedRequests))
+ .createPartitions(eq(IDENTIFIER), captor.capture(), eq(ignoreIfExists));
+ return captor.getAllValues();
+ }
+
+ private static List> flatten(List>> batches) {
+ return batches.stream().flatMap(List::stream).collect(Collectors.toList());
+ }
+
+ private static Partition partition(String year, String month) {
+ return new Partition(spec(year, month), 0, 0, 0, 0, -1, false);
+ }
+
+ private static Map spec(String year, String month) {
+ LinkedHashMap spec = new LinkedHashMap<>();
+ spec.put("year", year);
+ spec.put("month", month);
+ return spec;
+ }
+
+ private static List> specs(int count) {
+ List> specs = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ specs.add(spec("2025", String.format("%04d", i)));
+ }
+ return specs;
+ }
+
+ /** A {@link CatalogLoader} that survives serialization by holding no state of its own. */
+ private static class TestLoader implements CatalogLoader {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public Catalog load() {
+ return staticCatalog;
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java
new file mode 100644
index 000000000000..da74ea32732f
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/CatalogManagedPartitionScanTest.java
@@ -0,0 +1,451 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.PagedList;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.fs.FileStatus;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.manifest.PartitionEntry;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.predicate.Predicate;
+import org.apache.paimon.predicate.PredicateBuilder;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.source.Split;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.PartitionPathUtils;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/** Tests for Format Table scans whose partitions are catalog-managed. */
+class CatalogManagedPartitionScanTest {
+
+ private static final Identifier IDENTIFIER =
+ Identifier.create("catalog_partition_db", "catalog_partition_table");
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ void testLeadingPatternResidualFilterAndUnregisteredDirectory() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), eq("year=2025/%")))
+ .thenReturn(
+ new PagedList<>(
+ Arrays.asList(partition("2025", "10"), partition("2025", "11")),
+ null));
+ TrackingLocalFileIO fileIO = new TrackingLocalFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ Path octoberFile = writeDataFile(fileIO, tablePath, "year=2025/month=10");
+ Path novemberFile = writeDataFile(fileIO, tablePath, "year=2025/month=11");
+ Path unregisteredFile = writeDataFile(fileIO, tablePath, "year=2025/month=12");
+ FormatTable table = createTable(fileIO, tablePath, partitionManager(catalog), false);
+ PredicateBuilder builder = new PredicateBuilder(table.partitionType());
+ Predicate predicate =
+ PredicateBuilder.and(builder.equal(0, 2025), builder.greaterThan(1, 10));
+ PartitionPredicate filter =
+ PartitionPredicate.fromPredicate(table.partitionType(), predicate);
+
+ FormatTableScan scan = new FormatTableScan(table, filter, null);
+ List plannedFiles = plannedFiles(scan.plan().splits());
+
+ assertThat(plannedFiles).containsExactly(novemberFile);
+ assertThat(plannedFiles).doesNotContain(octoberFile, unregisteredFile);
+ assertThat(fileIO.listedPaths).containsExactly(new Path(tablePath, "year=2025/month=11"));
+ }
+
+ @Test
+ void testListPartitionEntriesUsesCatalogVisibility() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition catalogPartition =
+ new Partition(partition("2025", "11").spec(), 11L, 22L, 3L, 44L, 5, false);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(catalogPartition), null));
+ TrackingLocalFileIO fileIO = new TrackingLocalFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ writeDataFile(fileIO, tablePath, "year=2025/month=11");
+ writeDataFile(fileIO, tablePath, "year=2025/month=12");
+ FormatTable table = createTable(fileIO, tablePath, partitionManager(catalog), false);
+
+ List entries =
+ new FormatTableScan(table, null, null).listPartitionEntries();
+
+ assertThat(entries)
+ .extracting(
+ entry -> entry.partition().getInt(0), entry -> entry.partition().getInt(1))
+ .containsExactly(org.assertj.core.groups.Tuple.tuple(2025, 11));
+ assertThat(entries.get(0).recordCount()).isEqualTo(11L);
+ assertThat(entries.get(0).fileSizeInBytes()).isEqualTo(22L);
+ assertThat(entries.get(0).fileCount()).isEqualTo(3L);
+ assertThat(entries.get(0).lastFileCreationTime()).isEqualTo(44L);
+ assertThat(entries.get(0).totalBuckets()).isEqualTo(5);
+ assertThat(fileIO.listedPaths).isEmpty();
+ assertThat(fileIO.statusListedPaths).isEmpty();
+ }
+
+ @Test
+ void testUnderscoreInPartitionNameRemainsLiteralPrefix() {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTable table =
+ createStringPartitionTable(
+ fileIO, tablePath, recordingCatalog(Collections.emptyList()));
+ Predicate predicate =
+ new PredicateBuilder(table.partitionType())
+ .equal(0, BinaryString.fromString("a_b"));
+ PartitionPredicate filter =
+ PartitionPredicate.fromPredicate(table.partitionType(), predicate);
+
+ List plannedFiles =
+ plannedFiles(new FormatTableScan(table, filter, null).plan().splits());
+
+ // '_' has no special meaning in the prefix contract; it must not widen the match.
+ assertThat(plannedFiles).isEmpty();
+ assertThat(requestedPrefixes).containsExactly(Collections.singletonMap("year", "a_b"));
+ }
+
+ @Test
+ void testValueOnlyPartitionPath() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(
+ new PagedList<>(Collections.singletonList(partition("2025", "11")), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path dataFile = writeDataFile(fileIO, tablePath, "2025/11");
+ FormatTable table = createTable(fileIO, tablePath, partitionManager(catalog), true);
+
+ List plannedFiles =
+ plannedFiles(new FormatTableScan(table, null, null).plan().splits());
+
+ assertThat(plannedFiles).containsExactly(dataFile);
+ }
+
+ @Test
+ void testDuplicateCatalogPartitionPlansSplitsOnce() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(
+ new PagedList<>(
+ Arrays.asList(partition("2025", "11"), partition("2025", "11")),
+ null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path dataFile = writeDataFile(fileIO, tablePath, "year=2025/month=11");
+ FormatTable table = createTable(fileIO, tablePath, partitionManager(catalog), false);
+
+ List plannedFiles =
+ plannedFiles(new FormatTableScan(table, null, null).plan().splits());
+
+ assertThat(plannedFiles).containsExactly(dataFile);
+ }
+
+ @Test
+ void testWhitespacePartitionValueIsVisible() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition whitespacePartition = partition(" ", "11");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(whitespacePartition), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path dataFile =
+ writeDataFile(
+ fileIO,
+ tablePath,
+ PartitionPathUtils.generatePartitionPath(
+ new LinkedHashMap<>(whitespacePartition.spec())));
+ FormatTable table =
+ createStringPartitionTable(fileIO, tablePath, partitionManager(catalog));
+
+ List plannedFiles =
+ plannedFiles(new FormatTableScan(table, null, null).plan().splits());
+
+ assertThat(plannedFiles).containsExactly(dataFile);
+ }
+
+ @Test
+ void testEmptyPartitionValueReportsCorruptMetadata() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition emptyValuePartition = partition("2025", "");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(emptyValuePartition), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTable table =
+ createStringPartitionTable(fileIO, tablePath, partitionManager(catalog));
+
+ assertThatThrownBy(() -> new FormatTableScan(table, null, null).plan().splits())
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("corrupt partition metadata")
+ .hasMessageContaining(IDENTIFIER.getFullName());
+ }
+
+ @Test
+ void testCatalogFailureDoesNotFallBackToFileSystem() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ RuntimeException catalogFailure =
+ new RuntimeException("Catalog partition listing unavailable");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenThrow(catalogFailure);
+ TrackingLocalFileIO fileIO = new TrackingLocalFileIO();
+ Path tablePath = new Path(tempDir.toUri());
+ writeDataFile(fileIO, tablePath, "year=2025/month=11");
+ FormatTable table = createTable(fileIO, tablePath, partitionManager(catalog), false);
+
+ assertThatThrownBy(() -> new FormatTableScan(table, null, null).plan().splits())
+ .isSameAs(catalogFailure);
+ assertThat(fileIO.listedPaths).isEmpty();
+ }
+
+ @Test
+ @DisplayName(
+ "treats a registered partition with a missing directory as an empty partition "
+ + "(requires real object-store validation)")
+ void testMissingCatalogRegisteredPartitionReadsAsEmpty() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(
+ new PagedList<>(Collections.singletonList(partition("2025", "11")), null));
+ Path tablePath = new Path(tempDir.toUri());
+ Path missingPath = new Path(tablePath, "year=2025/month=11");
+ LocalFileIO fileIO =
+ new LocalFileIO() {
+ @Override
+ public FileStatus[] listFiles(Path path, boolean recursive) throws IOException {
+ assertThat(path).isEqualTo(missingPath);
+ throw new FileNotFoundException(path.toString());
+ }
+ };
+ FormatTable table = createTable(fileIO, tablePath, partitionManager(catalog), false);
+
+ // A registered partition whose directory is missing (e.g. ADD PARTITION before the first
+ // INSERT) must not fail the whole scan; it reads as an empty partition, matching Hive.
+ List plannedFiles =
+ plannedFiles(new FormatTableScan(table, null, null).plan().splits());
+ assertThat(plannedFiles).isEmpty();
+ }
+
+ @Test
+ void testTraversalPartitionValueReportsCorruptMetadata() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition traversalPartition = partition("2025", "..");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(traversalPartition), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ FormatTable table =
+ createStringPartitionTable(fileIO, tablePath, partitionManager(catalog), true);
+
+ assertThatThrownBy(() -> new FormatTableScan(table, null, null).plan().splits())
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("corrupt partition metadata")
+ .hasMessageContaining(IDENTIFIER.getFullName());
+ }
+
+ @Test
+ void testDotDotValueInKeyValueLayoutRemainsVisible() throws Exception {
+ Catalog catalog = mock(Catalog.class);
+ Partition partition = partition("2025", "..");
+ when(catalog.listPartitionsPaged(eq(IDENTIFIER), eq(1000), isNull(), isNull()))
+ .thenReturn(new PagedList<>(Collections.singletonList(partition), null));
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path dataFile = writeDataFile(fileIO, tablePath, "year=2025/month=..");
+ FormatTable table =
+ createStringPartitionTable(fileIO, tablePath, partitionManager(catalog));
+
+ List plannedFiles =
+ plannedFiles(new FormatTableScan(table, null, null).plan().splits());
+
+ assertThat(plannedFiles).containsExactly(dataFile);
+ }
+
+ private final List> requestedPrefixes = new ArrayList<>();
+
+ private FormatTablePartitionManager partitionManager(Catalog catalog) {
+ return FormatTablePartitionManager.create(
+ IDENTIFIER, Arrays.asList("year", "month"), () -> catalog);
+ }
+
+ /** Records the prefix the scan pushes down and answers from a fixed partition list. */
+ private FormatTablePartitionManager recordingCatalog(List partitions) {
+ List> prefixes = requestedPrefixes;
+ return new FormatTablePartitionManager() {
+ @Override
+ public List listPartitions(Map prefix) {
+ prefixes.add(new LinkedHashMap<>(prefix));
+ List matching = new ArrayList<>();
+ for (Partition partition : partitions) {
+ boolean matches = true;
+ for (Map.Entry entry : prefix.entrySet()) {
+ if (!entry.getValue().equals(partition.spec().get(entry.getKey()))) {
+ matches = false;
+ break;
+ }
+ }
+ if (matches) {
+ matching.add(partition);
+ }
+ }
+ return matching;
+ }
+
+ @Override
+ public List listPartitionsByNames(List> partitions) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void createPartitions(
+ List> partitions, boolean ignoreIfExists) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropPartitions(List> partitions) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ private FormatTable createTable(
+ LocalFileIO fileIO,
+ Path tablePath,
+ FormatTablePartitionManager partitionManager,
+ boolean valueOnlyPath) {
+ RowType rowType =
+ RowType.builder()
+ .field("year", DataTypes.INT())
+ .field("month", DataTypes.INT())
+ .field("id", DataTypes.INT())
+ .build();
+ return FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(IDENTIFIER)
+ .rowType(rowType)
+ .partitionKeys(Arrays.asList("year", "month"))
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.toString(valueOnlyPath)))
+ .partitionManager(partitionManager)
+ .build();
+ }
+
+ private FormatTable createStringPartitionTable(
+ LocalFileIO fileIO, Path tablePath, FormatTablePartitionManager partitionManager) {
+ return createStringPartitionTable(fileIO, tablePath, partitionManager, false);
+ }
+
+ private FormatTable createStringPartitionTable(
+ LocalFileIO fileIO,
+ Path tablePath,
+ FormatTablePartitionManager partitionManager,
+ boolean valueOnlyPath) {
+ RowType rowType =
+ RowType.builder()
+ .field("year", DataTypes.STRING())
+ .field("month", DataTypes.STRING())
+ .field("id", DataTypes.INT())
+ .build();
+ return FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(IDENTIFIER)
+ .rowType(rowType)
+ .partitionKeys(Arrays.asList("year", "month"))
+ .location(tablePath.toString())
+ .format(FormatTable.Format.CSV)
+ .options(
+ Collections.singletonMap(
+ FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.toString(valueOnlyPath)))
+ .partitionManager(partitionManager)
+ .build();
+ }
+
+ private Path writeDataFile(LocalFileIO fileIO, Path tablePath, String partitionPath)
+ throws IOException {
+ Path file = new Path(new Path(tablePath, partitionPath), "data.csv");
+ fileIO.mkdirs(file.getParent());
+ try (PositionOutputStream out = fileIO.newOutputStream(file, false)) {
+ out.write(1);
+ }
+ return file;
+ }
+
+ private static Partition partition(String year, String month) {
+ Map spec = new LinkedHashMap<>();
+ spec.put("year", year);
+ spec.put("month", month);
+ return new Partition(spec, 0, 0, 0, 0, -1, false);
+ }
+
+ private static List plannedFiles(List splits) {
+ return splits.stream()
+ .map(FormatDataSplit.class::cast)
+ .flatMap(split -> split.files().stream())
+ .map(FormatDataSplit.FileMeta::filePath)
+ .collect(Collectors.toList());
+ }
+
+ private static class TrackingLocalFileIO extends LocalFileIO {
+
+ private final List listedPaths = new ArrayList<>();
+ private final List statusListedPaths = new ArrayList<>();
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ statusListedPaths.add(path);
+ return super.listStatus(path);
+ }
+
+ @Override
+ public FileStatus[] listFiles(Path path, boolean recursive) throws IOException {
+ listedPaths.add(path);
+ return super.listFiles(path, recursive);
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java
new file mode 100644
index 000000000000..c7f8c8dac1af
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/format/FormatTableCommitTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.format;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.RenamingTwoPhaseOutputStream;
+import org.apache.paimon.fs.TwoPhaseOutputStream;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.utils.PartitionPathUtils;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.entry;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+/** Tests for {@link FormatTableCommit}. */
+class FormatTableCommitTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ void testPartitionRegistrationFailureDiscardsTheFilesItWrote() throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ Path targetPath = new Path(tablePath, "year=2025/month=10/data-1.csv");
+ RenamingTwoPhaseOutputStream outputStream =
+ new RenamingTwoPhaseOutputStream(fileIO, targetPath, false);
+ outputStream.write(1);
+ TwoPhaseOutputStream.Committer committer = outputStream.closeForCommit();
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ RuntimeException registrationFailure =
+ new RuntimeException("Catalog partition registration unavailable");
+ doThrow(registrationFailure).when(partitionManager).createPartitions(anyList(), eq(true));
+
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Arrays.asList("year", "month"),
+ fileIO,
+ false,
+ false,
+ Identifier.create("catalog_partition_db", "catalog_partition_table"),
+ null,
+ null,
+ null,
+ partitionManager);
+ CommitMessage message = new TwoPhaseCommitMessage(committer);
+
+ assertThatThrownBy(() -> commit.commit(Collections.singletonList(message)))
+ .isInstanceOf(RuntimeException.class)
+ .hasRootCauseMessage("Catalog partition registration unavailable");
+
+ // A failed write leaves nothing behind, whichever step failed: rerunning it converges,
+ // and an idempotent registration makes a partition that was registered anyway harmless.
+ assertThat(fileIO.exists(targetPath)).isFalse();
+ verify(partitionManager).createPartitions(anyList(), eq(true));
+ }
+
+ @Test
+ void testFileCommitFailureStillDiscardsUncommittedFiles() throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path tablePath = new Path(tempDir.toUri());
+ TwoPhaseOutputStream.Committer committer = mock(TwoPhaseOutputStream.Committer.class);
+ doThrow(new IOException("data commit failed")).when(committer).commit(fileIO);
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Arrays.asList("year", "month"),
+ fileIO,
+ false,
+ false,
+ Identifier.create("catalog_partition_db", "catalog_partition_table"),
+ null,
+ null,
+ null,
+ partitionManager);
+ CommitMessage message = new TwoPhaseCommitMessage(committer);
+
+ assertThatThrownBy(() -> commit.commit(Collections.singletonList(message)))
+ .isInstanceOf(RuntimeException.class)
+ .hasRootCauseMessage("data commit failed");
+
+ verify(committer).discard(fileIO);
+ verify(partitionManager, never()).createPartitions(anyList(), eq(true));
+ }
+
+ @Test
+ void testRegistersRawPartitionValuesForEscapedPath() throws Exception {
+ Path tablePath = new Path(tempDir.toUri());
+ LinkedHashMap rawSpec = new LinkedHashMap<>();
+ rawSpec.put("year", "2025");
+ rawSpec.put("month", "a b:c");
+ // The writer escapes partition values when building the directory layout.
+ String partitionDir = PartitionPathUtils.generatePartitionPathUtil(rawSpec, false);
+ assertThat(partitionDir).isEqualTo("year=2025/month=a b%3Ac/");
+
+ FormatTablePartitionManager partitionManager =
+ commitPartitionedFile(tablePath, false, partitionDir);
+
+ // The catalog must receive RAW values; readers re-escape them when probing directories.
+ assertThat(registeredSpec(partitionManager))
+ .containsExactly(entry("year", "2025"), entry("month", "a b:c"));
+ }
+
+ @Test
+ void testForeignKeyValueSegmentsInLocationDoNotLeakIntoSpec() throws Exception {
+ Path tablePath = new Path(new Path(tempDir.toUri()), "env=prod/warehouse/tbl");
+
+ FormatTablePartitionManager partitionManager =
+ commitPartitionedFile(tablePath, false, "year=2025/month=10");
+
+ assertThat(registeredSpec(partitionManager))
+ .containsExactly(entry("year", "2025"), entry("month", "10"));
+ }
+
+ @Test
+ void testValueOnlyPathUnderForeignKeyValueSegmentRegistersRawValues() throws Exception {
+ Path tablePath = new Path(new Path(tempDir.toUri()), "env=prod/warehouse/tbl");
+ LinkedHashMap rawSpec = new LinkedHashMap<>();
+ rawSpec.put("year", "2025");
+ rawSpec.put("month", "a:b");
+ String partitionDir = PartitionPathUtils.generatePartitionPathUtil(rawSpec, true);
+ assertThat(partitionDir).isEqualTo("2025/a%3Ab/");
+
+ FormatTablePartitionManager partitionManager =
+ commitPartitionedFile(tablePath, true, partitionDir);
+
+ assertThat(registeredSpec(partitionManager))
+ .containsExactly(entry("year", "2025"), entry("month", "a:b"));
+ }
+
+ @Test
+ void testPathNotMatchingThePartitionKeysFails() {
+ Path tablePath = new Path(tempDir.toUri());
+
+ // The message names the path and the declared keys, which is what tells a reader that
+ // 'day' is not where 'month' was expected.
+ assertThatThrownBy(() -> commitPartitionedFile(tablePath, false, "year=2025/day=10"))
+ .isInstanceOf(RuntimeException.class)
+ .hasRootCauseInstanceOf(IllegalArgumentException.class)
+ .rootCause()
+ .hasMessageContaining("year=2025/day=10")
+ .hasMessageContaining("catalog_partition_db.catalog_partition_table")
+ .hasMessageContaining("[year, month]");
+ }
+
+ @Test
+ void testValueOnlyStaticPartitionCannotEscapeTableLocation() throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path parentPath = new Path(tempDir.toUri());
+ Path tablePath = new Path(parentPath, "table");
+ Path siblingPath = new Path(parentPath, "keep");
+ fileIO.mkdirs(tablePath);
+ fileIO.mkdirs(siblingPath);
+ Map staticPartition = Collections.singletonMap("year", "..");
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tablePath.toString(),
+ Collections.singletonList("year"),
+ fileIO,
+ true,
+ true,
+ Identifier.create("catalog_partition_db", "catalog_partition_table"),
+ staticPartition,
+ null,
+ null,
+ null);
+
+ assertThatThrownBy(() -> commit.commit(Collections.emptyList()))
+ .isInstanceOf(RuntimeException.class)
+ .hasRootCauseInstanceOf(IllegalArgumentException.class)
+ .hasRootCauseMessage(
+ "Partition value '..' cannot be used as a partition path component.");
+ assertThat(fileIO.exists(tablePath)).isTrue();
+ assertThat(fileIO.exists(siblingPath)).isTrue();
+ }
+
+ private FormatTablePartitionManager commitPartitionedFile(
+ Path tableLocation, boolean onlyValueInPath, String partitionDir) throws Exception {
+ LocalFileIO fileIO = LocalFileIO.create();
+ Path targetPath = new Path(new Path(tableLocation, partitionDir), "data-1.csv");
+ RenamingTwoPhaseOutputStream outputStream =
+ new RenamingTwoPhaseOutputStream(fileIO, targetPath, false);
+ outputStream.write(1);
+ TwoPhaseOutputStream.Committer committer = outputStream.closeForCommit();
+ FormatTablePartitionManager partitionManager = mock(FormatTablePartitionManager.class);
+ FormatTableCommit commit =
+ new FormatTableCommit(
+ tableLocation.toString(),
+ Arrays.asList("year", "month"),
+ fileIO,
+ onlyValueInPath,
+ false,
+ Identifier.create("catalog_partition_db", "catalog_partition_table"),
+ null,
+ null,
+ null,
+ partitionManager);
+ commit.commit(Collections.singletonList(new TwoPhaseCommitMessage(committer)));
+ return partitionManager;
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private static Map registeredSpec(
+ FormatTablePartitionManager partitionManager) {
+ ArgumentCaptor>> captor =
+ ArgumentCaptor.forClass((Class) List.class);
+ verify(partitionManager).createPartitions(captor.capture(), eq(true));
+ assertThat(captor.getValue()).hasSize(1);
+ return captor.getValue().get(0);
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
index a21b168f54ea..6b2952778003 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionPathUtilsTest.java
@@ -19,17 +19,25 @@
package org.apache.paimon.utils;
import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.fs.Path;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.Arrays;
+import java.util.LinkedHashMap;
import static org.apache.paimon.utils.PartitionPathUtils.mightMatch;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assertions.entry;
-/** Tests for {@link PartitionPathUtils#mightMatch}. */
+/** Tests for {@link PartitionPathUtils}. */
class PartitionPathUtilsTest {
private final RowType partitionType =
@@ -39,6 +47,46 @@ class PartitionPathUtilsTest {
.build();
private final PredicateBuilder builder = new PredicateBuilder(partitionType);
+ @ParameterizedTest
+ @ValueSource(strings = {".", ".."})
+ void testValueOnlyDotSegmentIsRejected(String rawValue) {
+ LinkedHashMap partitionSpec = new LinkedHashMap<>();
+ partitionSpec.put("pt", rawValue);
+
+ assertThatThrownBy(() -> PartitionPathUtils.generatePartitionPathUtil(partitionSpec, true))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(rawValue);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {".", ".."})
+ void testKeyedDotValueKeepsCompatibleSafeLayout(String rawValue) {
+ LinkedHashMap partitionSpec = new LinkedHashMap<>();
+ partitionSpec.put("pt", rawValue);
+
+ String partitionPath = PartitionPathUtils.generatePartitionPathUtil(partitionSpec, false);
+ Path resolvedPath = new Path(new Path("file:///warehouse/table"), partitionPath);
+
+ assertThat(partitionPath).isEqualTo("pt=" + rawValue + Path.SEPARATOR);
+ assertThat(PartitionPathUtils.extractPartitionSpecFromPath(resolvedPath))
+ .containsExactly(entry("pt", rawValue));
+ }
+
+ @Test
+ void testTrailingPartitionExtractionStopsAtTableBoundary() {
+ Path partitionPath = new Path("file:///warehouse/env=prod/table/dt=20260718/hh=10");
+
+ assertThat(
+ PartitionPathUtils.extractPartitionSpecFromPath(
+ partitionPath, Arrays.asList("dt", "hh")))
+ .containsExactly(entry("dt", "20260718"), entry("hh", "10"));
+ assertThat(
+ PartitionPathUtils.extractPartitionSpecFromPath(
+ new Path("file:///warehouse/dt=parent/table/wrong=20260718/hh=10"),
+ Arrays.asList("dt", "hh")))
+ .isNull();
+ }
+
@Test
void testNullPredicate() {
assertThat(mightMatch(null, 0, 0, GenericRow.of(2024, 5))).isTrue();
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/format/FormatTablePartitionRepair.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/format/FormatTablePartitionRepair.java
new file mode 100644
index 000000000000..4aa99b812c14
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/format/FormatTablePartitionRepair.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.format.FormatTablePartitionManager;
+import org.apache.paimon.utils.Pair;
+import org.apache.paimon.utils.PartitionPathUtils;
+import org.apache.paimon.utils.Preconditions;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Repairs a Format Table with catalog-managed partitions by reconciling its partition directories
+ * with its catalog registration, backing {@code MSCK REPAIR TABLE [{ADD|DROP|SYNC} PARTITIONS]}.
+ * Only partition metadata is changed, data files are never touched.
+ */
+public class FormatTablePartitionRepair {
+
+ private FormatTablePartitionRepair() {}
+
+ /**
+ * Repair the partition metadata of a Format Table with catalog-managed partitions. The flag
+ * pair follows Spark's {@code RepairTable} plan (plain MSCK means ADD). Returns the number of
+ * applied actions.
+ */
+ public static int repair(
+ PaimonFormatTable sparkTable, boolean addPartitions, boolean dropPartitions) {
+ Preconditions.checkArgument(
+ addPartitions || dropPartitions,
+ "MSCK REPAIR TABLE must enable ADD and/or DROP partitions");
+ FormatTable formatTable = sparkTable.table();
+ FormatTablePartitionManager partitionManager = formatTable.partitionManager();
+ Preconditions.checkArgument(
+ partitionManager != null,
+ "%s does not have catalog-managed partitions",
+ formatTable.fullName());
+
+ return apply(
+ partitionManager,
+ listFilesystemPartitionSpecs(formatTable),
+ formatTable.partitionKeys(),
+ addPartitions,
+ dropPartitions);
+ }
+
+ private static List> listFilesystemPartitionSpecs(FormatTable formatTable) {
+ // Discover partitions from the raw directory names rather than through the table scan:
+ // the scan casts each value to its column type and back (e.g. month=01 -> 1), producing
+ // specs that can no longer round-trip to the real directory. The write path registers the
+ // raw directory value, so a repair must diff against the same raw values to avoid
+ // spuriously adding/dropping partition metadata.
+ boolean onlyValueInPath =
+ new CoreOptions(formatTable.options()).formatTablePartitionOnlyValueInPath();
+ List, Path>> found =
+ PartitionPathUtils.searchPartSpecAndPaths(
+ formatTable.fileIO(),
+ new Path(formatTable.location()),
+ formatTable.partitionKeys().size(),
+ formatTable.partitionKeys(),
+ onlyValueInPath);
+ List> specs = new ArrayList<>(found.size());
+ for (Pair, Path> pair : found) {
+ PartitionPathUtils.validatePartitionSpecForPath(pair.getKey(), onlyValueInPath);
+ specs.add(pair.getKey());
+ }
+ return specs;
+ }
+
+ /**
+ * Diff the filesystem partition set against the catalog registration set and apply the
+ * requested actions. ADD registers "directory exists but unregistered"; DROP is metadata-only
+ * cleanup of "registered but directory missing". Scan-completeness guard: the filesystem
+ * listing that feeds {@code filesystemPartitions} ({@link
+ * PartitionPathUtils#searchPartSpecAndPaths}) fails on any mid-scan LIST error instead of
+ * returning a truncated set, so a DROP diff can only be produced from a complete listing and a
+ * transient failure never deregisters partitions that still exist.
+ */
+ static int apply(
+ FormatTablePartitionManager partitionManager,
+ List> filesystemPartitions,
+ List partitionKeys,
+ boolean addPartitions,
+ boolean dropPartitions) {
+ Set> registeredPartitions = new HashSet<>();
+ for (Partition partition : partitionManager.listPartitions(Collections.emptyMap())) {
+ registeredPartitions.add(partition.spec());
+ }
+
+ Set> filesystemSet = new HashSet<>(filesystemPartitions);
+
+ List> addDiff = new ArrayList<>();
+ if (addPartitions) {
+ for (Map partition : filesystemPartitions) {
+ if (!registeredPartitions.contains(partition)) {
+ addDiff.add(partition);
+ }
+ }
+ sortByCanonicalPath(addDiff, partitionKeys);
+ }
+ List> dropDiff = new ArrayList<>();
+ if (dropPartitions) {
+ for (Map partition : registeredPartitions) {
+ if (!filesystemSet.contains(partition)) {
+ dropDiff.add(partition);
+ }
+ }
+ sortByCanonicalPath(dropDiff, partitionKeys);
+ }
+
+ // A first repair of a pre-existing table can discover far more partitions than any regular
+ // write. Splitting such a diff into per-request batches is the partition catalog's job;
+ // registration is an idempotent upsert and unregistration ignores missing partitions, so a
+ // mid-way failure leaves a state a rerun converges from.
+ if (!addDiff.isEmpty()) {
+ partitionManager.createPartitions(addDiff, true);
+ }
+ if (!dropDiff.isEmpty()) {
+ partitionManager.dropPartitions(dropDiff);
+ }
+ return addDiff.size() + dropDiff.size();
+ }
+
+ /** Sort partitions by their canonical path for a stable, deterministic apply order. */
+ private static void sortByCanonicalPath(
+ List> partitions, List partitionKeys) {
+ partitions.sort(Comparator.comparing(partition -> canonicalPath(partition, partitionKeys)));
+ }
+
+ private static String canonicalPath(Map partition, List partitionKeys) {
+ LinkedHashMap orderedSpec = new LinkedHashMap<>();
+ for (String key : partitionKeys) {
+ if (partition.containsKey(key)) {
+ orderedSpec.put(key, partition.get(key));
+ }
+ }
+ return PartitionPathUtils.generatePartitionPath(orderedSpec);
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
index cd5955b0ff39..42234eea8779 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala
@@ -19,11 +19,13 @@
package org.apache.paimon.spark
import org.apache.paimon.CoreOptions
+import org.apache.paimon.fs.Path
import org.apache.paimon.partition.PartitionStatistics
-import org.apache.paimon.table.{FileStoreTable, Table}
+import org.apache.paimon.table.{FileStoreTable, FormatTable, Table}
+import org.apache.paimon.table.format.FormatTablePartitionManager
import org.apache.paimon.table.source.ScanMode
import org.apache.paimon.types.RowType
-import org.apache.paimon.utils.{InternalRowPartitionComputer, TypeUtils}
+import org.apache.paimon.utils.{InternalRowPartitionComputer, PartitionPathUtils, TypeUtils}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
@@ -32,61 +34,46 @@ import org.apache.spark.sql.catalyst.util.CharVarcharUtils
import org.apache.spark.sql.connector.catalog.SupportsAtomicPartitionManagement
import org.apache.spark.sql.types.StructType
-import java.util.{Map => JMap, Objects}
+import java.util.{Collections, Map => JMap, Objects}
import scala.collection.JavaConverters._
+import scala.collection.mutable.{ArrayBuffer, HashSet}
trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with Logging {
val table: Table
+ def partitionManager: FormatTablePartitionManager = null
+
lazy val partitionRowType: RowType = TypeUtils.project(table.rowType, table.partitionKeys)
override lazy val partitionSchema: StructType = SparkTypeUtils.fromPaimonRowType(partitionRowType)
private def toPaimonPartitions(rows: Array[InternalRow]): Array[java.util.Map[String, String]] = {
- table match {
- case fileStoreTable: FileStoreTable =>
- val partitionKeys = table.partitionKeys().asScala.toSeq
- val partitionDefaultName = fileStoreTable.coreOptions().partitionDefaultName()
- val legacyPartitionName = CoreOptions.fromMap(table.options()).legacyPartitionName
-
- rows.map {
- r =>
- val partitionFieldCount = r.numFields
- require(
- partitionFieldCount <= partitionKeys.length,
- s"Partition values length $partitionFieldCount exceeds partition keys " +
- s"${partitionKeys.mkString("[", ", ", "]")}."
- )
- val partitionNames = partitionKeys.take(partitionFieldCount)
- val currentPartitionRowType =
- if (partitionFieldCount == partitionRowType.getFieldCount) {
- partitionRowType
- } else {
- TypeUtils.project(table.rowType, partitionNames.asJava)
- }
- val currentPartitionSchema =
- if (partitionFieldCount == partitionSchema.length) {
- partitionSchema
- } else {
- SparkTypeUtils.fromPaimonRowType(currentPartitionRowType)
- }
- val rowConverter = CatalystTypeConverters.createToScalaConverter(
- CharVarcharUtils.replaceCharVarcharWithString(currentPartitionSchema))
- val rowDataPartitionComputer = new InternalRowPartitionComputer(
- partitionDefaultName,
- currentPartitionRowType,
- partitionNames.toArray,
- legacyPartitionName
- )
-
- rowDataPartitionComputer.generatePartValues(
- new SparkRow(currentPartitionRowType, rowConverter(r).asInstanceOf[Row]))
- }
- case _ =>
- throw new UnsupportedOperationException("Only FileStoreTable supports partitions.")
- }
+ val partitionKeys = table.partitionKeys().asScala.toSeq
+
+ rows.map(r => toPaimonPartition(r, partitionKeys.take(r.numFields)))
+ }
+
+ private def toPaimonPartition(
+ row: InternalRow,
+ partitionNames: Seq[String]): java.util.Map[String, String] = {
+ val coreOptions = CoreOptions.fromMap(table.options())
+ val partitionDefaultName = coreOptions.partitionDefaultName()
+ val legacyPartitionName = coreOptions.legacyPartitionName
+ val currentPartitionRowType = TypeUtils.project(table.rowType, partitionNames.asJava)
+ val currentPartitionSchema = SparkTypeUtils.fromPaimonRowType(currentPartitionRowType)
+ val rowConverter = CatalystTypeConverters.createToScalaConverter(
+ CharVarcharUtils.replaceCharVarcharWithString(currentPartitionSchema))
+ val rowDataPartitionComputer = new InternalRowPartitionComputer(
+ partitionDefaultName,
+ currentPartitionRowType,
+ partitionNames.toArray,
+ legacyPartitionName
+ )
+
+ rowDataPartitionComputer.generatePartValues(
+ new SparkRow(currentPartitionRowType, rowConverter(row).asInstanceOf[Row]))
}
override def dropPartitions(rows: Array[InternalRow]): Boolean = {
@@ -116,6 +103,137 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L
}
}
+ /**
+ * Resolves, with a single catalog list-by-names lookup, which of the given complete partition
+ * specs are registered for a Format Table with catalog-managed partitions. The result is aligned
+ * with the input arrays.
+ */
+ private[spark] def formatTablePartitionsRegistered(
+ partitionNames: Array[Array[String]],
+ rows: Array[InternalRow]): Array[Boolean] = {
+ if (rows.isEmpty) {
+ return Array.empty
+ }
+
+ table match {
+ case formatTable: FormatTable if partitionManager != null =>
+ val requested =
+ rows.zip(partitionNames).map { case (row, names) => toPaimonPartition(row, names.toSeq) }
+ val registered = requirePartitionManager().listPartitionsByNames(requested.toSeq.asJava)
+ val registeredSpecs = registered.asScala.map(_.spec().asScala.toMap).toSet
+ requested.map(spec => registeredSpecs.contains(spec.asScala.toMap))
+ case _ =>
+ throw new UnsupportedOperationException(
+ "Partition registration lookup is supported only for a Format Table with " +
+ "catalog-managed partitions.")
+ }
+ }
+
+ /**
+ * Drops the given partitions: complete specs are unregistered and their directories deleted
+ * as-is, partial specs are expanded to the registered leaf partitions they cover. Callers are
+ * responsible for resolving which complete specs are actually registered first (see
+ * [[formatTablePartitionsRegistered]]), so unregistered data directories are never deleted.
+ */
+ private[spark] def dropFormatTablePartitions(
+ partitionNames: Array[Array[String]],
+ rows: Array[InternalRow]): Boolean = {
+ table match {
+ case formatTable: FormatTable if partitionManager != null =>
+ val partitionKeyCount = formatTable.partitionKeys().size()
+ val requested =
+ rows.zip(partitionNames).map { case (row, names) => toPaimonPartition(row, names.toSeq) }
+ val partitions = ArrayBuffer.empty[java.util.Map[String, String]]
+ val seenPartitions = HashSet.empty[Map[String, String]]
+
+ def addPartition(partition: java.util.Map[String, String]): Unit = {
+ if (seenPartitions.add(partition.asScala.toMap)) {
+ partitions += partition
+ }
+ }
+
+ // Preserve exact requests as-is and let discovery add only missing complete leaves.
+ requested.filter(_.size() == partitionKeyCount).foreach(addPartition)
+ val partialSpecs = requested.filter(_.size() < partitionKeyCount).toSeq.distinct
+ if (partialSpecs.nonEmpty) {
+ def matchesRequestedPartial(partition: java.util.Map[String, String]): Boolean = {
+ partialSpecs.exists(_.asScala.forall {
+ case (key, value) => Objects.equals(value, partition.get(key))
+ })
+ }
+
+ // One unfiltered traversal resolves every partial spec; the requested constraints are
+ // enforced client-side.
+ requirePartitionManager()
+ .listPartitions(Collections.emptyMap[String, String]())
+ .asScala
+ .foreach {
+ partition =>
+ val validated = validateCatalogRegisteredPartition(formatTable, partition.spec())
+ if (matchesRequestedPartial(validated)) {
+ addPartition(validated)
+ }
+ }
+ }
+ dropCatalogRegisteredPartitions(formatTable, partitions.toSeq)
+ case _ =>
+ throw new UnsupportedOperationException(
+ "Named partition drop is supported only for a Format Table with catalog-managed " +
+ "partitions.")
+ }
+ }
+
+ private def dropCatalogRegisteredPartitions(
+ formatTable: FormatTable,
+ partitions: Seq[java.util.Map[String, String]]): Boolean = {
+ // Unregister first so new queries stop seeing the partition, then delete the data directory
+ // with the table FileIO (client-side; the server never deletes data). A deletion failure leaves
+ // the possibly incomplete directory invisible; it must not be registered again automatically.
+ if (partitions.isEmpty) {
+ return true
+ }
+
+ val onlyValueInPath =
+ CoreOptions.fromMap(formatTable.options()).formatTablePartitionOnlyValueInPath()
+ // Resolve (and path-safety validate) every partition directory before any mutation, so a
+ // traversal attempt ('.'/'..') fails the whole DROP before unregistering anything.
+ val partitionPaths = partitions.map {
+ spec =>
+ resolvePartitionPathWithinTable(
+ formatTable,
+ orderedSpec(formatTable, spec),
+ onlyValueInPath)
+ }
+ logInfo("Try to drop catalog-registered partitions: " + partitions.mkString(","))
+ requirePartitionManager().dropPartitions(partitions.asJava)
+ val fileIO = formatTable.fileIO()
+ partitionPaths.foreach {
+ partitionPath =>
+ val deleted = fileIO.delete(partitionPath, true)
+ if (!deleted && fileIO.exists(partitionPath)) {
+ throw new java.io.IOException(
+ s"FileIO reported that partition directory $partitionPath was not deleted.")
+ }
+ }
+ true
+ }
+
+ private def validateCatalogRegisteredPartition(
+ formatTable: FormatTable,
+ partition: java.util.Map[String, String]): java.util.Map[String, String] = {
+ val partitionKeys = formatTable.partitionKeys().asScala
+ if (partitionKeys.exists(key => partition.get(key) == null)) {
+ throw new IllegalStateException(
+ s"Catalog must return a complete partition spec with keys " +
+ s"${partitionKeys.mkString("[", ", ", "]")} for format table " +
+ s"${formatTable.fullName()}, but returned $partition.")
+ }
+
+ val ordered = new java.util.LinkedHashMap[String, String]()
+ partitionKeys.foreach(key => ordered.put(key, partition.get(key)))
+ ordered
+ }
+
override def truncatePartitions(idents: Array[InternalRow]): Boolean = {
val partitions = toPaimonPartitions(idents).toSeq.asJava
val commit = table.newBatchWriteBuilder().newCommit()
@@ -216,4 +334,77 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L
throw new UnsupportedOperationException("Only FileStoreTable supports create partitions.")
}
}
+
+ private[spark] def createFormatTablePartitions(
+ rows: Array[InternalRow],
+ maps: Array[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ if (maps.exists(_.keySet().asScala.exists(_.equalsIgnoreCase("location")))) {
+ throw new UnsupportedOperationException(
+ s"ADD PARTITION with LOCATION is not supported for Format Table ${table.fullName()}.")
+ }
+ val formatTable = table.asInstanceOf[FormatTable]
+ val onlyValueInPath =
+ CoreOptions.fromMap(formatTable.options()).formatTablePartitionOnlyValueInPath()
+ val specs = toPaimonPartitions(rows).toSeq
+ // Resolve (and path-safety validate) every directory before mutating anything.
+ val partitionPaths = specs.map {
+ spec =>
+ resolvePartitionPathWithinTable(
+ formatTable,
+ orderedSpec(formatTable, spec),
+ onlyValueInPath)
+ }
+ requirePartitionManager().createPartitions(specs.asJava, ignoreIfExists)
+ // Create the partition directories client-side (symmetric with DROP deleting them), so an
+ // added partition exists on the filesystem and a subsequent scan returns an empty partition
+ // rather than depending on lazy directory creation, matching Hive ADD PARTITION semantics.
+ val fileIO = formatTable.fileIO()
+ partitionPaths.foreach(partitionPath => fileIO.mkdirs(partitionPath))
+ }
+
+ private def orderedSpec(
+ formatTable: FormatTable,
+ spec: java.util.Map[String, String]): java.util.LinkedHashMap[String, String] = {
+ val ordered = new java.util.LinkedHashMap[String, String]()
+ formatTable.partitionKeys().asScala.foreach {
+ key => if (spec.containsKey(key)) ordered.put(key, spec.get(key))
+ }
+ ordered
+ }
+
+ /**
+ * Build the partition directory for a spec and verify it stays strictly under the table location.
+ * Value-only path components are validated (including rejecting '.'/'..'), and the normalized
+ * path is checked against the table location so neither DROP (recursive delete) nor ADD (mkdirs)
+ * can escape the table directory via crafted or corrupt partition values.
+ */
+ private def resolvePartitionPathWithinTable(
+ formatTable: FormatTable,
+ orderedSpec: java.util.LinkedHashMap[String, String],
+ onlyValueInPath: Boolean): Path = {
+ PartitionPathUtils.validatePartitionSpecForPath(orderedSpec, onlyValueInPath)
+ val tablePath = new Path(formatTable.location())
+ val partitionPath = new Path(
+ tablePath,
+ PartitionPathUtils.generatePartitionPathUtil(orderedSpec, onlyValueInPath)
+ )
+ val normalizedTable = tablePath.toUri.normalize().getPath
+ val tablePrefix = if (normalizedTable.endsWith("/")) normalizedTable else normalizedTable + "/"
+ val normalizedPartition = partitionPath.toUri.normalize().getPath
+ if (!normalizedPartition.startsWith(tablePrefix)) {
+ throw new IllegalArgumentException(
+ s"Resolved partition path $partitionPath escapes the table location $tablePath for " +
+ s"partition spec $orderedSpec of Format Table ${formatTable.fullName()}.")
+ }
+ partitionPath
+ }
+
+ private def requirePartitionManager(): FormatTablePartitionManager = {
+ if (partitionManager == null) {
+ throw new UnsupportedOperationException(
+ s"Catalog-managed partitions are not configured for format table ${table.fullName()}.")
+ }
+ partitionManager
+ }
}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala
new file mode 100644
index 000000000000..c6a1f301da37
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonFormatTablePartitionDdlExec.scala
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.execution
+
+import org.apache.paimon.spark.format.{FormatTablePartitionRepair, PaimonFormatTable}
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec}
+import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.execution.datasources.v2.LeafV2CommandExec
+
+import java.util.{Map => JMap}
+
+import scala.collection.JavaConverters._
+
+object PaimonFormatTablePartitionDdlExec {
+
+ /**
+ * A Format Table uses catalog-managed partitions exactly when the catalog gave it a partition
+ * manager; otherwise its partitions are discovered from the filesystem.
+ */
+ def usesCatalogManagedPartitions(table: PaimonFormatTable): Boolean =
+ table.partitionManager != null
+
+ /**
+ * Run an operation and refresh Spark's cached plans afterwards, whether it succeeded or not. A
+ * refresh failure never replaces the operation's own failure — that one explains what went wrong
+ * — and is attached to it instead, unless the two are the same throwable.
+ */
+ private[execution] def refreshingCache[T](refreshCache: () => Unit)(operation: => T): T = {
+ var operationFailure: Throwable = null
+ try {
+ operation
+ } catch {
+ case failure: Throwable =>
+ operationFailure = failure
+ throw failure
+ } finally {
+ try {
+ refreshCache()
+ } catch {
+ case refreshFailure: Throwable =>
+ if (operationFailure == null) {
+ throw refreshFailure
+ } else if (refreshFailure ne operationFailure) {
+ operationFailure.addSuppressed(refreshFailure)
+ }
+ }
+ }
+ }
+
+ private[execution] def unsupportedWithoutCatalogManagedPartitions(
+ operation: String,
+ table: PaimonFormatTable): UnsupportedOperationException =
+ new UnsupportedOperationException(
+ s"$operation PARTITION is supported only for a Format Table with catalog-managed " +
+ s"partitions, but partitions of ${table.name()} are discovered from the filesystem.")
+}
+
+/**
+ * Registers catalog-managed partitions of a Format Table without Spark's client-side existence
+ * precheck. The whole batch and IF NOT EXISTS flag are forwarded to the catalog so it can apply
+ * them atomically.
+ */
+case class PaimonAddFormatTablePartitionsExec(
+ table: PaimonFormatTable,
+ partSpecs: Seq[ResolvedPartitionSpec],
+ ignoreIfExists: Boolean,
+ refreshCache: () => Unit)
+ extends LeafV2CommandExec {
+
+ override protected def run(): Seq[InternalRow] = {
+ if (!PaimonFormatTablePartitionDdlExec.usesCatalogManagedPartitions(table)) {
+ throw PaimonFormatTablePartitionDdlExec.unsupportedWithoutCatalogManagedPartitions(
+ "ADD",
+ table)
+ }
+
+ if (partSpecs.nonEmpty) {
+ val properties: Array[JMap[String, String]] =
+ partSpecs.map(spec => spec.location.map("location" -> _).toMap.asJava).toArray
+ PaimonFormatTablePartitionDdlExec.refreshingCache(refreshCache) {
+ table.createFormatTablePartitions(
+ partSpecs.map(_.ident).toArray,
+ properties,
+ ignoreIfExists)
+ }
+ }
+ Seq.empty
+ }
+
+ override def output: Seq[Attribute] = Seq.empty
+}
+
+/**
+ * Physical command providing explicit DROP semantics for Format Tables with catalog-managed
+ * partitions (tables using filesystem partition discovery always fail with a clear error). Complete
+ * specs are resolved against the catalog registration first: missing specs fail with
+ * [[NoSuchPartitionsException]] unless IF EXISTS was given, and only registered specs are
+ * unregistered and have their directories deleted, so data that is merely awaiting registration
+ * (e.g. by MSCK REPAIR TABLE) is never removed. Partial specs resolve to the registered leaf
+ * partitions they cover.
+ */
+case class PaimonDropFormatTablePartitionsExec(
+ table: PaimonFormatTable,
+ partSpecs: Seq[ResolvedPartitionSpec],
+ ifExists: Boolean,
+ purge: Boolean,
+ refreshCache: () => Unit)
+ extends LeafV2CommandExec {
+
+ override protected def run(): Seq[InternalRow] = {
+ if (!PaimonFormatTablePartitionDdlExec.usesCatalogManagedPartitions(table)) {
+ throw PaimonFormatTablePartitionDdlExec.unsupportedWithoutCatalogManagedPartitions(
+ "DROP",
+ table)
+ }
+ if (purge) {
+ // Match Spark's v2 default for purgePartitions. DROP PARTITION on a Format Table with
+ // catalog-managed partitions already removes the partition data, so PURGE adds nothing.
+ throw new UnsupportedOperationException(
+ s"DROP PARTITION ... PURGE is not supported for Format Table ${table.name()}. " +
+ "DROP PARTITION already removes the partition data.")
+ }
+ if (partSpecs.isEmpty) {
+ return Seq.empty
+ }
+
+ val partitionKeyCount = table.table.partitionKeys().size()
+ val (completeSpecs, partialSpecs) =
+ partSpecs.partition(_.ident.numFields == partitionKeyCount)
+ val registration = table
+ .formatTablePartitionsRegistered(
+ completeSpecs.map(_.names.toArray).toArray,
+ completeSpecs.map(_.ident).toArray)
+ .toSeq
+ val missingSpecs = completeSpecs.zip(registration).collect { case (spec, false) => spec }
+ if (missingSpecs.nonEmpty && !ifExists) {
+ throw new NoSuchPartitionsException(
+ table.name(),
+ missingSpecs.map(_.ident),
+ table.partitionSchema)
+ }
+
+ val specsToDrop =
+ completeSpecs.zip(registration).collect { case (spec, true) => spec } ++ partialSpecs
+ if (specsToDrop.nonEmpty) {
+ // Catalog-managed semantics (PaimonPartitionManagement#dropFormatTablePartitions): resolve
+ // partial specs, unregister the exact catalog partitions, then delete their directories
+ // with the table FileIO client-side. A directory-deletion failure stays unregistered so
+ // partially deleted data is not exposed again.
+ PaimonFormatTablePartitionDdlExec.refreshingCache(refreshCache) {
+ table.dropFormatTablePartitions(
+ specsToDrop.map(_.names.toArray).toArray,
+ specsToDrop.map(_.ident).toArray)
+ }
+ }
+ Seq.empty
+ }
+
+ override def output: Seq[Attribute] = Seq.empty
+}
+
+/**
+ * MSCK REPAIR TABLE for Format Tables with catalog-managed partitions. Spark rejects RepairTable
+ * for v2 tables in DataSourceV2Strategy, so PaimonStrategy intercepts first; plain MSCK means ADD.
+ * Tables using filesystem partition discovery are not intercepted and keep Spark's own v2
+ * rejection.
+ */
+case class PaimonRepairFormatTablePartitionsExec(
+ table: PaimonFormatTable,
+ addPartitions: Boolean,
+ dropPartitions: Boolean,
+ refreshCache: () => Unit)
+ extends LeafV2CommandExec {
+
+ override protected def run(): Seq[InternalRow] = {
+ PaimonFormatTablePartitionDdlExec.refreshingCache(refreshCache) {
+ FormatTablePartitionRepair.repair(table, addPartitions, dropPartitions)
+ }
+ Seq.empty
+ }
+
+ override def output: Seq[Attribute] = Seq.empty
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala
index 74c0e6c47dbd..3f376a978d1c 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/PaimonStrategy.scala
@@ -29,6 +29,7 @@ import org.apache.paimon.spark.catalog.{SparkBaseCatalog, SupportView}
import org.apache.paimon.spark.catalyst.analysis.ResolvedPaimonView
import org.apache.paimon.spark.catalyst.plans.logical.{CopyIntoLocationCommand, CopyIntoLocationSource, CopyIntoTableCommand, CreateOrReplaceTagCommand, CreatePaimonView, DeleteTagCommand, DropPaimonView, LateralVectorSearch, PaimonCallCommand, PaimonDropPartitions, PaimonTableValuedFunctions, RenameTagCommand, ResolvedIdentifier, ShowPaimonViews, ShowTagsCommand, TruncatePaimonTableWithFilter}
import org.apache.paimon.spark.data.SparkInternalRow
+import org.apache.paimon.spark.format.PaimonFormatTable
import org.apache.paimon.spark.read.VectorSearchResultUtils
import org.apache.paimon.spark.schema.PaimonMetadataColumn
import org.apache.paimon.table.{InnerTable, SpecialFields, Table}
@@ -42,7 +43,7 @@ import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{ResolvedNamespace, ResolvedTable}
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, GenericInternalRow, JoinedRow, PredicateHelper, UnsafeProjection}
-import org.apache.spark.sql.catalyst.plans.logical.{CreateTableAsSelect, DescribeRelation, LogicalPlan, ReplaceTable, ReplaceTableAsSelect, ShowCreateTable}
+import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, CreateTableAsSelect, DescribeRelation, DropPartitions, LogicalPlan, RepairTable, ReplaceTable, ReplaceTableAsSelect, ShowCreateTable}
import org.apache.spark.sql.catalyst.util.ArrayData
import org.apache.spark.sql.connector.catalog.{Identifier, PaimonLookupCatalog, TableCatalog}
import org.apache.spark.sql.execution.{PaimonDescribeTableExec, SparkPlan, SparkStrategy}
@@ -161,6 +162,52 @@ case class PaimonStrategy(spark: SparkSession)
case _ => Nil
}
+ case AddPartitions(r @ ResolvedTable(_, _, table: PaimonFormatTable, _), parts, ifNotExists) =>
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ parts.asResolvedPartitionSpecs,
+ ifNotExists,
+ recacheTable(r)) :: Nil
+
+ // Spark's DataSourceV2Strategy rejects RepairTable for every v2 table; Format Tables with
+ // catalog-managed partitions support it through the sync engine, so intercept here (extension
+ // strategies run first). Tables using filesystem partition discovery fall through and keep
+ // the upstream rejection.
+ case RepairTable(
+ r @ ResolvedTable(_, _, table: PaimonFormatTable, _),
+ enableAddPartitions,
+ enableDropPartitions)
+ if PaimonFormatTablePartitionDdlExec.usesCatalogManagedPartitions(table) =>
+ PaimonRepairFormatTablePartitionsExec(
+ table,
+ enableAddPartitions,
+ enableDropPartitions,
+ recacheTable(r)) :: Nil
+
+ case DropPartitions(
+ r @ ResolvedTable(_, _, table: PaimonFormatTable, _),
+ parts,
+ ifExists,
+ purge) =>
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ parts.asResolvedPartitionSpecs,
+ ifExists,
+ purge,
+ recacheTable(r)) :: Nil
+
+ case PaimonDropPartitions(
+ r @ ResolvedTable(_, _, table: PaimonFormatTable, _),
+ parts,
+ ifExists,
+ purge) =>
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ parts.asResolvedPartitionSpecs,
+ ifExists,
+ purge,
+ recacheTable(r)) :: Nil
+
case PaimonDropPartitions(
r @ ResolvedTable(_, _, table: SparkTable, _),
parts,
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala
index dc7effd39f72..48b4d85a7b19 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/format/PaimonFormatTable.scala
@@ -22,6 +22,7 @@ import org.apache.paimon.format.csv.CsvOptions
import org.apache.paimon.spark.{BaseTable, FormatTableScanBuilder}
import org.apache.paimon.spark.write.BaseV2WriteBuilder
import org.apache.paimon.table.FormatTable
+import org.apache.paimon.table.format.FormatTablePartitionManager
import org.apache.paimon.types.RowType
import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, TableCapability, TableCatalog}
@@ -41,6 +42,10 @@ case class PaimonFormatTable(table: FormatTable)
with SupportsRead
with SupportsWrite {
+ // Format Tables with catalog-managed partitions carry their own partition manager; tables using
+ // filesystem partition discovery return null.
+ override def partitionManager: FormatTablePartitionManager = table.partitionManager()
+
override def capabilities(): util.Set[TableCapability] = {
util.EnumSet.of(BATCH_READ, BATCH_WRITE, OVERWRITE_DYNAMIC, OVERWRITE_BY_FILTER)
}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
index 37521ceda653..4c6de87ff658 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/util/OptionUtils.scala
@@ -22,7 +22,7 @@ import org.apache.paimon.CoreOptions
import org.apache.paimon.catalog.Identifier
import org.apache.paimon.options.ConfigOption
import org.apache.paimon.spark.{SparkCatalogOptions, SparkConnectorOptions}
-import org.apache.paimon.table.Table
+import org.apache.paimon.table.{FormatTable, Table}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.SparkSession
@@ -186,10 +186,45 @@ object OptionUtils extends SQLConfHelper with Logging {
if (mergedOptions.isEmpty) {
table
} else {
+ normalizeCatalogManagedPartitionOptions(table, mergedOptions)
table.copy(mergedOptions).asInstanceOf[T]
}
}
+ /**
+ * Whether a format table's partitions are catalog-managed is a persisted property and cannot be
+ * flipped by a dynamic option. A dynamic {@code metastore.partitioned-table} (typically a
+ * session-global {@code spark.paimon.*} config that used to be a harmless no-op) is dropped so
+ * the persisted value always wins, warning only when it actually disagrees, instead of failing
+ * every format table load in that session.
+ */
+ private def normalizeCatalogManagedPartitionOptions(
+ table: Table,
+ dynamicOptions: JMap[String, String]): Unit = {
+ table match {
+ case formatTable: FormatTable =>
+ val partitionSourceKey = CoreOptions.FORMAT_TABLE_PARTITION_SOURCE.key()
+ val persistedPartitionsFromCatalog =
+ new CoreOptions(formatTable.options()).formatTablePartitionsFromCatalog()
+ if (dynamicOptions.containsKey(partitionSourceKey)) {
+ val dynamicPartitionsFromCatalog =
+ new CoreOptions(dynamicOptions).formatTablePartitionsFromCatalog()
+ if (dynamicPartitionsFromCatalog != persistedPartitionsFromCatalog) {
+ logWarning(
+ s"Ignoring dynamic option " +
+ s"'$partitionSourceKey=$dynamicPartitionsFromCatalog' for format table " +
+ s"${formatTable.fullName()}: whether its partitions are catalog-managed is fixed " +
+ s"at creation time (persisted value: $persistedPartitionsFromCatalog). " +
+ s"Use ALTER TABLE to change it.")
+ }
+ dynamicOptions.remove(partitionSourceKey)
+ }
+ // The remaining options are validated by FormatTable#copy, which sees exactly the same
+ // effective combination.
+ case _ =>
+ }
+ }
+
private def getMergedOptions(
catalogName: String = null,
ident: Identifier = null,
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala
index 5227d1f0f604..d14154cd2b1f 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/RewriteSparkDDLCommands.scala
@@ -18,14 +18,18 @@
package org.apache.spark.sql.catalyst.parser.extensions
-import org.apache.paimon.spark.catalog.SupportView
-import org.apache.paimon.spark.catalyst.plans.logical.{PaimonDropPartitions, ResolvedIdentifier}
+import org.apache.paimon.spark.SparkTable
+import org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions
+import org.apache.paimon.spark.execution.PaimonFormatTablePartitionDdlExec
+import org.apache.paimon.spark.format.PaimonFormatTable
import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.analysis.UnresolvedIdentifier
+import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, UnresolvedTable}
import org.apache.spark.sql.catalyst.plans.logical.{DropPartitions, LogicalPlan}
import org.apache.spark.sql.catalyst.rules.Rule
-import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog}
+import org.apache.spark.sql.connector.catalog.{CatalogManager, LookupCatalog, TableCatalog}
+
+import scala.util.Try
case class RewriteSparkDDLCommands(spark: SparkSession)
extends Rule[LogicalPlan]
@@ -37,8 +41,41 @@ case class RewriteSparkDDLCommands(spark: SparkSession)
// A new member was added to CreatePaimonView since spark4.0,
// unapply pattern matching is not used here to ensure compatibility across multiple spark versions.
- case DropPartitions(UnresolvedPaimonRelation(aliasedTable), parts, ifExists, purge) =>
+ //
+ // Both Paimon data tables and Format Tables with catalog-managed partitions route DROP
+ // PARTITION through PaimonDropPartitions (the latter resolve partial specs through the
+ // partition gateway, so they must bypass Spark's strict partition-spec resolution too). A
+ // single extractor loads the table once and classifies it, avoiding a second
+ // catalog.loadTable per analyzer iteration. Format Tables using filesystem partition
+ // discovery are not matched and keep Spark's own resolution plus the explicit unsupported
+ // error at execution.
+ case DropPartitions(UnresolvedPaimonDropTarget(aliasedTable), parts, ifExists, purge) =>
PaimonDropPartitions(aliasedTable, parts, ifExists, purge)
+ }
+
+ private object UnresolvedPaimonDropTarget {
+ def unapply(plan: LogicalPlan): Option[LogicalPlan] = {
+ EliminateSubqueryAliases(plan) match {
+ case UnresolvedTable(multipartIdentifier, _, _)
+ if isPaimonDropTarget(multipartIdentifier) =>
+ Some(plan)
+ case _ => None
+ }
+ }
+ }
+ private def isPaimonDropTarget(multipartIdentifier: Seq[String]): Boolean = {
+ multipartIdentifier match {
+ case CatalogAndIdentifier(catalog: TableCatalog, ident) =>
+ Try(catalog.loadTable(ident))
+ .map {
+ case _: SparkTable => true
+ case formatTable: PaimonFormatTable =>
+ PaimonFormatTablePartitionDdlExec.usesCatalogManagedPartitions(formatTable)
+ case _ => false
+ }
+ .getOrElse(false)
+ case _ => false
+ }
}
}
diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/format/FormatTablePartitionRepairTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/format/FormatTablePartitionRepairTest.java
new file mode 100644
index 000000000000..7dec20e866e7
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/format/FormatTablePartitionRepairTest.java
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.FileStatus;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.partition.Partition;
+import org.apache.paimon.table.FormatTable;
+import org.apache.paimon.table.format.FormatTablePartitionManager;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for the catalog-managed partition repair engine of Format Tables. */
+class FormatTablePartitionRepairTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ @Test
+ void applyDiffsAgainstTheWholeUnfilteredCatalogListing() {
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ catalog.register(Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")));
+
+ int applied =
+ FormatTablePartitionRepair.apply(
+ catalog,
+ Arrays.asList(
+ spec("dt", "20260701"),
+ spec("dt", "20260702"),
+ spec("dt", "20260703")),
+ Collections.singletonList("dt"),
+ true,
+ false);
+
+ // Only the partition missing from the catalog listing is registered.
+ assertThat(applied).isEqualTo(1);
+ assertThat(catalog.createdPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260703")));
+ // The diff needs every registered partition, so the listing is asked for all of them.
+ assertThat(catalog.requestedPrefixes).containsExactly(Collections.emptyMap());
+ }
+
+ @Test
+ void applyCreatesTheWholeDiffAsAnIdempotentBatch() {
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+
+ int applied =
+ FormatTablePartitionRepair.apply(
+ catalog,
+ Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")),
+ Collections.singletonList("dt"),
+ true,
+ false);
+
+ assertThat(applied).isEqualTo(2);
+ assertThat(catalog.createdPartitions)
+ .containsExactly(Arrays.asList(spec("dt", "20260701"), spec("dt", "20260702")));
+ assertThat(catalog.createIgnoreFlags).containsExactly(true);
+ assertThat(catalog.droppedPartitions).isEmpty();
+ }
+
+ @Test
+ void applyPassesALargeAddDiffToTheCatalogInOneCall() {
+ List> filesystemPartitions = new ArrayList<>();
+ for (int index = 0; index < 1001; index++) {
+ filesystemPartitions.add(spec("dt", String.format("%04d", index)));
+ }
+
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+
+ int applied =
+ FormatTablePartitionRepair.apply(
+ catalog,
+ filesystemPartitions,
+ Collections.singletonList("dt"),
+ true,
+ false);
+
+ // Splitting a diff into per-request batches belongs to the partition catalog, so the
+ // repair hands over the complete diff in a single call.
+ assertThat(applied).isEqualTo(1001);
+ assertThat(catalog.createdPartitions).hasSize(1);
+ assertThat(catalog.createdPartitions.get(0)).isEqualTo(filesystemPartitions);
+ assertThat(catalog.createIgnoreFlags).containsExactly(true);
+ }
+
+ @Test
+ void applyPassesALargeDropDiffToTheCatalogInOneCall() {
+ List> registered = new ArrayList<>();
+ for (int index = 0; index < 1001; index++) {
+ registered.add(spec("dt", String.format("%04d", index)));
+ }
+
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ catalog.register(registered);
+
+ int applied =
+ FormatTablePartitionRepair.apply(
+ catalog,
+ Collections.emptyList(),
+ Collections.singletonList("dt"),
+ false,
+ true);
+
+ assertThat(applied).isEqualTo(1001);
+ assertThat(catalog.droppedPartitions).hasSize(1);
+ assertThat(catalog.droppedPartitions.get(0)).isEqualTo(registered);
+ assertThat(catalog.createdPartitions).isEmpty();
+ }
+
+ @Test
+ void dropOnlyUnregistersOnlyMissingDirectories() {
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ catalog.register(Arrays.asList(spec("dt", "20260714"), spec("dt", "20260715")));
+
+ // dt=20260715 exists on the filesystem, dt=20260714 does not: only the latter is dropped.
+ int applied =
+ FormatTablePartitionRepair.apply(
+ catalog,
+ Collections.singletonList(spec("dt", "20260715")),
+ Collections.singletonList("dt"),
+ false,
+ true);
+
+ assertThat(applied).isEqualTo(1);
+ assertThat(catalog.droppedPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260714")));
+ assertThat(catalog.createdPartitions).isEmpty();
+ }
+
+ @Test
+ void addOnlyNeverDropsStaleCatalogPartitions() {
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ catalog.register(Collections.singletonList(spec("dt", "20260714")));
+
+ int applied =
+ FormatTablePartitionRepair.apply(
+ catalog,
+ Collections.singletonList(spec("dt", "20260715")),
+ Collections.singletonList("dt"),
+ true,
+ false);
+
+ assertThat(applied).isEqualTo(1);
+ assertThat(catalog.createdPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260715")));
+ assertThat(catalog.droppedPartitions).isEmpty();
+ }
+
+ @Test
+ void syncAddsAndDropsInOneCall() {
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ catalog.register(Collections.singletonList(spec("dt", "20260714")));
+
+ int applied =
+ FormatTablePartitionRepair.apply(
+ catalog,
+ Collections.singletonList(spec("dt", "20260715")),
+ Collections.singletonList("dt"),
+ true,
+ true);
+
+ assertThat(applied).isEqualTo(2);
+ assertThat(catalog.createdPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260715")));
+ assertThat(catalog.droppedPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260714")));
+ }
+
+ @Test
+ void applyWithNoDiffDoesNotIssueAnEmptyMutation() {
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ catalog.register(Collections.singletonList(spec("dt", "20260701")));
+
+ int applied =
+ FormatTablePartitionRepair.apply(
+ catalog,
+ Collections.singletonList(spec("dt", "20260701")),
+ Collections.singletonList("dt"),
+ true,
+ true);
+
+ assertThat(applied).isZero();
+ assertThat(catalog.createdPartitions).isEmpty();
+ assertThat(catalog.droppedPartitions).isEmpty();
+ }
+
+ @Test
+ void applyPropagatesFailureAfterAPartiallyAppliedMutation() {
+ IllegalStateException failure =
+ new IllegalStateException("injected partition drop failure");
+ RecordingPartitionManager catalog =
+ new RecordingPartitionManager() {
+ @Override
+ public void dropPartitions(List> partitions) {
+ throw failure;
+ }
+ };
+ catalog.register(Collections.singletonList(spec("dt", "20260714")));
+
+ assertThatThrownBy(
+ () ->
+ FormatTablePartitionRepair.apply(
+ catalog,
+ Collections.singletonList(spec("dt", "20260715")),
+ Collections.singletonList("dt"),
+ true,
+ true))
+ .isSameAs(failure);
+ // The ADD half is already committed; a rerun converges from there.
+ assertThat(catalog.createdPartitions)
+ .containsExactly(Collections.singletonList(spec("dt", "20260715")));
+ }
+
+ @Test
+ void repairRegistersRawDirectoryValuesWithoutCastingThem() throws Exception {
+ java.nio.file.Path partitionDirectory =
+ Files.createDirectories(tempDir.resolve("dt=20260701/month=01"));
+ Files.write(
+ partitionDirectory.resolve("data.csv"),
+ Collections.singletonList("1"),
+ StandardCharsets.UTF_8);
+
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ PaimonFormatTable sparkTable =
+ new PaimonFormatTable(
+ monthPartitionedTable(
+ LocalFileIO.create(), tempDir.toUri().toString(), catalog));
+
+ int applied = FormatTablePartitionRepair.repair(sparkTable, true, false);
+
+ // The raw directory value must survive: a scan-based discovery would cast month=01 to 1
+ // and register a spec that no longer round-trips to the real directory.
+ Map expected = new LinkedHashMap<>();
+ expected.put("dt", "20260701");
+ expected.put("month", "01");
+ assertThat(applied).isEqualTo(1);
+ assertThat(catalog.createdPartitions).containsExactly(Collections.singletonList(expected));
+ assertThat(catalog.createIgnoreFlags).containsExactly(true);
+ }
+
+ @Test
+ void repairRejectsUnsafeValueOnlyDirectoryBeforeCatalogMutation() throws Exception {
+ Files.createDirectories(tempDir.resolve("%2E%2E"));
+
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ PaimonFormatTable sparkTable =
+ new PaimonFormatTable(formatTable(tempDir.toUri().toString(), true, catalog));
+
+ assertThatThrownBy(() -> FormatTablePartitionRepair.repair(sparkTable, true, true))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("..");
+ assertThat(catalog.requestedPrefixes).isEmpty();
+ assertThat(catalog.createdPartitions).isEmpty();
+ assertThat(catalog.droppedPartitions).isEmpty();
+ }
+
+ @Test
+ void repairFailsClosedWhenTheCatalogListingFails() throws Exception {
+ java.nio.file.Path partitionDirectory =
+ Files.createDirectories(tempDir.resolve("dt=20260715"));
+ Files.write(
+ partitionDirectory.resolve("data.csv"),
+ Collections.singletonList("15"),
+ StandardCharsets.UTF_8);
+
+ IllegalStateException listFailure =
+ new IllegalStateException("injected catalog listing failure");
+ AtomicInteger listCount = new AtomicInteger();
+ RecordingPartitionManager catalog =
+ new RecordingPartitionManager() {
+ @Override
+ public List listPartitions(Map prefix) {
+ listCount.incrementAndGet();
+ throw listFailure;
+ }
+ };
+ PaimonFormatTable sparkTable =
+ new PaimonFormatTable(formatTable(tempDir.toUri().toString(), catalog));
+
+ assertThatThrownBy(() -> FormatTablePartitionRepair.repair(sparkTable, true, true))
+ .isSameAs(listFailure);
+ assertThat(listCount).hasValue(1);
+ assertThat(catalog.createdPartitions).isEmpty();
+ assertThat(catalog.droppedPartitions).isEmpty();
+ assertThat(partitionDirectory).exists();
+ }
+
+ @Test
+ void repairFailsClosedWhenFilesystemDiscoveryFailsPartway() throws Exception {
+ Files.createDirectories(tempDir.resolve("dt=20260715/month=01"));
+ Files.createDirectories(tempDir.resolve("dt=20260716/month=01"));
+
+ IOException listFailure = new IOException("injected nested filesystem LIST failure");
+ AtomicInteger completedPartitionListings = new AtomicInteger();
+ FileIO fileIO =
+ new LocalFileIO() {
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ if ("dt=20260716".equals(path.getName())) {
+ throw listFailure;
+ }
+ FileStatus[] statuses = super.listStatus(path);
+ Arrays.sort(
+ statuses,
+ Comparator.comparing(status -> status.getPath().toString()));
+ if ("dt=20260715".equals(path.getName())) {
+ completedPartitionListings.incrementAndGet();
+ }
+ return statuses;
+ }
+ };
+
+ RecordingPartitionManager catalog = new RecordingPartitionManager();
+ PaimonFormatTable sparkTable =
+ new PaimonFormatTable(
+ monthPartitionedTable(fileIO, tempDir.toUri().toString(), catalog));
+
+ assertThatThrownBy(() -> FormatTablePartitionRepair.repair(sparkTable, true, true))
+ .isInstanceOf(RuntimeException.class)
+ .hasCause(listFailure);
+ // A truncated listing must never reach the diff: no catalog call at all.
+ assertThat(completedPartitionListings).hasValue(1);
+ assertThat(catalog.requestedPrefixes).isEmpty();
+ assertThat(catalog.createdPartitions).isEmpty();
+ assertThat(catalog.droppedPartitions).isEmpty();
+ }
+
+ private static Map spec(String key, String value) {
+ Map spec = new LinkedHashMap<>();
+ spec.put(key, value);
+ return spec;
+ }
+
+ private static FormatTable formatTable(String location, FormatTablePartitionManager catalog) {
+ return formatTable(location, false, catalog);
+ }
+
+ private static FormatTable formatTable(
+ String location, boolean onlyValueInPath, FormatTablePartitionManager catalog) {
+ RowType rowType =
+ RowType.builder()
+ .field("id", DataTypes.INT())
+ .field("dt", DataTypes.STRING())
+ .build();
+ return build(
+ LocalFileIO.create(),
+ location,
+ rowType,
+ Collections.singletonList("dt"),
+ onlyValueInPath,
+ catalog);
+ }
+
+ /** A table whose second partition key is an INT, so a cast would rewrite {@code 01} to 1. */
+ private static FormatTable monthPartitionedTable(
+ FileIO fileIO, String location, FormatTablePartitionManager catalog) {
+ RowType rowType =
+ RowType.builder()
+ .field("id", DataTypes.INT())
+ .field("dt", DataTypes.STRING())
+ .field("month", DataTypes.INT())
+ .build();
+ return build(fileIO, location, rowType, Arrays.asList("dt", "month"), false, catalog);
+ }
+
+ private static FormatTable build(
+ FileIO fileIO,
+ String location,
+ RowType rowType,
+ List partitionKeys,
+ boolean onlyValueInPath,
+ FormatTablePartitionManager catalog) {
+ Map options = new LinkedHashMap<>();
+ options.put(CoreOptions.FORMAT_TABLE_PARTITION_SOURCE.key(), "rest");
+ options.put(
+ CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
+ Boolean.toString(onlyValueInPath));
+ return FormatTable.builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("db", "t"))
+ .rowType(rowType)
+ .partitionKeys(partitionKeys)
+ .location(new Path(location).toString())
+ .format(FormatTable.Format.CSV)
+ .options(options)
+ .partitionManager(catalog)
+ .build();
+ }
+
+ private static class RecordingPartitionManager implements FormatTablePartitionManager {
+
+ private static final long serialVersionUID = 1L;
+
+ private final List> registered = new ArrayList<>();
+ private final List> requestedPrefixes = new ArrayList<>();
+ private final List>> createdPartitions = new ArrayList<>();
+ private final List createIgnoreFlags = new ArrayList<>();
+ private final List>> droppedPartitions = new ArrayList<>();
+
+ private void register(List> partitions) {
+ registered.addAll(partitions);
+ }
+
+ @Override
+ public void createPartitions(List> partitions, boolean ignoreIfExists) {
+ createdPartitions.add(new ArrayList<>(partitions));
+ createIgnoreFlags.add(ignoreIfExists);
+ }
+
+ @Override
+ public void dropPartitions(List> partitions) {
+ droppedPartitions.add(new ArrayList<>(partitions));
+ }
+
+ @Override
+ public List listPartitionsByNames(List> partitions) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List listPartitions(Map prefix) {
+ requestedPrefixes.add(prefix);
+ List partitions = new ArrayList<>(registered.size());
+ for (Map spec : registered) {
+ partitions.add(new Partition(spec, 0L, 0L, 0L, 0L, 0, false));
+ }
+ return partitions;
+ }
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala
new file mode 100644
index 000000000000..7d8c9981198c
--- /dev/null
+++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/util/OptionUtilsTest.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.util
+
+import org.apache.paimon.CoreOptions.{FORMAT_TABLE_IMPLEMENTATION, FORMAT_TABLE_PARTITION_SOURCE}
+import org.apache.paimon.catalog.Identifier
+import org.apache.paimon.fs.local.LocalFileIO
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.table.FormatTable.Format
+import org.apache.paimon.table.format.FormatTablePartitionManager
+import org.apache.paimon.types.{DataTypes, RowType}
+
+import org.apache.spark.sql.internal.SQLConf
+import org.scalatest.funsuite.AnyFunSuite
+
+import java.util.Collections
+
+import scala.collection.JavaConverters._
+
+/** Tests for [[OptionUtils]]. */
+class OptionUtilsTest extends AnyFunSuite {
+
+ test("reject engine implementation from SQL conf for catalog-managed partitions") {
+ val exception = intercept[IllegalArgumentException] {
+ SQLConf.withExistingConf(engineSQLConf) {
+ OptionUtils.copyWithSQLConf(formatTable(withCatalogManagedPartitions = true))
+ }
+ }
+
+ assert(exception.getMessage.contains(FORMAT_TABLE_PARTITION_SOURCE.key()))
+ assert(exception.getMessage.contains(FORMAT_TABLE_IMPLEMENTATION.key()))
+ }
+
+ test("allow engine implementation from SQL conf for filesystem-discovered partitions") {
+ val copied = SQLConf.withExistingConf(engineSQLConf) {
+ OptionUtils.copyWithSQLConf(formatTable(withCatalogManagedPartitions = false))
+ }
+
+ assert(copied.options().get(FORMAT_TABLE_IMPLEMENTATION.key()) == "engine")
+ }
+
+ test("reject invalid partition-mode option from SQL conf with option context") {
+ val sqlConf = new SQLConf
+ sqlConf.setConfString(s"spark.paimon.${FORMAT_TABLE_PARTITION_SOURCE.key()}", "yes")
+
+ val exception = intercept[IllegalArgumentException] {
+ SQLConf.withExistingConf(sqlConf) {
+ OptionUtils.copyWithSQLConf(formatTable(withCatalogManagedPartitions = false))
+ }
+ }
+
+ assert(exception.getMessage.contains("yes"))
+ assert(exception.getMessage.contains(FORMAT_TABLE_PARTITION_SOURCE.key()))
+ }
+
+ test(
+ "session-level metastore.partitioned-table is ignored, not failed, with catalog-managed partitions") {
+ val sqlConf = new SQLConf
+ sqlConf.setConfString(s"spark.paimon.${FORMAT_TABLE_PARTITION_SOURCE.key()}", "filesystem")
+
+ val copied = SQLConf.withExistingConf(sqlConf) {
+ OptionUtils.copyWithSQLConf(formatTable(withCatalogManagedPartitions = true))
+ }
+
+ // The persisted flag wins; the session-global override is dropped with a warning instead of
+ // failing every format table load in the session.
+ assert(copied.options().get(FORMAT_TABLE_PARTITION_SOURCE.key()) == "rest")
+ }
+
+ test(
+ "session-level metastore.partitioned-table is ignored, not failed, with filesystem partitions") {
+ val sqlConf = new SQLConf
+ sqlConf.setConfString(s"spark.paimon.${FORMAT_TABLE_PARTITION_SOURCE.key()}", "rest")
+
+ val copied = SQLConf.withExistingConf(sqlConf) {
+ OptionUtils.copyWithSQLConf(formatTable(withCatalogManagedPartitions = false))
+ }
+
+ assert(copied.options().get(FORMAT_TABLE_PARTITION_SOURCE.key()) == "filesystem")
+ }
+
+ private def engineSQLConf: SQLConf = {
+ val sqlConf = new SQLConf
+ sqlConf.setConfString(
+ s"spark.paimon.${FORMAT_TABLE_IMPLEMENTATION.key()}",
+ "engine"
+ )
+ sqlConf
+ }
+
+ private def formatTable(withCatalogManagedPartitions: Boolean): FormatTable = {
+ FormatTable
+ .builder()
+ .fileIO(new LocalFileIO)
+ .identifier(Identifier.create("test_db", "format_table"))
+ .rowType(RowType.of(DataTypes.INT(), DataTypes.STRING()))
+ .partitionKeys(Collections.singletonList("dt"))
+ .location("file:///tmp/test_db.db/format_table")
+ .format(Format.PARQUET)
+ .options(Map(
+ FORMAT_TABLE_PARTITION_SOURCE.key() -> (if (withCatalogManagedPartitions) "rest"
+ else "filesystem"),
+ FORMAT_TABLE_IMPLEMENTATION.key() -> "paimon").asJava)
+ // A table has catalog-managed partitions when it carries a partition manager, so a fixture
+ // claiming them must supply one.
+ .partitionManager(if (withCatalogManagedPartitions) {
+ FormatTablePartitionManager.create(
+ Identifier.create("test_db", "format_table"),
+ Collections.singletonList("dt"),
+ () => null)
+ } else {
+ null
+ })
+ .build()
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala
new file mode 100644
index 000000000000..a6ae2d4f0651
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/execution/FormatTablePartitionDdlPlanningTest.scala
@@ -0,0 +1,951 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.execution
+
+import org.apache.paimon.catalog.{CatalogContext, Identifier}
+import org.apache.paimon.fs.{FileIO, Path}
+import org.apache.paimon.fs.local.LocalFileIO
+import org.apache.paimon.options.Options
+import org.apache.paimon.partition.Partition
+import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase
+import org.apache.paimon.spark.catalyst.plans.logical.PaimonDropPartitions
+import org.apache.paimon.spark.format.PaimonFormatTable
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.table.format.FormatTablePartitionManager
+import org.apache.paimon.types.DataTypes
+
+import org.apache.spark.sql.{AnalysisException, Row}
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.{NoSuchPartitionsException, ResolvedPartitionSpec, ResolvedTable}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GenericInternalRow}
+import org.apache.spark.sql.catalyst.plans.logical.{AddPartitions, DropPartitions, RepairTable, ShowPartitions}
+import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier, TableCatalog}
+import org.apache.spark.sql.types.StringType
+
+import java.io.IOException
+import java.lang.reflect.InvocationTargetException
+import java.nio.file.Files
+import java.util.{Collections, List => JList, Map => JMap}
+import java.util.concurrent.{Callable, Executors, TimeUnit}
+
+import scala.collection.JavaConverters._
+
+class FormatTablePartitionDdlPlanningTest extends PaimonSparkTestWithRestCatalogBase {
+
+ test("strategy preserves the complete ADD batch for the catalog-managed partition command") {
+ val (table, _) = formatTable(withCatalogManagedPartitions = true)
+ val resolved = ResolvedTable.create(
+ spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog],
+ SparkIdentifier.of(Array("test"), "format_table"),
+ table)
+ val parts = Seq(partition(20260715, 10), partition(20260716, 11))
+
+ val plans = PaimonStrategy(spark).apply(AddPartitions(resolved, parts, ifNotExists = true))
+
+ assert(plans.size == 1)
+ val add = plans.head.asInstanceOf[PaimonAddFormatTablePartitionsExec]
+ assert(add.partSpecs == parts)
+ assert(add.ignoreIfExists)
+ }
+
+ test(
+ "catalog-managed ADD performs no client-side existence lookup and forwards one atomic batch") {
+ val (table, gateway) = formatTable(withCatalogManagedPartitions = true)
+ var refreshCalls = 0
+ val parts = Seq(partition(20260715, 10), partition(20260716, 11))
+ val command = PaimonAddFormatTablePartitionsExec(
+ table,
+ parts,
+ ignoreIfExists = true,
+ () => refreshCalls += 1)
+
+ runCommand(command)
+
+ assert(gateway.createCalls == 1)
+ assert(gateway.lookupCalls == 0)
+ assert(gateway.ignoreIfExists)
+ assert(
+ gateway.created.map(_.asScala.toMap) == Seq(
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260716", "hh" -> "11")))
+ assert(refreshCalls == 1)
+ }
+
+ test("mock service owns partial repeats, all repeats, atomic failure, and concurrent ADD") {
+ val gateway = new AtomicGateway(Set(Map("dt" -> "20260715", "hh" -> "10")))
+ val table = new PaimonFormatTable(rawFormatTable(withCatalogManagedPartitions = true, gateway))
+ val existing = partition(20260715, 10)
+ val added = partition(20260716, 11)
+
+ runCommand(
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ Seq(existing, added),
+ ignoreIfExists = true,
+ () => ()))
+ runCommand(
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ Seq(existing, added),
+ ignoreIfExists = true,
+ () => ()))
+
+ assert(
+ gateway.state == Set(
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260716", "hh" -> "11")))
+ assert(gateway.batches.take(2).forall(_.size == 2))
+
+ val ordinaryError = intercept[IllegalStateException] {
+ runCommand(
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ Seq(existing, partition(20260717, 12)),
+ ignoreIfExists = false,
+ () => ()))
+ }
+ assert(ordinaryError.getMessage.contains("already exists"))
+ assert(!gateway.state.exists(_("dt") == "20260717"))
+
+ val concurrent = partition(20260718, 13)
+ val executor = Executors.newFixedThreadPool(8)
+ try {
+ val tasks = (1 to 20).map {
+ _ =>
+ executor.submit(new Callable[Unit] {
+ override def call(): Unit = runCommand(
+ PaimonAddFormatTablePartitionsExec(
+ table,
+ Seq(concurrent),
+ ignoreIfExists = true,
+ () => ()))
+ })
+ }
+ tasks.foreach(_.get(30, TimeUnit.SECONDS))
+ } finally {
+ executor.shutdownNow()
+ }
+ assert(gateway.state.count(_("dt") == "20260718") == 1)
+ }
+
+ test(
+ "ADD and DROP always fail with an explicit unsupported error without catalog-managed partitions") {
+ val (table, gateway) = formatTable(withCatalogManagedPartitions = false)
+ val part = partition(20260715, 10)
+
+ val addError = intercept[UnsupportedOperationException] {
+ runCommand(
+ PaimonAddFormatTablePartitionsExec(table, Seq(part), ignoreIfExists = false, () => ()))
+ }
+ assert(addError.getMessage.contains("ADD PARTITION is supported only"))
+ assert(addError.getMessage.contains("catalog-managed"))
+
+ val dropError = intercept[UnsupportedOperationException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(part),
+ ifExists = true,
+ purge = false,
+ () => ()))
+ }
+ assert(dropError.getMessage.contains("DROP PARTITION is supported only"))
+ assert(dropError.getMessage.contains("catalog-managed"))
+ assert(gateway.createCalls == 0)
+ assert(gateway.dropCalls == 0)
+ assert(gateway.lookupCalls == 0)
+ }
+
+ test("SQL ADD PARTITION registers the partition and makes it queryable as empty") {
+ val tableName = "catalog_partition_format_add_sql"
+ withTable(tableName) {
+ sql(s"""CREATE TABLE $tableName (id INT, dt STRING, hh STRING)
+ |USING CSV
+ |PARTITIONED BY (dt, hh)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'format-table.partition-source' = 'rest')
+ |""".stripMargin)
+
+ sql(s"ALTER TABLE $tableName ADD PARTITION (dt='20260715', hh='10')").collect()
+
+ // The catalog is what makes the partition exist, and the directory is what lets a scan
+ // read it as empty instead of failing.
+ assert(registeredPartitionSpecs(tableName) == Set(Map("dt" -> "20260715", "hh" -> "10")))
+ assert(shownPartitions(tableName) == Set("dt=20260715/hh=10"))
+ checkAnswer(sql(s"SELECT * FROM $tableName"), Seq.empty[Row])
+
+ // Adding it again without IF NOT EXISTS is the catalog's rejection, not a local one.
+ intercept[Exception] {
+ sql(s"ALTER TABLE $tableName ADD PARTITION (dt='20260715', hh='10')").collect()
+ }
+ sql(s"ALTER TABLE $tableName ADD IF NOT EXISTS PARTITION (dt='20260715', hh='10')").collect()
+ assert(registeredPartitionSpecs(tableName) == Set(Map("dt" -> "20260715", "hh" -> "10")))
+ }
+ }
+
+ test("SQL partition DDL on filesystem-discovered partitions always fails with a clear error") {
+ val tableName = "filesystem_format_partition_ddl"
+ withTable(tableName) {
+ sql(
+ s"CREATE TABLE $tableName (id INT, dt INT, hh INT) USING CSV " +
+ "TBLPROPERTIES ('format-table.implementation'='paimon') PARTITIONED BY (dt, hh)")
+
+ val addError = intercept[Exception] {
+ sql(s"ALTER TABLE $tableName ADD PARTITION (dt=20260715, hh=10)").collect()
+ }
+ assert(causeMessages(addError).contains("ADD PARTITION is supported only"))
+
+ val dropError = intercept[Exception] {
+ sql(s"ALTER TABLE $tableName DROP IF EXISTS PARTITION (dt=20260715, hh=10)").collect()
+ }
+ assert(causeMessages(dropError).contains("DROP PARTITION is supported only"))
+
+ // Partial specs keep Spark's strict partition-spec resolution when partitions are
+ // discovered from the filesystem.
+ intercept[AnalysisException] {
+ sql(s"ALTER TABLE $tableName DROP PARTITION (dt=20260715)").collect()
+ }
+ }
+ }
+
+ test("rewrite routes catalog-managed DROP PARTITION through PaimonDropPartitions") {
+ val catalogManagedName = "catalog_partition_format_drop_rewrite"
+ val filesystemName = "filesystem_format_drop_rewrite"
+ withTable(catalogManagedName, filesystemName) {
+ sql(s"""CREATE TABLE $catalogManagedName (id INT, dt STRING, hh STRING)
+ |USING CSV
+ |PARTITIONED BY (dt, hh)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'format-table.partition-source' = 'rest')
+ |""".stripMargin)
+ sql(s"""CREATE TABLE $filesystemName (id INT, dt STRING, hh STRING)
+ |USING CSV
+ |PARTITIONED BY (dt, hh)
+ |TBLPROPERTIES ('format-table.implementation' = 'paimon')
+ |""".stripMargin)
+
+ // RewriteSparkDDLCommands runs inside the extension parser, so parsePlan already returns
+ // the rewritten plan for Format Tables with catalog-managed partitions.
+ val catalogManagedPlan = spark.sessionState.sqlParser.parsePlan(
+ s"ALTER TABLE $catalogManagedName DROP IF EXISTS PARTITION (dt='20260715')")
+ assert(catalogManagedPlan.isInstanceOf[PaimonDropPartitions])
+ val paimonDrop = catalogManagedPlan.asInstanceOf[PaimonDropPartitions]
+ assert(paimonDrop.ifExists)
+ assert(!paimonDrop.purge)
+
+ // Format Tables using filesystem partition discovery keep Spark's own DropPartitions and
+ // strict spec resolution.
+ val filesystemPlan = spark.sessionState.sqlParser.parsePlan(
+ s"ALTER TABLE $filesystemName DROP IF EXISTS PARTITION (dt='20260715')")
+ assert(filesystemPlan.isInstanceOf[DropPartitions])
+ }
+ }
+
+ test("SQL partial DROP with catalog-managed partitions unregisters the matching partitions") {
+ val tableName = "catalog_partition_format_partial_drop_sql"
+ withTable(tableName) {
+ sql(s"""CREATE TABLE $tableName (id INT, dt STRING, hh STRING)
+ |USING CSV
+ |PARTITIONED BY (dt, hh)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'format-table.partition-source' = 'rest')
+ |""".stripMargin)
+ registerPartitions(
+ tableName,
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260715", "hh" -> "11"),
+ Map("dt" -> "20260716", "hh" -> "10"))
+
+ // Leading partial spec expands to every registered leaf partition below it.
+ sql(s"ALTER TABLE $tableName DROP PARTITION (dt='20260715')")
+ assert(shownPartitions(tableName) == Set("dt=20260716/hh=10"))
+
+ // Complete specs honor IF EXISTS and fail loudly for unregistered partitions.
+ sql(s"ALTER TABLE $tableName DROP IF EXISTS PARTITION (dt='20990101', hh='00')")
+ val missingError = intercept[Exception] {
+ sql(s"ALTER TABLE $tableName DROP PARTITION (dt='20990101', hh='00')").collect()
+ }
+ assert(
+ Iterator
+ .iterate(missingError: Throwable)(_.getCause)
+ .takeWhile(_ != null)
+ .exists(_.isInstanceOf[NoSuchPartitionsException]))
+ assert(shownPartitions(tableName) == Set("dt=20260716/hh=10"))
+
+ // Non-leading partial specs resolve by partition name through the gateway.
+ sql(s"ALTER TABLE $tableName DROP PARTITION (hh='10')")
+ assert(shownPartitions(tableName) == Set.empty)
+ }
+ }
+
+ test(
+ "strategy preserves the DROP batch and catalog-managed DROP unregisters through the gateway") {
+ val (table, gateway) = formatTable(withCatalogManagedPartitions = true)
+ val resolved = ResolvedTable.create(
+ spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog],
+ SparkIdentifier.of(Array("test"), "format_table"),
+ table)
+ val parts = Seq(partition(20260715, 10))
+ val logical = PaimonDropPartitions(resolved, parts, true, false)
+
+ val plans = PaimonStrategy(spark).apply(logical)
+ assert(plans.size == 1)
+ val command = plans.head.asInstanceOf[PaimonDropFormatTablePartitionsExec]
+ assert(command.partSpecs == parts)
+ assert(command.ifExists)
+ assert(!command.purge)
+
+ var refreshCalls = 0
+ runCommand(command.copy(refreshCache = () => refreshCalls += 1))
+ assert(gateway.lookupCalls == 1)
+ assert(gateway.dropCalls == 1)
+ assert(gateway.dropped.map(_.asScala.toMap) == Seq(Map("dt" -> "20260715", "hh" -> "10")))
+ assert(refreshCalls == 1)
+ }
+
+ test("strategy threads IF EXISTS and PURGE flags into the format-table DROP command") {
+ val (table, _) = formatTable(withCatalogManagedPartitions = true)
+ val resolved = ResolvedTable.create(
+ spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog],
+ SparkIdentifier.of(Array("test"), "format_table"),
+ table)
+ val parts = Seq(partition(20260715, 10))
+
+ val plans =
+ PaimonStrategy(spark).apply(DropPartitions(resolved, parts, ifExists = false, purge = true))
+
+ assert(plans.size == 1)
+ val command = plans.head.asInstanceOf[PaimonDropFormatTablePartitionsExec]
+ assert(command.partSpecs == parts)
+ assert(!command.ifExists)
+ assert(command.purge)
+ }
+
+ test("catalog-managed DROP PARTITION PURGE is rejected before any catalog access") {
+ val (table, gateway) = formatTable(withCatalogManagedPartitions = true)
+
+ val error = intercept[UnsupportedOperationException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(partition(20260715, 10)),
+ ifExists = false,
+ purge = true,
+ () => fail("PURGE must not refresh the cache")))
+ }
+
+ assert(error.getMessage.contains("PURGE"))
+ assert(gateway.lookupCalls == 0)
+ assert(gateway.dropCalls == 0)
+ }
+
+ test("catalog-managed complete DROP without IF EXISTS fails for unregistered partitions") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("format-table-drop-unregistered").toUri)
+ val pendingDir = new Path(tablePath, "dt=20260715/hh=10")
+ var dropCalls = 0
+ var refreshCalls = 0
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ throw new AssertionError("Complete specs must resolve through list-by-names")
+ }
+
+ try {
+ fileIO.mkdirs(pendingDir)
+ val table = new PaimonFormatTable(
+ rawFormatTable(withCatalogManagedPartitions = true, gateway, tablePath.toString))
+
+ intercept[NoSuchPartitionsException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(partition(20260715, 10)),
+ ifExists = false,
+ purge = false,
+ () => refreshCalls += 1))
+ }
+
+ // The unregistered directory (e.g. awaiting MSCK registration) must survive untouched.
+ assert(dropCalls == 0)
+ assert(refreshCalls == 0)
+ assert(fileIO.exists(pendingDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test(
+ "catalog-managed DROP IF EXISTS drops only registered partitions and preserves pending data") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("format-table-drop-if-exists").toUri)
+ val registeredSpec = Map("dt" -> "20260716", "hh" -> "11")
+ val registeredDir = new Path(tablePath, "dt=20260716/hh=11")
+ val pendingDir = new Path(tablePath, "dt=20260715/hh=10")
+ var lookedUp = Seq.empty[Map[String, String]]
+ var dropped = Seq.empty[Map[String, String]]
+ var refreshCalls = 0
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] = {
+ lookedUp = partitions.asScala.map(_.asScala.toMap).toSeq
+ registeredPartitions(
+ partitions.asScala.map(_.asScala.toMap).filter(_ == registeredSpec).toSeq: _*)
+ }
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ throw new AssertionError("Complete specs must resolve through list-by-names")
+ }
+
+ try {
+ fileIO.mkdirs(pendingDir)
+ fileIO.mkdirs(registeredDir)
+ val table = new PaimonFormatTable(
+ rawFormatTable(withCatalogManagedPartitions = true, gateway, tablePath.toString))
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(partition(20260715, 10), partition(20260716, 11)),
+ ifExists = true,
+ purge = false,
+ () => refreshCalls += 1))
+
+ assert(
+ lookedUp == Seq(
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260716", "hh" -> "11")))
+ assert(dropped == Seq(registeredSpec))
+ assert(fileIO.exists(pendingDir))
+ assert(!fileIO.exists(registeredDir))
+ assert(refreshCalls == 1)
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed failed directory deletion stays unregistered and preserves pending data") {
+ val tablePath = new Path(Files.createTempDirectory("format-table-drop-retry-safe").toUri)
+ val failedSpec = Map("dt" -> "20260715", "hh" -> "10")
+ val pendingSpec = Map("dt" -> "20260716", "hh" -> "11")
+ val failedDir = new Path(tablePath, "dt=20260715/hh=10")
+ val pendingDir = new Path(tablePath, "dt=20260716/hh=11")
+ var failFirstDelete = true
+ val fileIO = new LocalFileIO {
+ override def delete(path: Path, recursive: Boolean): Boolean = {
+ if (path == failedDir && failFirstDelete) {
+ failFirstDelete = false
+ throw new IOException("injected exact partition delete failure")
+ }
+ super.delete(path, recursive)
+ }
+ }
+ var registered = Set(failedSpec)
+ var compensationCreates = Seq.empty[(Seq[Map[String, String]], Boolean)]
+ def newGateway(): FormatTablePartitionManager = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ val specs = partitions.asScala.map(_.asScala.toMap).toSeq
+ compensationCreates :+= ((specs, ignoreIfExists))
+ registered ++= specs
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ registered --= partitions.asScala.map(_.asScala.toMap)
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ registeredPartitions(partitions.asScala.map(_.asScala.toMap).filter(registered).toSeq: _*)
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ throw new AssertionError("Exact specs must resolve through list-by-names")
+ }
+ var refreshCalls = 0
+
+ try {
+ fileIO.mkdirs(failedDir)
+ fileIO.mkdirs(pendingDir)
+ val firstTable =
+ new PaimonFormatTable(
+ rawFormatTable(
+ withCatalogManagedPartitions = true,
+ newGateway(),
+ tablePath.toString,
+ fileIO))
+
+ val firstError = intercept[IOException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ firstTable,
+ Seq(partition(20260715, 10)),
+ ifExists = true,
+ purge = false,
+ () => refreshCalls += 1))
+ }
+ assert(firstError.getMessage.contains("injected exact partition delete failure"))
+ // The failed partition stays unregistered and is never recreated automatically.
+ assert(registered.isEmpty)
+ assert(compensationCreates.isEmpty)
+ assert(fileIO.exists(failedDir))
+ assert(fileIO.exists(pendingDir))
+ assert(refreshCalls == 1)
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed ambiguous catalog response does not recreate registration") {
+ val tablePath = new Path(Files.createTempDirectory("format-table-drop-ambiguous").toUri)
+ val droppedSpec = Map("dt" -> "20260715", "hh" -> "10")
+ val droppedDir = new Path(tablePath, "dt=20260715/hh=10")
+ val pendingDir = new Path(tablePath, "dt=20260716/hh=11")
+ val lostResponse = new IOException("injected lost catalog DROP response")
+ var failAfterCommit = true
+ var registered = Set(droppedSpec)
+ var compensationCreates = Seq.empty[(Seq[Map[String, String]], Boolean)]
+ def newGateway(): FormatTablePartitionManager = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ val specs = partitions.asScala.map(_.asScala.toMap).toSeq
+ compensationCreates :+= ((specs, ignoreIfExists))
+ registered ++= specs
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ registered --= partitions.asScala.map(_.asScala.toMap)
+ if (failAfterCommit) {
+ failAfterCommit = false
+ throw lostResponse
+ }
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ registeredPartitions(partitions.asScala.map(_.asScala.toMap).filter(registered).toSeq: _*)
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ throw new AssertionError("Exact specs must resolve through list-by-names")
+ }
+ val firstFileIO = LocalFileIO.create
+ var refreshCalls = 0
+
+ try {
+ firstFileIO.mkdirs(droppedDir)
+ firstFileIO.mkdirs(pendingDir)
+ val firstTable = new PaimonFormatTable(
+ rawFormatTable(
+ withCatalogManagedPartitions = true,
+ newGateway(),
+ tablePath.toString,
+ firstFileIO))
+
+ val firstError = intercept[IOException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ firstTable,
+ Seq(partition(20260715, 10)),
+ ifExists = true,
+ purge = false,
+ () => refreshCalls += 1))
+ }
+
+ assert(firstError eq lostResponse)
+ assert(registered.isEmpty)
+ assert(compensationCreates.isEmpty)
+ assert(firstFileIO.exists(droppedDir))
+ assert(firstFileIO.exists(pendingDir))
+ assert(refreshCalls == 1)
+ } finally {
+ firstFileIO.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed DROP resolves a non-leading partial spec by partition name") {
+ var listedPrefixes = Seq.empty[Map[String, String]]
+ var dropped = Seq.empty[Map[String, String]]
+ val matching = Map("dt" -> "20260715", "hh" -> "10")
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] = {
+ listedPrefixes :+= prefix.asScala.toMap
+ registeredPartitions(matching)
+ }
+ }
+ val table = new PaimonFormatTable(rawFormatTable(withCatalogManagedPartitions = true, gateway))
+ val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10)))
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(hhOnly),
+ ifExists = false,
+ purge = false,
+ () => ()))
+
+ assert(listedPrefixes == Seq(Map.empty))
+ assert(dropped == Seq(matching))
+ }
+
+ test("catalog-managed multiple non-leading partial DROP specs share one catalog traversal") {
+ val first = Map("dt" -> "20260715", "hh" -> "10")
+ val second = Map("dt" -> "20260716", "hh" -> "11")
+ val third = Map("dt" -> "20260717", "hh" -> "10")
+ val unrelated = Map("dt" -> "20260718", "hh" -> "12")
+ var listedPrefixes = Seq.empty[Map[String, String]]
+ var dropped = Seq.empty[Map[String, String]]
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] = {
+ listedPrefixes :+= prefix.asScala.toMap
+ registeredPartitions(first, unrelated, second, third)
+ }
+ }
+ val table = new PaimonFormatTable(rawFormatTable(withCatalogManagedPartitions = true, gateway))
+ val hhTen = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10)))
+ val hhEleven = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](11)))
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(hhTen, hhEleven),
+ ifExists = false,
+ purge = false,
+ () => ()))
+
+ // Every partial spec is resolved from a single unfiltered listing.
+ assert(listedPrefixes == Seq(Map.empty))
+ assert(dropped == Seq(first, second, third))
+ assert(dropped.distinct == dropped)
+ }
+
+ test("catalog-managed DROP with an empty batch does not refresh cache") {
+ val (table, gateway) = formatTable(withCatalogManagedPartitions = true)
+ var refreshCalls = 0
+
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq.empty,
+ ifExists = false,
+ purge = false,
+ () => refreshCalls += 1))
+
+ assert(gateway.dropCalls == 0)
+ assert(refreshCalls == 0)
+ }
+
+ test("catalog-managed partial DROP rejects an incomplete catalog result before mutation") {
+ var dropCalls = 0
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ registeredPartitions(Map("dt" -> "20260715"))
+ }
+ val table = new PaimonFormatTable(rawFormatTable(withCatalogManagedPartitions = true, gateway))
+ val hhOnly = ResolvedPartitionSpec(Seq("hh"), new GenericInternalRow(Array[Any](10)))
+
+ val error = intercept[IllegalStateException] {
+ runCommand(
+ PaimonDropFormatTablePartitionsExec(
+ table,
+ Seq(hhOnly),
+ ifExists = false,
+ purge = false,
+ () => ()))
+ }
+
+ assert(error.getMessage.contains("complete partition spec"))
+ assert(dropCalls == 0)
+ }
+
+ test("MSCK strategy intercepts only catalog-managed partitions and maps the mode flags") {
+ val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog]
+ val identifier = SparkIdentifier.of(Array("test"), "format_table")
+ val (catalogManaged, _) = formatTable(withCatalogManagedPartitions = true)
+ val (filesystem, _) = formatTable(withCatalogManagedPartitions = false)
+ val catalogManagedResolved = ResolvedTable.create(catalog, identifier, catalogManaged)
+
+ // (plain MSCK) -> ADD; DROP PARTITIONS -> DROP; SYNC PARTITIONS -> both.
+ Seq((true, false), (false, true), (true, true)).foreach {
+ case (add, drop) =>
+ val plans = PaimonStrategy(spark).apply(RepairTable(catalogManagedResolved, add, drop))
+ assert(plans.size == 1)
+ val repair = plans.head.asInstanceOf[PaimonRepairFormatTablePartitionsExec]
+ assert(repair.addPartitions == add)
+ assert(repair.dropPartitions == drop)
+ }
+
+ // Tables using filesystem partition discovery keep Spark's own v2 rejection: no
+ // interception.
+ val filesystemPlans = PaimonStrategy(spark)
+ .apply(RepairTable(ResolvedTable.create(catalog, identifier, filesystem), true, false))
+ assert(filesystemPlans.isEmpty)
+ }
+
+ test("MSCK repair reuses the sync engine: DROP unregisters catalog-only partitions") {
+ val gateway = new AtomicGateway(Set(Map("dt" -> "20260715", "hh" -> "10")))
+ val table = new PaimonFormatTable(rawFormatTable(withCatalogManagedPartitions = true, gateway))
+ var refreshCalls = 0
+
+ // The table location has no partition directories, so the registered partition is
+ // catalog-only; MSCK DROP PARTITIONS must unregister it (metadata-only).
+ runCommand(
+ PaimonRepairFormatTablePartitionsExec(
+ table,
+ addPartitions = false,
+ dropPartitions = true,
+ () => refreshCalls += 1))
+
+ assert(gateway.state.isEmpty)
+ assert(refreshCalls == 1)
+ }
+
+ test("strategy leaves Format Table SHOW PARTITIONS to Spark's default executor") {
+ val output = Seq(AttributeReference("partition", StringType, nullable = false)())
+ val (catalogManaged, _) = formatTable(withCatalogManagedPartitions = true)
+ val (filesystem, _) = formatTable(withCatalogManagedPartitions = false)
+ val catalog = spark.sessionState.catalogManager.currentCatalog.asInstanceOf[TableCatalog]
+ val identifier = SparkIdentifier.of(Array("test"), "format_table")
+ val spec = Some(partition(20260715, 10))
+
+ // SHOW PARTITIONS is served by Spark's default v2 executor via listPartitionIdentifiers
+ // (backed by the catalog listing when partitions are catalog-managed), exactly like every
+ // other Paimon
+ // table, so the strategy must not intercept it.
+ assert(
+ PaimonStrategy(spark)
+ .apply(
+ ShowPartitions(ResolvedTable.create(catalog, identifier, catalogManaged), spec, output))
+ .isEmpty)
+ assert(
+ PaimonStrategy(spark)
+ .apply(ShowPartitions(ResolvedTable.create(catalog, identifier, filesystem), spec, output))
+ .isEmpty)
+ }
+
+ private def partition(dt: Int, hh: Int): ResolvedPartitionSpec =
+ ResolvedPartitionSpec(Seq("dt", "hh"), new GenericInternalRow(Array[Any](dt, hh)))
+
+ /** Partition metadata as a catalog returns it; only the spec matters for these tests. */
+ private def registeredPartitions(specs: Map[String, String]*): JList[Partition] =
+ specs.map(spec => new Partition(spec.asJava, 0L, 0L, 0L, 0L, 0, false)).asJava
+
+ private def formatTable(
+ withCatalogManagedPartitions: Boolean): (PaimonFormatTable, RecordingGateway) = {
+ val gateway = new RecordingGateway
+ (new PaimonFormatTable(rawFormatTable(withCatalogManagedPartitions, gateway)), gateway)
+ }
+
+ private def rawFormatTable(
+ withCatalogManagedPartitions: Boolean,
+ gateway: FormatTablePartitionManager): FormatTable =
+ rawFormatTable(
+ withCatalogManagedPartitions,
+ gateway,
+ new Path(Files.createTempDirectory("format-table-ddl-planning-test").toUri).toString)
+
+ private def rawFormatTable(
+ withCatalogManagedPartitions: Boolean,
+ gateway: FormatTablePartitionManager,
+ location: String): FormatTable = {
+ rawFormatTable(withCatalogManagedPartitions, gateway, location, LocalFileIO.create)
+ }
+
+ private def rawFormatTable(
+ withCatalogManagedPartitions: Boolean,
+ gateway: FormatTablePartitionManager,
+ location: String,
+ fileIO: FileIO): FormatTable = {
+ val options = if (withCatalogManagedPartitions) {
+ Map("format-table.partition-source" -> "rest")
+ } else {
+ Map.empty[String, String]
+ }
+ FormatTable
+ .builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("test", "format_table"))
+ .rowType(
+ DataTypes.ROW(
+ DataTypes.FIELD(0, "id", DataTypes.INT()),
+ DataTypes.FIELD(1, "dt", DataTypes.INT()),
+ DataTypes.FIELD(2, "hh", DataTypes.INT())))
+ .partitionKeys(Seq("dt", "hh").asJava)
+ .location(location)
+ .format(FormatTable.Format.CSV)
+ .options(options.asJava)
+ .catalogContext(CatalogContext.create(new Options))
+ // A table uses catalog-managed partitions exactly when it carries a partition manager,
+ // so a filesystem-discovered one must not have any.
+ .partitionManager(if (withCatalogManagedPartitions) gateway else null)
+ .build()
+ }
+
+ private def registerPartitions(tableName: String, specs: Map[String, String]*): Unit =
+ paimonCatalog.createPartitions(
+ Identifier.create(dbName0, tableName),
+ specs.map(_.asJava).asJava,
+ true)
+
+ private def registeredPartitionSpecs(tableName: String): Set[Map[String, String]] =
+ paimonCatalog
+ .listPartitions(Identifier.create(dbName0, tableName))
+ .asScala
+ .map(_.spec().asScala.toMap)
+ .toSet
+
+ private def shownPartitions(tableName: String): Set[String] =
+ sql(s"SHOW PARTITIONS $tableName").collect().map(_.getString(0)).toSet
+
+ private def runCommand(command: AnyRef): Unit = {
+ try {
+ command.getClass.getMethod("run").invoke(command)
+ } catch {
+ case e: InvocationTargetException => throw e.getCause
+ }
+ }
+
+ private def causeMessages(error: Throwable): String = {
+ Iterator
+ .iterate(error)(_.getCause)
+ .takeWhile(_ != null)
+ .flatMap(e => Option(e.getMessage))
+ .mkString(" | ")
+ }
+
+ private class RecordingGateway extends FormatTablePartitionManager {
+ var createCalls = 0
+ var lookupCalls = 0
+ var created = Seq.empty[JMap[String, String]]
+ var ignoreIfExists = false
+ var dropCalls = 0
+ var dropped = Seq.empty[JMap[String, String]]
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ createCalls += 1
+ created = partitions.asScala.toSeq
+ this.ignoreIfExists = ignoreIfExists
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropCalls += 1
+ dropped = partitions.asScala.toSeq
+ }
+
+ // Reports every requested spec as registered. ADD tests assert lookupCalls == 0 because
+ // Catalog-managed ADD must never perform a client-side existence lookup.
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] = {
+ lookupCalls += 1
+ registeredPartitions(partitions.asScala.map(_.asScala.toMap).toSeq: _*)
+ }
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ Collections.emptyList()
+ }
+
+ private class AtomicGateway(initial: Set[Map[String, String]])
+ extends FormatTablePartitionManager {
+
+ private var partitions = initial
+ var batches = Seq.empty[Seq[Map[String, String]]]
+
+ def state: Set[Map[String, String]] = synchronized(partitions)
+
+ override def createPartitions(
+ partitionsToCreate: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = synchronized {
+ val batch = partitionsToCreate.asScala.map(_.asScala.toMap).toSeq
+ batches :+= batch
+ val duplicates = batch.filter(partitions.contains)
+ if (duplicates.nonEmpty && !ignoreIfExists) {
+ throw new IllegalStateException(s"Partition already exists: ${duplicates.head}")
+ }
+ partitions ++= batch
+ }
+
+ override def dropPartitions(partitionsToDrop: JList[JMap[String, String]]): Unit =
+ synchronized {
+ partitions --= partitionsToDrop.asScala.map(_.asScala.toMap)
+ }
+
+ override def listPartitionsByNames(partitions: JList[JMap[String, String]]): JList[Partition] =
+ throw new AssertionError("ADD must not perform a client-side existence lookup")
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] = synchronized {
+ registeredPartitions(partitions.toSeq: _*)
+ }
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogManagedPartitionLoadTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogManagedPartitionLoadTest.scala
new file mode 100644
index 000000000000..58b6d51e7cab
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/CatalogManagedPartitionLoadTest.scala
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format
+
+import org.apache.paimon.spark.{PaimonSparkTestWithRestCatalogBase, SparkCatalog}
+
+import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier}
+
+class CatalogManagedPartitionLoadTest extends PaimonSparkTestWithRestCatalogBase {
+
+ test(
+ "a Format Table with catalog-managed partitions loaded through SparkCatalog carries its partition manager") {
+ val tableName = "catalog_partition_format_table_load"
+ withTable(tableName) {
+ sql(
+ s"CREATE TABLE $tableName (id INT, dt STRING) USING CSV " +
+ "TBLPROPERTIES ('format-table.implementation'='paimon', " +
+ "'format-table.partition-source'='rest') PARTITIONED BY (dt)")
+
+ val sparkCatalog = spark.sessionState.catalogManager
+ .catalog("paimon")
+ .asInstanceOf[SparkCatalog]
+ val sparkTable = sparkCatalog
+ .loadTable(SparkIdentifier.of(Array(dbName0), tableName))
+ .asInstanceOf[PaimonFormatTable]
+
+ assert(sparkTable.partitionManager != null)
+ }
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala
new file mode 100644
index 000000000000..9edba592e26e
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/format/FormatTablePartitionManagementTest.scala
@@ -0,0 +1,686 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.format
+
+import org.apache.paimon.catalog.{CatalogContext, Identifier}
+import org.apache.paimon.fs.{FileIO, Path}
+import org.apache.paimon.fs.local.LocalFileIO
+import org.apache.paimon.options.Options
+import org.apache.paimon.partition.Partition
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.table.format.FormatTablePartitionManager
+import org.apache.paimon.types.DataTypes
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.GenericInternalRow
+import org.apache.spark.unsafe.types.UTF8String
+
+import java.lang.reflect.{InvocationHandler, InvocationTargetException, Method, Proxy}
+import java.nio.file.Files
+import java.util.{ArrayList, Collections, List => JList, Map => JMap}
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+class FormatTablePartitionManagementTest extends SparkFunSuite {
+
+ test(
+ "catalog-managed ADD forwards the complete batch and ignoreIfExists to the partition catalog") {
+ var forwardedPartitions = Seq.empty[JMap[String, String]]
+ var forwardedIgnoreIfExists = false
+
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ forwardedPartitions = partitions.asScala.toSeq
+ forwardedIgnoreIfExists = ignoreIfExists
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {}
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ Collections.emptyList()
+ }
+ val sparkTable =
+ new PaimonFormatTable(formatTableWithCatalogManagedPartitions(partitionManager = gateway))
+
+ val rows = Array[InternalRow](
+ new GenericInternalRow(Array[Any](20260715, 10)),
+ new GenericInternalRow(Array[Any](20260716, 11)))
+ val properties = Array.fill[JMap[String, String]](2)(Collections.emptyMap())
+ sparkTable.createFormatTablePartitions(rows, properties, ignoreIfExists = true)
+
+ assert(forwardedIgnoreIfExists)
+ assert(
+ forwardedPartitions.map(_.asScala.toMap) == Seq(
+ Map("dt" -> "20260715", "hh" -> "10"),
+ Map("dt" -> "20260716", "hh" -> "11")))
+ }
+
+ test("catalog-managed ADD creates the partition directory") {
+ val fileIO = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("catalog-partition-format-add-mkdirs").toUri)
+ val table = formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, emptyGateway)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+ try {
+ val sparkTable = new PaimonFormatTable(table)
+ sparkTable.createFormatTablePartitions(
+ Array(partitionRow(20260715, 10)),
+ Array[JMap[String, String]](Collections.emptyMap()),
+ ignoreIfExists = true)
+
+ // ADD PARTITION creates the directory client-side, so a scan sees an empty partition
+ // rather than failing on a missing directory (Hive ADD PARTITION semantics).
+ assert(fileIO.exists(partitionDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed DROP rejects a partition value that would escape the table location") {
+ var dropCalls = 0
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ Collections.emptyList()
+ }
+
+ val sparkTable =
+ new PaimonFormatTable(
+ stringFormatTableWithCatalogManagedPartitions(gateway, onlyValueInPath = true))
+ val error = intercept[IllegalArgumentException] {
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt")),
+ Array(new GenericInternalRow(Array[Any](UTF8String.fromString("..")))))
+ }
+
+ // Traversal values are rejected before any catalog mutation, so no partition is unregistered.
+ assert(error.getMessage.contains(".."))
+ assert(dropCalls == 0)
+ }
+
+ test("catalog-managed ADD with LOCATION is rejected before any catalog RPC") {
+ var createCalls = 0
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = createCalls += 1
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {}
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ Collections.emptyList()
+ }
+
+ val sparkTable =
+ new PaimonFormatTable(formatTableWithCatalogManagedPartitions(partitionManager = gateway))
+ val error = intercept[UnsupportedOperationException] {
+ sparkTable.createFormatTablePartitions(
+ Array(new GenericInternalRow(Array[Any](20260715, 10))),
+ Array(Map("location" -> "file:/tmp/custom").asJava),
+ ignoreIfExists = true)
+ }
+
+ assert(error.getMessage.contains("LOCATION"))
+ assert(createCalls == 0)
+ }
+
+ test(
+ "catalog-managed DROP PARTITION unregisters first and then deletes the partition directory") {
+ var dropped = Seq.empty[JMap[String, String]]
+ var directoryExistedAtUnregister = false
+ val fileIO = LocalFileIO.create()
+ val tablePath =
+ new Path(Files.createTempDirectory("catalog-partition-format-drop-ordering").toUri)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ // Ordering contract: the directory must still exist when the catalog unregisters.
+ directoryExistedAtUnregister = fileIO.exists(partitionDir)
+ dropped = partitions.asScala.toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ Collections.emptyList()
+ }
+
+ try {
+ fileIO.mkdirs(partitionDir)
+ assert(fileIO.exists(partitionDir))
+
+ val sparkTable =
+ new PaimonFormatTable(
+ formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway))
+ assert(
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt", "hh")),
+ Array(new GenericInternalRow(Array[Any](20260715, 10)))))
+
+ assert(dropped.map(_.asScala.toMap) == Seq(Map("dt" -> "20260715", "hh" -> "10")))
+ assert(directoryExistedAtUnregister)
+ assert(!fileIO.exists(partitionDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed direct partial DROP expands leading values to exact leaf partitions") {
+ val fileIO = LocalFileIO.create()
+ val tablePath =
+ new Path(Files.createTempDirectory("catalog-partition-format-direct-partial-drop").toUri)
+ val dtDir = new Path(tablePath, "dt=20260715")
+ val firstDir = new Path(dtDir, "hh=10")
+ val secondDir = new Path(dtDir, "hh=11")
+ val first = partitionSpec(20260715, 10)
+ val second = partitionSpec(20260715, 11)
+ var listedPrefixes = Seq.empty[Map[String, String]]
+ var dropped = Seq.empty[Map[String, String]]
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] = {
+ listedPrefixes :+= prefix.asScala.toMap
+ registeredPartitions(first, second)
+ }
+ }
+
+ try {
+ fileIO.mkdirs(firstDir)
+ fileIO.mkdirs(secondDir)
+ val sparkTable =
+ new PaimonFormatTable(
+ formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway))
+
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt")),
+ Array(new GenericInternalRow(Array[Any](20260715))))
+
+ // The expansion always runs one unfiltered traversal; constraints apply client-side.
+ assert(listedPrefixes == Seq(Map.empty))
+ assert(dropped == Seq(first, second))
+ assert(!fileIO.exists(firstDir))
+ assert(!fileIO.exists(secondDir))
+ assert(fileIO.exists(dtDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed DROP fails when FileIO reports an undeleted partition directory") {
+ val delegate = LocalFileIO.create()
+ val tablePath = new Path(Files.createTempDirectory("catalog-partition-format-drop").toUri)
+ val fileIO = falseDeleteFileIO(delegate)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+ delegate.mkdirs(partitionDir)
+ val gateway = new RecordingDropCatalog
+ val table = formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway)
+
+ try {
+ val error = intercept[java.io.IOException] {
+ new PaimonFormatTable(table)
+ .dropFormatTablePartitions(
+ Array(Array("dt", "hh")),
+ Array(new GenericInternalRow(Array[Any](20260715, 10))))
+ }
+
+ assert(error.getMessage.contains("was not deleted"))
+ assert(error.getMessage.contains("dt=20260715"))
+ assert(error.getMessage.contains("hh=10"))
+ assert(gateway.dropCalls == 1)
+ assert(delegate.exists(partitionDir))
+ } finally {
+ delegate.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed DROP treats an already missing partition directory as deleted") {
+ val delegate = LocalFileIO.create()
+ val tablePath =
+ new Path(Files.createTempDirectory("catalog-partition-format-drop-missing").toUri)
+ val fileIO = falseDeleteFileIO(delegate)
+ val gateway = new RecordingDropCatalog
+ val table = formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway)
+
+ try {
+ assert(
+ new PaimonFormatTable(table)
+ .dropFormatTablePartitions(
+ Array(Array("dt", "hh")),
+ Array(new GenericInternalRow(Array[Any](20260715, 10)))))
+ assert(gateway.dropCalls == 1)
+ } finally {
+ delegate.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed DROP deduplicates overlapping full and partial specs before deleting") {
+ val delegate = LocalFileIO.create()
+ val tablePath =
+ new Path(Files.createTempDirectory("catalog-partition-format-drop-overlap").toUri)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+ var deleteCalls = 0
+ val fileIO = Proxy
+ .newProxyInstance(
+ classOf[FileIO].getClassLoader,
+ Array(classOf[FileIO]),
+ new InvocationHandler {
+ override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = {
+ if (method.getName == "delete" && args(0) == partitionDir) {
+ deleteCalls += 1
+ }
+ try {
+ method.invoke(delegate, Option(args).getOrElse(Array.empty[AnyRef]): _*)
+ } catch {
+ case error: InvocationTargetException => throw error.getCause
+ }
+ }
+ }
+ )
+ .asInstanceOf[FileIO]
+ val matching = partitionSpec(20260715, 10)
+ var dropped = Seq.empty[Map[String, String]]
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropped = partitions.asScala.map(_.asScala.toMap).toSeq
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ registeredPartitions(matching)
+ }
+
+ try {
+ delegate.mkdirs(partitionDir)
+ val sparkTable =
+ new PaimonFormatTable(
+ formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway))
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt", "hh"), Array("hh")),
+ Array(partitionRow(20260715, 10), new GenericInternalRow(Array[Any](10))))
+
+ assert(dropped == Seq(matching))
+ assert(deleteCalls == 1)
+ assert(!delegate.exists(partitionDir))
+ assert(delegate.exists(new Path(tablePath, "dt=20260715")))
+ } finally {
+ delegate.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed partial DROP leaves catalog and data untouched when the listing fails") {
+ val fileIO = LocalFileIO.create()
+ val tablePath =
+ new Path(Files.createTempDirectory("catalog-partition-format-drop-list-failure").toUri)
+ val partitionDir = new Path(tablePath, "dt=20260715/hh=10")
+ var dropCalls = 0
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ throw new TestPartitionListException
+ }
+
+ try {
+ fileIO.mkdirs(partitionDir)
+ val sparkTable =
+ new PaimonFormatTable(
+ formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway))
+
+ intercept[TestPartitionListException] {
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("hh")),
+ Array(new GenericInternalRow(Array[Any](10))))
+ }
+
+ assert(dropCalls == 0)
+ assert(fileIO.exists(partitionDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed partial DROP keeps data untouched after an ambiguous catalog response") {
+ val fileIO = LocalFileIO.create()
+ val tablePath =
+ new Path(Files.createTempDirectory("catalog-partition-format-drop-response-loss").toUri)
+ val firstDir = new Path(tablePath, "dt=20260715/hh=10")
+ val secondDir = new Path(tablePath, "dt=20260715/hh=11")
+ val first = partitionSpec(20260715, 10)
+ val second = partitionSpec(20260715, 11)
+ val registered = mutable.LinkedHashSet(first, second)
+ val responseFailure = new TestCatalogDropResponseException
+ var dropCalls = 0
+ var failDropResponse = true
+ val gateway = new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit =
+ throw new AssertionError("An ambiguous DROP must not recreate catalog partitions")
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropCalls += 1
+ registered --= partitions.asScala.map(_.asScala.toMap)
+ if (failDropResponse) {
+ failDropResponse = false
+ throw responseFailure
+ }
+ }
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ registeredPartitions(registered.toSeq: _*)
+ }
+
+ try {
+ fileIO.mkdirs(firstDir)
+ fileIO.mkdirs(secondDir)
+ val sparkTable =
+ new PaimonFormatTable(
+ formatTableWithCatalogManagedPartitions(fileIO, tablePath.toString, gateway))
+
+ val error = intercept[TestCatalogDropResponseException] {
+ sparkTable.dropFormatTablePartitions(
+ Array(Array("dt")),
+ Array(new GenericInternalRow(Array[Any](20260715))))
+ }
+
+ assert(error eq responseFailure)
+ assert(dropCalls == 1)
+ assert(registered.isEmpty)
+ assert(fileIO.exists(firstDir))
+ assert(fileIO.exists(secondDir))
+ } finally {
+ fileIO.delete(tablePath, true)
+ }
+ }
+
+ test("catalog-managed ADD forwards each batch whole, without deduplicating across calls") {
+ val first = partitionSpec(20260715, 10)
+ val second = partitionSpec(20260716, 11)
+ val third = partitionSpec(20260717, 12)
+ val gateway = new InMemoryPartitionManager(Seq(first, second))
+ val sparkTable =
+ new PaimonFormatTable(formatTableWithCatalogManagedPartitions(partitionManager = gateway))
+
+ sparkTable.createFormatTablePartitions(
+ Array(partitionRow(20260715, 10), partitionRow(20260716, 11)),
+ Array.fill[JMap[String, String]](2)(Collections.emptyMap()),
+ ignoreIfExists = true)
+ sparkTable.createFormatTablePartitions(
+ Array(partitionRow(20260716, 11), partitionRow(20260717, 12)),
+ Array.fill[JMap[String, String]](2)(Collections.emptyMap()),
+ ignoreIfExists = true)
+
+ // Both batches reach the catalog exactly as asked, including the partition the previous
+ // call already created: resolving duplicates is the catalog's job, not Spark's.
+ assert(gateway.createRequests == Seq((Seq(first, second), true), (Seq(second, third), true)))
+ assert(gateway.lookupCalls == 0)
+ }
+
+ test("catalog-managed strict ADD asks the catalog to reject duplicates, without looking first") {
+ val existing = partitionSpec(20260715, 10)
+ val gateway = new InMemoryPartitionManager(Seq(existing))
+ val sparkTable =
+ new PaimonFormatTable(formatTableWithCatalogManagedPartitions(partitionManager = gateway))
+
+ val error = intercept[TestPartitionAlreadyExistsException] {
+ sparkTable.createFormatTablePartitions(
+ Array(partitionRow(20260715, 10)),
+ Array[JMap[String, String]](Collections.emptyMap()),
+ ignoreIfExists = false)
+ }
+
+ assert(error.partition == existing)
+ // Rejecting the batch is the catalog's decision; Spark must not pre-empt it with a lookup,
+ // which is what would break the batch's atomicity.
+ assert(gateway.lookupCalls == 0)
+ }
+
+ test("catalog-managed ADD preserves typed special partition values") {
+ val gateway = new InMemoryPartitionManager
+ val sparkTable = new PaimonFormatTable(stringFormatTableWithCatalogManagedPartitions(gateway))
+ val values = Seq("a/b", "a=b", "a%", "中文 value")
+ val rows: Seq[InternalRow] = values
+ .map(
+ value => new GenericInternalRow(Array[Any](UTF8String.fromString(value))): InternalRow) :+
+ new GenericInternalRow(Array[Any](null))
+
+ sparkTable.createFormatTablePartitions(
+ rows.toArray,
+ Array.fill[JMap[String, String]](rows.size)(Collections.emptyMap()),
+ ignoreIfExists = true)
+
+ assert(
+ gateway.createRequests == Seq(
+ (values.map(value => Map("dt" -> value)) :+ Map("dt" -> "__DEFAULT_PARTITION__"), true)))
+ }
+
+ private def emptyGateway: FormatTablePartitionManager =
+ new FormatTablePartitionManager {
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {}
+
+ override def listPartitionsByNames(
+ partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ Collections.emptyList()
+ }
+
+ private class InMemoryPartitionManager(initialPartitions: Seq[Map[String, String]] = Seq.empty)
+ extends FormatTablePartitionManager {
+
+ private val storedPartitions = mutable.LinkedHashSet(initialPartitions: _*)
+ private var requests = Seq.empty[(Seq[Map[String, String]], Boolean)]
+ var lookupCalls = 0
+
+ def createRequests: Seq[(Seq[Map[String, String]], Boolean)] = synchronized(requests)
+
+ def partitions: Seq[Map[String, String]] = synchronized(storedPartitions.toSeq)
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = synchronized {
+ val requested = partitions.asScala.map(_.asScala.toMap).toSeq
+ requests :+= ((requested, ignoreIfExists))
+ if (!ignoreIfExists) {
+ requested.find(storedPartitions.contains).foreach {
+ duplicate => throw new TestPartitionAlreadyExistsException(duplicate)
+ }
+ }
+ storedPartitions ++= requested
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = synchronized {
+ storedPartitions --= partitions.asScala.map(_.asScala.toMap)
+ }
+
+ override def listPartitionsByNames(partitions: JList[JMap[String, String]]): JList[Partition] =
+ synchronized {
+ lookupCalls += 1
+ registeredPartitions(
+ partitions.asScala.map(_.asScala.toMap).filter(storedPartitions.contains).toSeq: _*)
+ }
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ throw new AssertionError("ADD must not enumerate catalog or filesystem partitions")
+ }
+
+ private class TestPartitionAlreadyExistsException(val partition: Map[String, String])
+ extends RuntimeException
+
+ private class TestPartitionListException extends RuntimeException
+
+ private class TestCatalogDropResponseException extends RuntimeException
+
+ private class RecordingDropCatalog extends FormatTablePartitionManager {
+ var dropCalls = 0
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {}
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = dropCalls += 1
+
+ override def listPartitionsByNames(partitions: JList[JMap[String, String]]): JList[Partition] =
+ Collections.emptyList()
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] =
+ Collections.emptyList()
+ }
+
+ private def partitionRow(dt: Int, hh: Int): InternalRow =
+ new GenericInternalRow(Array[Any](dt, hh))
+
+ private def partitionSpec(dt: Int, hh: Int): Map[String, String] =
+ Map("dt" -> dt.toString, "hh" -> hh.toString)
+
+ /** Partition metadata as a catalog returns it; only the spec matters for these tests. */
+ private def registeredPartitions(specs: Map[String, String]*): JList[Partition] =
+ specs.map(spec => new Partition(spec.asJava, 0L, 0L, 0L, 0L, 0, false)).asJava
+
+ private def uniqueTableLocation(prefix: String): String =
+ new Path(Files.createTempDirectory(prefix).toUri).toString
+
+ private def formatTableWithCatalogManagedPartitions(
+ fileIO: FileIO = LocalFileIO.create,
+ location: String = uniqueTableLocation("format_table"),
+ partitionManager: FormatTablePartitionManager = null): FormatTable = {
+ FormatTable
+ .builder()
+ .fileIO(fileIO)
+ .identifier(Identifier.create("test_db", "format_table"))
+ .rowType(
+ DataTypes.ROW(
+ DataTypes.FIELD(0, "id", DataTypes.INT()),
+ DataTypes.FIELD(1, "dt", DataTypes.INT()),
+ DataTypes.FIELD(2, "hh", DataTypes.INT())))
+ .partitionKeys(Seq("dt", "hh").asJava)
+ .location(location)
+ .format(FormatTable.Format.CSV)
+ .options(Map("format-table.partition-source" -> "rest").asJava)
+ .catalogContext(CatalogContext.create(new Options))
+ .partitionManager(partitionManager)
+ .build()
+ }
+
+ private def falseDeleteFileIO(delegate: FileIO): FileIO = {
+ Proxy
+ .newProxyInstance(
+ classOf[FileIO].getClassLoader,
+ Array(classOf[FileIO]),
+ new InvocationHandler {
+ override def invoke(proxy: Any, method: Method, args: Array[AnyRef]): AnyRef = {
+ if (method.getName == "delete" && method.getParameterCount == 2) {
+ java.lang.Boolean.FALSE
+ } else {
+ try {
+ method.invoke(delegate, Option(args).getOrElse(Array.empty[AnyRef]): _*)
+ } catch {
+ case error: InvocationTargetException => throw error.getCause
+ }
+ }
+ }
+ }
+ )
+ .asInstanceOf[FileIO]
+ }
+
+ private def stringFormatTableWithCatalogManagedPartitions(
+ partitionManager: FormatTablePartitionManager,
+ onlyValueInPath: Boolean = false): FormatTable = {
+ FormatTable
+ .builder()
+ .fileIO(LocalFileIO.create)
+ .identifier(Identifier.create("test_db", "format_table_special"))
+ .rowType(DataTypes.ROW(
+ DataTypes.FIELD(0, "id", DataTypes.INT()),
+ DataTypes.FIELD(1, "dt", DataTypes.STRING())))
+ .partitionKeys(Seq("dt").asJava)
+ .location(uniqueTableLocation("format_table_special"))
+ .format(FormatTable.Format.CSV)
+ .options(Map(
+ "format-table.partition-source" -> "rest",
+ "format-table.partition-path-only-value" -> onlyValueInPath.toString).asJava)
+ .catalogContext(CatalogContext.create(new Options))
+ .partitionManager(partitionManager)
+ .build()
+ }
+}
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionMsckRepairTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionMsckRepairTest.scala
new file mode 100644
index 000000000000..151875609d29
--- /dev/null
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogManagedPartitionMsckRepairTest.scala
@@ -0,0 +1,635 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.sql
+
+import org.apache.paimon.catalog.Identifier
+import org.apache.paimon.fs.Path
+import org.apache.paimon.partition.Partition
+import org.apache.paimon.spark.{PaimonSparkTestWithRestCatalogBase, SparkCatalog}
+import org.apache.paimon.spark.execution.PaimonRepairFormatTablePartitionsExec
+import org.apache.paimon.spark.format.PaimonFormatTable
+import org.apache.paimon.table.FormatTable
+import org.apache.paimon.table.format.FormatTablePartitionManager
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.connector.catalog.{Identifier => SparkIdentifier, Table => SparkTable}
+import org.apache.spark.sql.execution.CommandResultExec
+
+import java.lang.reflect.InvocationTargetException
+import java.util.{List => JList, Map => JMap}
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+class CatalogManagedPartitionMsckRepairTest extends PaimonSparkTestWithRestCatalogBase {
+
+ override protected def sparkConf: SparkConf =
+ super.sparkConf
+ .set("spark.sql.catalog.paimon", classOf[FaultInjectingFormatTableSparkCatalog].getName)
+
+ override protected def beforeEach(): Unit = {
+ MsckFaultInjection.reset()
+ super.beforeEach()
+ }
+
+ override protected def afterEach(): Unit = {
+ try {
+ super.afterEach()
+ } finally {
+ MsckFaultInjection.reset()
+ }
+ }
+
+ test("bare MSCK, explicit ADD, and REPAIR alias register filesystem-only partitions") {
+ val tableName = "msck_add_forms"
+ val first = "20260715"
+ val second = "20260716"
+ val third = "20260717"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+
+ writeCsvPartition(tableName, first, 15, "bare-msck")
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName")
+ assertPartitionState(tableName, Set(first))
+
+ writeCsvPartition(tableName, second, 16, "explicit-add")
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS")
+ assertPartitionState(tableName, Set(first, second))
+
+ writeCsvPartition(tableName, third, 17, "repair-alias")
+ executeCatalogManagedRepair(s"REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS")
+ assertPartitionState(tableName, Set(first, second, third))
+
+ executeCatalogManagedRepair(s"REPAIR TABLE paimon.$dbName0.$tableName ADD PARTITIONS")
+
+ assertPartitionState(tableName, Set(first, second, third))
+ assert(MsckFaultInjection.createCalls == 3)
+ assertRowsAndChecksum(
+ tableName,
+ Seq(
+ Row(15, "bare-msck", first),
+ Row(16, "explicit-add", second),
+ Row(17, "repair-alias", third)),
+ 48L)
+ }
+ }
+
+ test("DROP unregisters only catalog-only partitions and preserves valid files") {
+ val tableName = "msck_drop_catalog_only"
+ val catalogOnly = "20260714"
+ val common = "20260715"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+ val commonPath = writeCsvPartition(tableName, common, 15, "common")
+ registerPartitions(tableName, catalogOnly, common)
+
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName DROP PARTITIONS")
+
+ assertPartitionState(tableName, Set(common))
+ assert(formatTable(tableName).fileIO().exists(commonPath))
+ assertRowsAndChecksum(tableName, Seq(Row(15, "common", common)), 15L)
+
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName DROP PARTITIONS")
+ assertPartitionState(tableName, Set(common))
+ assert(MsckFaultInjection.createCalls == 0)
+ assert(MsckFaultInjection.dropCalls == 1)
+ }
+ }
+
+ test("SYNC converges both directions and is idempotent") {
+ val tableName = "msck_sync_both_directions"
+ val catalogOnly = "20260714"
+ val filesystemOnly = "20260715"
+ val common = "20260716"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+ val filesystemPath =
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ val commonPath = writeCsvPartition(tableName, common, 16, "common")
+ registerPartitions(tableName, catalogOnly, common)
+
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+
+ assertPartitionState(tableName, Set(filesystemOnly, common))
+ assert(formatTable(tableName).fileIO().exists(filesystemPath))
+ assert(formatTable(tableName).fileIO().exists(commonPath))
+ assertRowsAndChecksum(
+ tableName,
+ Seq(Row(15, "filesystem-only", filesystemOnly), Row(16, "common", common)),
+ 31L)
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 1)
+
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+ assertPartitionState(tableName, Set(filesystemOnly, common))
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 1)
+ }
+ }
+
+ test("MSCK rejects filesystem-discovered Format Tables and native Paimon tables") {
+ val filesystemPartitioned = "msck_filesystem_format_table"
+ val native = "msck_native_paimon_table"
+ val partition = "20260715"
+
+ withTable(filesystemPartitioned, native) {
+ createFormatTableWithFilesystemPartitions(filesystemPartitioned)
+ writeCsvPartition(filesystemPartitioned, partition, 15, "filesystem")
+ checkAnswer(
+ sql(s"SELECT id, payload, dt FROM $filesystemPartitioned"),
+ Seq(Row(15, "filesystem", partition)))
+
+ assertMsckRejected(s"MSCK REPAIR TABLE paimon.$dbName0.$filesystemPartitioned")
+ checkAnswer(
+ sql(s"SELECT id, payload, dt FROM $filesystemPartitioned"),
+ Seq(Row(15, "filesystem", partition)))
+
+ sql(
+ s"CREATE TABLE $native (id INT, payload STRING, dt STRING) " +
+ "USING paimon PARTITIONED BY (dt)")
+ sql(s"INSERT INTO $native VALUES (16, 'native', '$partition')")
+
+ assertMsckRejected(s"MSCK REPAIR TABLE paimon.$dbName0.$native")
+ checkAnswer(sql(s"SELECT id, payload, dt FROM $native"), Seq(Row(16, "native", partition)))
+ assert(MsckFaultInjection.createCalls == 0)
+ assert(MsckFaultInjection.dropCalls == 0)
+ }
+ }
+
+ test("SYNC propagates catalog LIST failure without mutating either side") {
+ val tableName = "msck_sync_list_failure"
+ val catalogOnly = "20260714"
+ val filesystemOnly = "20260715"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+ val filesystemPath =
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ registerPartitions(tableName, catalogOnly)
+ MsckFaultInjection.failList = true
+
+ val error = intercept[Exception] {
+ sql(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS").collect()
+ }
+
+ assert(causeMessages(error).contains(MsckFaultInjection.LIST_FAILURE))
+ assert(MsckFaultInjection.listCalls == 1)
+ assert(MsckFaultInjection.createCalls == 0)
+ assert(MsckFaultInjection.dropCalls == 0)
+ assert(registeredPartitions(tableName) == Set(catalogOnly))
+ assert(formatTable(tableName).fileIO().exists(filesystemPath))
+
+ MsckFaultInjection.failList = false
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+ assertPartitionState(tableName, Set(filesystemOnly))
+ assertRowsAndChecksum(tableName, Seq(Row(15, "filesystem-only", filesystemOnly)), 15L)
+ }
+ }
+
+ test("direct repair mutates nothing but still refreshes when catalog LIST fails") {
+ val tableName = "msck_direct_list_failure"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+ val gateway = new StatefulFaultCatalog
+ gateway.failList = true
+ var refreshCalls = 0
+ val command = PaimonRepairFormatTablePartitionsExec(
+ sparkTable(tableName, gateway),
+ addPartitions = true,
+ dropPartitions = true,
+ () => refreshCalls += 1)
+
+ val error = intercept[IllegalStateException] {
+ runCommand(command)
+ }
+
+ assert(error.getMessage == MsckFaultInjection.LIST_FAILURE)
+ assert(gateway.createCalls == 0)
+ assert(gateway.dropCalls == 0)
+ // A failed response cannot prove the service did nothing, so every repair attempt
+ // refreshes the Spark-side caches.
+ assert(refreshCalls == 1)
+ }
+ }
+
+ test("direct repair refreshes after create applies and loses its response") {
+ val tableName = "msck_direct_create_lost_response"
+ val filesystemOnly = "20260715"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ val gateway = new StatefulFaultCatalog
+ gateway.failCreateAfterApply = true
+ var refreshCalls = 0
+ val command = PaimonRepairFormatTablePartitionsExec(
+ sparkTable(tableName, gateway),
+ addPartitions = true,
+ dropPartitions = true,
+ () => refreshCalls += 1)
+
+ val error = intercept[IllegalStateException] {
+ runCommand(command)
+ }
+
+ assert(error.getMessage == MsckFaultInjection.CREATE_LOST_RESPONSE)
+ assert(gateway.partitions == Set(Map("dt" -> filesystemOnly)))
+ assert(gateway.createCalls == 1)
+ assert(gateway.dropCalls == 0)
+ assert(refreshCalls == 1)
+ }
+ }
+
+ test("direct repair keeps the DROP failure primary when refresh also fails") {
+ val tableName = "msck_direct_drop_and_refresh_failure"
+ val catalogOnly = "20260714"
+ val filesystemOnly = "20260715"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ val gateway = new StatefulFaultCatalog(Set(Map("dt" -> catalogOnly)))
+ gateway.failDrop = true
+ var refreshCalls = 0
+ val command = PaimonRepairFormatTablePartitionsExec(
+ sparkTable(tableName, gateway),
+ addPartitions = true,
+ dropPartitions = true,
+ () => {
+ refreshCalls += 1
+ throw new IllegalStateException(MsckFaultInjection.REFRESH_FAILURE)
+ }
+ )
+
+ val error = intercept[IllegalStateException] {
+ runCommand(command)
+ }
+
+ // The DROP failure is what explains the repair, so it propagates and the refresh
+ // failure rides along; the ADD stays durable and the refresh is still attempted.
+ assert(error.getMessage == MsckFaultInjection.DROP_FAILURE)
+ assert(error.getSuppressed.map(_.getMessage).toSeq == Seq(MsckFaultInjection.REFRESH_FAILURE))
+ assert(gateway.partitions == Set(Map("dt" -> catalogOnly), Map("dt" -> filesystemOnly)))
+ assert(gateway.createCalls == 1)
+ assert(gateway.dropCalls == 1)
+ assert(refreshCalls == 1)
+ }
+ }
+
+ test("direct repair does not suppress a DROP throwable into itself") {
+ val tableName = "msck_direct_same_drop_refresh_failure"
+ val catalogOnly = "20260714"
+ val filesystemOnly = "20260715"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ val gateway = new StatefulFaultCatalog(Set(Map("dt" -> catalogOnly)))
+ val dropFailure = new IllegalStateException(MsckFaultInjection.DROP_FAILURE)
+ gateway.dropFailure = dropFailure
+ var refreshCalls = 0
+ val command = PaimonRepairFormatTablePartitionsExec(
+ sparkTable(tableName, gateway),
+ addPartitions = true,
+ dropPartitions = true,
+ () => {
+ refreshCalls += 1
+ throw dropFailure
+ }
+ )
+
+ val error = intercept[Throwable] {
+ runCommand(command)
+ }
+
+ // Refresh rethrows the very throwable the DROP failed with. Attaching it to itself would
+ // throw IllegalArgumentException, so the two must be recognised as the same failure.
+ assert(error eq dropFailure)
+ assert(!error.isInstanceOf[MatchError])
+ assert(error.getSuppressed.isEmpty)
+ assert(gateway.partitions == Set(Map("dt" -> catalogOnly), Map("dt" -> filesystemOnly)))
+ assert(gateway.createCalls == 1)
+ assert(gateway.dropCalls == 1)
+ assert(refreshCalls == 1)
+ }
+ }
+
+ test("SYNC invalidates cached data and converges after ADD succeeds but DROP fails") {
+ val tableName = "msck_partial_sync_failure"
+ val filesystemOnly = "20260715"
+ val catalogOnly = "20260714"
+
+ withTable(tableName) {
+ createFormatTableWithCatalogManagedPartitions(tableName)
+ sql(s"CACHE TABLE $tableName").collect()
+ assert(spark.catalog.isCached(tableName))
+ checkAnswer(sql(s"SELECT * FROM $tableName"), Seq.empty[Row])
+
+ writeCsvPartition(tableName, filesystemOnly, 15, "filesystem-only")
+ registerPartitions(tableName, catalogOnly)
+ MsckFaultInjection.failDrop = true
+
+ val error = intercept[Exception] {
+ sql(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS").collect()
+ }
+
+ assert(causeMessages(error).contains(MsckFaultInjection.DROP_FAILURE))
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 1)
+ assert(registeredPartitions(tableName) == Set(filesystemOnly, catalogOnly))
+ assert(spark.catalog.isCached(tableName))
+
+ // ADD is already durable even though the command failed. A stale CACHE TABLE must not hide it.
+ checkAnswer(
+ sql(s"SELECT id, payload, dt FROM $tableName WHERE dt = '$filesystemOnly'"),
+ Seq(Row(15, "filesystem-only", filesystemOnly)))
+
+ MsckFaultInjection.failDrop = false
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+
+ assertPartitionState(tableName, Set(filesystemOnly))
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 2)
+ assertRowsAndChecksum(tableName, Seq(Row(15, "filesystem-only", filesystemOnly)), 15L)
+
+ executeCatalogManagedRepair(s"MSCK REPAIR TABLE paimon.$dbName0.$tableName SYNC PARTITIONS")
+ assertPartitionState(tableName, Set(filesystemOnly))
+ assert(MsckFaultInjection.createCalls == 1)
+ assert(MsckFaultInjection.dropCalls == 2)
+ }
+ }
+
+ private def createFormatTableWithCatalogManagedPartitions(tableName: String): Unit =
+ sql(s"""CREATE TABLE $tableName (id INT, payload STRING, dt STRING)
+ |USING CSV
+ |PARTITIONED BY (dt)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'format-table.partition-source' = 'rest')
+ |""".stripMargin)
+
+ private def createFormatTableWithFilesystemPartitions(tableName: String): Unit =
+ sql(s"""CREATE TABLE $tableName (id INT, payload STRING, dt STRING)
+ |USING CSV
+ |PARTITIONED BY (dt)
+ |TBLPROPERTIES (
+ | 'format-table.implementation' = 'paimon',
+ | 'format-table.partition-source' = 'filesystem')
+ |""".stripMargin)
+
+ private def executeCatalogManagedRepair(statement: String): Unit = {
+ val result = sql(statement)
+ val resultPlan = result.queryExecution.executedPlan
+ assert(resultPlan.isInstanceOf[CommandResultExec], resultPlan.treeString)
+ val commandPlan = resultPlan.asInstanceOf[CommandResultExec].commandPhysicalPlan
+ assert(commandPlan.isInstanceOf[PaimonRepairFormatTablePartitionsExec], commandPlan.treeString)
+ result.collect()
+ }
+
+ private def assertMsckRejected(statement: String): Unit = {
+ val error = intercept[Exception] {
+ sql(statement).collect()
+ }
+ val messages = causeMessages(error)
+ assert(messages.contains("MSCK REPAIR TABLE"), messages)
+ assert(messages.toLowerCase(java.util.Locale.ROOT).contains("not supported"), messages)
+ }
+
+ private def assertPartitionState(tableName: String, expected: Set[String]): Unit = {
+ assert(registeredPartitions(tableName) == expected)
+ assert(
+ sql(s"SHOW PARTITIONS $tableName").collect().map(_.getString(0)).toSet ==
+ expected.map(value => s"dt=$value"))
+ }
+
+ private def assertRowsAndChecksum(
+ tableName: String,
+ expectedRows: Seq[Row],
+ expectedIdSum: Long): Unit = {
+ checkAnswer(sql(s"SELECT id, payload, dt FROM $tableName ORDER BY id"), expectedRows)
+ checkAnswer(
+ sql(s"SELECT COUNT(*), COALESCE(SUM(id), 0) FROM $tableName"),
+ Seq(Row(expectedRows.size.toLong, expectedIdSum)))
+ }
+
+ private def runCommand(command: AnyRef): Unit = {
+ try {
+ command.getClass.getMethod("run").invoke(command)
+ } catch {
+ case error: InvocationTargetException => throw error.getCause
+ }
+ }
+
+ private def tableIdentifier(tableName: String): Identifier =
+ Identifier.create(dbName0, tableName)
+
+ private def formatTable(tableName: String): FormatTable =
+ paimonCatalog.getTable(tableIdentifier(tableName)).asInstanceOf[FormatTable]
+
+ private def sparkTable(
+ tableName: String,
+ partitionManager: FormatTablePartitionManager): PaimonFormatTable =
+ new PaimonFormatTable(
+ MsckTestFixtures.withPartitionManager(formatTable(tableName), partitionManager))
+
+ private def writeCsvPartition(
+ tableName: String,
+ partition: String,
+ id: Int,
+ payload: String): Path = {
+ val table = formatTable(tableName)
+ val partitionPath = new Path(table.location(), s"dt=$partition")
+ table.fileIO().mkdirs(partitionPath)
+ table.fileIO().writeFile(new Path(partitionPath, f"part-$id%05d.csv"), s"$id,$payload\n", false)
+ partitionPath
+ }
+
+ private def registerPartitions(tableName: String, partitions: String*): Unit =
+ paimonCatalog.createPartitions(
+ tableIdentifier(tableName),
+ partitions.map(value => Map("dt" -> value).asJava).asJava,
+ true)
+
+ private def registeredPartitions(tableName: String): Set[String] =
+ paimonCatalog
+ .listPartitions(tableIdentifier(tableName))
+ .asScala
+ .map(_.spec().get("dt"))
+ .toSet
+
+ private def causeMessages(error: Throwable): String =
+ Iterator
+ .iterate(error)(_.getCause)
+ .takeWhile(_ != null)
+ .flatMap(cause => Option(cause.getMessage))
+ .mkString(" | ")
+}
+
+private[sql] object MsckFaultInjection {
+ val DROP_FAILURE = "injected MSCK partition drop failure"
+ val LIST_FAILURE = "injected MSCK partition list failure"
+ val CREATE_LOST_RESPONSE = "injected MSCK create lost response"
+ val REFRESH_FAILURE = "injected MSCK cache refresh failure"
+
+ @volatile var failDrop = false
+ @volatile var failList = false
+ @volatile var createCalls = 0
+ @volatile var dropCalls = 0
+ @volatile var listCalls = 0
+
+ def reset(): Unit = {
+ failDrop = false
+ failList = false
+ createCalls = 0
+ dropCalls = 0
+ listCalls = 0
+ }
+}
+
+private[sql] class StatefulFaultCatalog(initial: Set[Map[String, String]] = Set.empty)
+ extends FormatTablePartitionManager {
+
+ private val state = mutable.LinkedHashSet(initial.toSeq: _*)
+ var failList = false
+ var failCreateAfterApply = false
+ var failDrop = false
+ var dropFailure: RuntimeException = null
+ var createCalls = 0
+ var dropCalls = 0
+
+ def partitions: Set[Map[String, String]] = state.toSet
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ createCalls += 1
+ state ++= partitions.asScala.map(_.asScala.toMap)
+ if (failCreateAfterApply) {
+ throw new IllegalStateException(MsckFaultInjection.CREATE_LOST_RESPONSE)
+ }
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ dropCalls += 1
+ if (dropFailure != null) {
+ throw dropFailure
+ }
+ if (failDrop) {
+ throw new IllegalStateException(MsckFaultInjection.DROP_FAILURE)
+ }
+ state --= partitions.asScala.map(_.asScala.toMap)
+ }
+
+ override def listPartitionsByNames(partitions: JList[JMap[String, String]]): JList[Partition] =
+ MsckTestFixtures.toPartitions(
+ partitions.asScala.map(_.asScala.toMap).filter(state.contains).toSeq)
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] = {
+ if (failList) {
+ throw new IllegalStateException(MsckFaultInjection.LIST_FAILURE)
+ }
+ MsckTestFixtures.toPartitions(state.toSeq)
+ }
+}
+
+/** Test helpers shared by the fault-injecting catalogs of this suite. */
+private[sql] object MsckTestFixtures {
+
+ /** Partition metadata as a catalog returns it; only the spec matters for these tests. */
+ def toPartitions(specs: Seq[Map[String, String]]): JList[Partition] =
+ specs.map(spec => new Partition(spec.asJava, 0L, 0L, 0L, 0L, 0, false)).asJava
+
+ private val rebound: JMap[FormatTable, FormatTable] =
+ java.util.Collections.synchronizedMap(new java.util.IdentityHashMap[FormatTable, FormatTable]())
+
+ /** Rebind a table once per source instance, so repeated loads return the same table. */
+ def rebindOnce(table: FormatTable, partitionManager: FormatTablePartitionManager): FormatTable =
+ rebound.computeIfAbsent(table, _ => withPartitionManager(table, partitionManager))
+
+ /** Rebind a Format Table to another partition catalog, as loading it from a catalog would. */
+ def withPartitionManager(
+ table: FormatTable,
+ partitionManager: FormatTablePartitionManager): FormatTable =
+ FormatTable
+ .builder()
+ .fileIO(table.fileIO())
+ .identifier(Identifier.fromString(table.fullName()))
+ .rowType(table.rowType())
+ .partitionKeys(table.partitionKeys())
+ .location(table.location())
+ .format(table.format())
+ .options(table.options())
+ .comment(table.comment().orElse(null))
+ .catalogContext(table.catalogContext())
+ .partitionManager(partitionManager)
+ .build()
+}
+
+/** Spark-loadable test catalog which delegates all state to the embedded Paimon REST catalog. */
+private[sql] class FaultInjectingFormatTableSparkCatalog extends SparkCatalog {
+
+ override def loadTable(ident: SparkIdentifier): SparkTable = {
+ super.loadTable(ident) match {
+ case table: PaimonFormatTable if table.partitionManager != null =>
+ // The catalog hands out one table instance per table, and Spark's cached plans compare
+ // that instance; rebinding must be memoized per source instance or CACHE TABLE stops
+ // matching the relation it cached.
+ new PaimonFormatTable(
+ MsckTestFixtures.rebindOnce(
+ table.table,
+ new FaultInjectingFormatTablePartitionManager(table.partitionManager)))
+ case table => table
+ }
+ }
+}
+
+private[sql] class FaultInjectingFormatTablePartitionManager(delegate: FormatTablePartitionManager)
+ extends FormatTablePartitionManager {
+
+ override def createPartitions(
+ partitions: JList[JMap[String, String]],
+ ignoreIfExists: Boolean): Unit = {
+ delegate.createPartitions(partitions, ignoreIfExists)
+ MsckFaultInjection.createCalls += 1
+ }
+
+ override def dropPartitions(partitions: JList[JMap[String, String]]): Unit = {
+ MsckFaultInjection.dropCalls += 1
+ if (MsckFaultInjection.failDrop) {
+ throw new IllegalStateException(MsckFaultInjection.DROP_FAILURE)
+ }
+ delegate.dropPartitions(partitions)
+ }
+
+ override def listPartitionsByNames(partitions: JList[JMap[String, String]]): JList[Partition] =
+ delegate.listPartitionsByNames(partitions)
+
+ override def listPartitions(prefix: JMap[String, String]): JList[Partition] = {
+ MsckFaultInjection.listCalls += 1
+ if (MsckFaultInjection.failList) {
+ throw new IllegalStateException(MsckFaultInjection.LIST_FAILURE)
+ }
+ delegate.listPartitions(prefix)
+ }
+}