From 6b45d504400c8abcf74ef3e6f05f427a58a2fdfe Mon Sep 17 00:00:00 2001 From: Alessandro Nori Date: Fri, 31 Jul 2026 13:19:25 +0200 Subject: [PATCH 1/2] Fix orphan file matching for sibling table prefixes --- .../actions/DeleteOrphanFilesSparkAction.java | 3 +- .../actions/TestRemoveOrphanFilesAction.java | 43 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/DeleteOrphanFilesSparkAction.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/DeleteOrphanFilesSparkAction.java index b47922820d21..2e4d917faf99 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/DeleteOrphanFilesSparkAction.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/DeleteOrphanFilesSparkAction.java @@ -225,7 +225,8 @@ public DeleteOrphanFilesSparkAction usePrefixListing(boolean newUsePrefixListing private Dataset filteredCompareToFileList() { Dataset files = compareToFileList; if (location != null) { - files = files.filter(files.col(FILE_PATH).startsWith(location)); + String normalizedLocation = location.endsWith("/") ? location : location + "/"; + files = files.filter(files.col(FILE_PATH).startsWith(normalizedLocation)); } return files .filter(files.col(LAST_MODIFIED).lt(new Timestamp(olderThanTimestamp))) diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java index 78e8a0b000a4..a80f1bc55f9b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java @@ -1020,6 +1020,49 @@ public void testCompareToFileList() throws IOException { assertThat(result4.orphanFilesCount()).as("Action should find nothing").isEqualTo(0L); } + @TestTemplate + public void testCompareToFileListWithSiblingTableLocation() throws IOException { + Table table = TABLES.create(SCHEMA, PartitionSpec.unpartitioned(), properties, tableLocation); + + List records = + Lists.newArrayList(new ThreeColumnRecord(1, "AAAAAAAAAA", "AAAA")); + Dataset df = spark.createDataFrame(records, ThreeColumnRecord.class).coalesce(1); + df.select("c1", "c2", "c3").write().format("iceberg").mode("append").save(tableLocation); + + // strip the trailing slash to reproduce a location as it would be passed via + // DeleteOrphanFiles#location(String), e.g. "/tmp/table" instead of "/tmp/table/" + String locationWithoutTrailingSlash = tableLocation.substring(0, tableLocation.length() - 1); + + // sibling table whose location has this table's location as a string prefix, + // e.g. "/tmp/table" is a prefix of "/tmp/table_2" + String siblingTableLocation = locationWithoutTrailingSlash + "_2"; + String siblingFilePath = siblingTableLocation + "/data/sibling-file.parquet"; + + waitUntilAfter(System.currentTimeMillis()); + + List compareToFiles = + Lists.newArrayList(new FilePathLastModifiedRecord(siblingFilePath, new Timestamp(0L))); + + Dataset compareToFileList = + spark + .createDataFrame(compareToFiles, FilePathLastModifiedRecord.class) + .withColumnRenamed("filePath", "file_path") + .withColumnRenamed("lastModified", "last_modified"); + + DeleteOrphanFiles.Result result = + SparkActions.get() + .deleteOrphanFiles(table) + .location(locationWithoutTrailingSlash) + .compareToFileList(compareToFileList) + .olderThan(System.currentTimeMillis()) + .deleteWith(s -> {}) + .execute(); + + assertThat(result.orphanFileLocations()) + .as("Files belonging to a sibling table must not be reported as orphan") + .isEmpty(); + } + protected long waitUntilAfter(long timestampMillis) { long current = System.currentTimeMillis(); while (current <= timestampMillis) { From ec0ccf2cbcd1fca22ecf2a67d4802e311c95d6e2 Mon Sep 17 00:00:00 2001 From: Alessandro Nori Date: Fri, 31 Jul 2026 13:31:24 +0200 Subject: [PATCH 2/2] Skip redundant usePrefixListing variant in new test --- .../iceberg/spark/actions/TestRemoveOrphanFilesAction.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java index a80f1bc55f9b..1b999d867156 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java @@ -1022,6 +1022,9 @@ public void testCompareToFileList() throws IOException { @TestTemplate public void testCompareToFileListWithSiblingTableLocation() throws IOException { + assumeThat(usePrefixListing) + .as("Should not test both prefix listing and Hadoop file listing (redundant)") + .isEqualTo(false); Table table = TABLES.create(SCHEMA, PartitionSpec.unpartitioned(), properties, tableLocation); List records =