From 9f7e6751e52ee8861a291a4a27fc10f59610e38f Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 19 Jul 2026 00:40:06 -0400 Subject: [PATCH] [iceberg] Omit unknown null counts and bounds in IcebergDataFileMeta --- .../iceberg/manifest/IcebergDataFileMeta.java | 39 ++-- .../manifest/IcebergDataFileMetaTest.java | 183 ++++++++++++++++++ 2 files changed, 208 insertions(+), 14 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java index 07543e10a049..7334f7528af7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java @@ -174,25 +174,36 @@ public static IcebergDataFileMeta create( } int idx = indexMap.get(field.name()); - nullValueCounts.put(field.id(), stats.nullCounts().getLong(idx)); + // an unknown null count must be omitted, not published as 0 + if (!stats.nullCounts().isNullAt(idx)) { + nullValueCounts.put(field.id(), stats.nullCounts().getLong(idx)); + } + + // these types have no bounds; skip before reading the stats slots + DataTypeRoot typeRoot = field.dataType().getTypeRoot(); + if (typeRoot == DataTypeRoot.ARRAY + || typeRoot == DataTypeRoot.MAP + || typeRoot == DataTypeRoot.ROW + || typeRoot == DataTypeRoot.MULTISET + || typeRoot == DataTypeRoot.VARIANT + || typeRoot == DataTypeRoot.VECTOR + || typeRoot == DataTypeRoot.BLOB) { + continue; + } + // use the nullable copy of the type, so that an unknown (null) min/max slot + // of a required field reads as null instead of garbage InternalRow.FieldGetter fieldGetter = - InternalRow.createFieldGetter(field.dataType(), idx); + InternalRow.createFieldGetter(field.dataType().nullable(), idx); Object minValue = fieldGetter.getFieldOrNull(stats.minValues()); Object maxValue = fieldGetter.getFieldOrNull(stats.maxValues()); if (minValue != null && maxValue != null) { - DataTypeRoot typeRoot = field.dataType().getTypeRoot(); - if (typeRoot != DataTypeRoot.ARRAY - && typeRoot != DataTypeRoot.MAP - && typeRoot != DataTypeRoot.ROW - && typeRoot != DataTypeRoot.MULTISET) { - lowerBounds.put( - field.id(), - IcebergConversions.toByteBuffer(field.dataType(), minValue).array()); - upperBounds.put( - field.id(), - IcebergConversions.toByteBuffer(field.dataType(), maxValue).array()); - } + lowerBounds.put( + field.id(), + IcebergConversions.toByteBuffer(field.dataType(), minValue).array()); + upperBounds.put( + field.id(), + IcebergConversions.toByteBuffer(field.dataType(), maxValue).array()); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java index 9f9c3b962172..c4888312b053 100644 --- a/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java @@ -19,12 +19,22 @@ package org.apache.paimon.iceberg.manifest; import org.apache.paimon.TestKeyValueGenerator; +import org.apache.paimon.data.BinaryArray; +import org.apache.paimon.data.BinaryArrayWriter; +import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.data.BinaryRowWriter; +import org.apache.paimon.data.GenericMap; +import org.apache.paimon.iceberg.metadata.IcebergDataField; +import org.apache.paimon.iceberg.metadata.IcebergSchema; +import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -45,4 +55,177 @@ void testPartitionIsNotNull() { assertThat(partitionField.get().name()).isEqualTo("partition"); assertThat(partitionField.get().type()).isEqualTo(partitionType.notNull()); } + + @Test + @DisplayName("Test unknown null count is omitted instead of published as 0") + void testUnknownNullCountOmitted() { + IcebergSchema icebergSchema = + new IcebergSchema( + 0, + Arrays.asList( + new IcebergDataField(1, "a", false, "int", null), + new IcebergDataField(2, "b", false, "int", null))); + + BinaryRow values = new BinaryRow(2); + BinaryRowWriter rowWriter = new BinaryRowWriter(values); + rowWriter.setNullAt(0); + rowWriter.setNullAt(1); + rowWriter.complete(); + + // "a" has an unknown null count, "b" has a known null count of 0 + BinaryArray nullCounts = new BinaryArray(); + BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 2, 8); + arrayWriter.setNullLong(0); + arrayWriter.writeLong(1, 0L); + arrayWriter.complete(); + + IcebergDataFileMeta meta = + IcebergDataFileMeta.create( + IcebergDataFileMeta.Content.DATA, + "path", + "parquet", + BinaryRow.EMPTY_ROW, + 10, + 100, + icebergSchema, + new SimpleStats(values, values, nullCounts), + null); + + assertThat(meta.nullValueCounts().size()).isEqualTo(1); + assertThat(((GenericMap) meta.nullValueCounts()).get(1)).isNull(); + assertThat(((GenericMap) meta.nullValueCounts()).get(2)).isEqualTo(0L); + } + + @Test + @DisplayName("Test required field with unknown stats does not publish garbage bounds") + void testRequiredFieldWithUnknownStats() { + IcebergSchema icebergSchema = + new IcebergSchema( + 0, + Arrays.asList( + new IcebergDataField(1, "id", true, "int", null), + new IcebergDataField(2, "cnt", true, "long", null))); + + BinaryRow values = new BinaryRow(2); + BinaryRowWriter rowWriter = new BinaryRowWriter(values); + rowWriter.writeInt(0, 1); + rowWriter.setNullAt(1); + rowWriter.complete(); + + BinaryArray nullCounts = new BinaryArray(); + BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 2, 8); + arrayWriter.writeLong(0, 0L); + arrayWriter.setNullLong(1); + arrayWriter.complete(); + + IcebergDataFileMeta meta = + IcebergDataFileMeta.create( + IcebergDataFileMeta.Content.DATA, + "path", + "parquet", + BinaryRow.EMPTY_ROW, + 10, + 100, + icebergSchema, + new SimpleStats(values, values, nullCounts), + null); + + assertThat(meta.nullValueCounts().size()).isEqualTo(1); + assertThat(((GenericMap) meta.nullValueCounts()).get(1)).isEqualTo(0L); + assertThat(meta.lowerBounds().size()).isEqualTo(1); + assertThat(meta.upperBounds().size()).isEqualTo(1); + } + + @Test + @DisplayName("Test stats of a statsColumns subset are read by stats index, not field ordinal") + void testStatsColumnsSubsetAlignment() { + IcebergSchema icebergSchema = + new IcebergSchema( + 0, + Arrays.asList( + new IcebergDataField(1, "a", false, "int", null), + new IcebergDataField(2, "b", false, "long", null))); + + // stats only cover "b": slot 0 in the stats, ordinal 1 in the schema + BinaryRow values = new BinaryRow(1); + BinaryRowWriter rowWriter = new BinaryRowWriter(values); + rowWriter.writeLong(0, 7L); + rowWriter.complete(); + + BinaryArray nullCounts = new BinaryArray(); + BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 1, 8); + arrayWriter.writeLong(0, 2L); + arrayWriter.complete(); + + IcebergDataFileMeta meta = + IcebergDataFileMeta.create( + IcebergDataFileMeta.Content.DATA, + "path", + "parquet", + BinaryRow.EMPTY_ROW, + 10, + 100, + icebergSchema, + new SimpleStats(values, values, nullCounts), + Arrays.asList("b")); + + assertThat(meta.nullValueCounts().size()).isEqualTo(1); + assertThat(((GenericMap) meta.nullValueCounts()).get(2)).isEqualTo(2L); + assertThat(meta.lowerBounds().size()).isEqualTo(1); + byte[] expectedLong7 = {7, 0, 0, 0, 0, 0, 0, 0}; + assertThat((byte[]) ((GenericMap) meta.lowerBounds()).get(2)).isEqualTo(expectedLong7); + assertThat((byte[]) ((GenericMap) meta.upperBounds()).get(2)).isEqualTo(expectedLong7); + } + + @Test + @DisplayName("Test required nested field with unknown stats is skipped before reading slots") + void testRequiredNestedFieldSkipped() { + IcebergSchema icebergSchema = + new IcebergSchema( + 0, + Arrays.asList( + new IcebergDataField(1, "id", true, "int", null), + new IcebergDataField( + new DataField( + 2, + "arr", + DataTypes.ARRAY(DataTypes.INT()).notNull())))); + + BinaryRow minValues = new BinaryRow(2); + BinaryRowWriter minWriter = new BinaryRowWriter(minValues); + minWriter.writeInt(0, 1); + minWriter.setNullAt(1); + minWriter.complete(); + + BinaryRow maxValues = new BinaryRow(2); + BinaryRowWriter maxWriter = new BinaryRowWriter(maxValues); + maxWriter.writeInt(0, 5); + maxWriter.setNullAt(1); + maxWriter.complete(); + + BinaryArray nullCounts = new BinaryArray(); + BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 2, 8); + arrayWriter.writeLong(0, 0L); + arrayWriter.setNullLong(1); + arrayWriter.complete(); + + IcebergDataFileMeta meta = + IcebergDataFileMeta.create( + IcebergDataFileMeta.Content.DATA, + "path", + "parquet", + BinaryRow.EMPTY_ROW, + 10, + 100, + icebergSchema, + new SimpleStats(minValues, maxValues, nullCounts), + null); + + assertThat(meta.lowerBounds().size()).isEqualTo(1); + assertThat(meta.upperBounds().size()).isEqualTo(1); + assertThat((byte[]) ((GenericMap) meta.lowerBounds()).get(1)) + .isEqualTo(new byte[] {1, 0, 0, 0}); + assertThat((byte[]) ((GenericMap) meta.upperBounds()).get(1)) + .isEqualTo(new byte[] {5, 0, 0, 0}); + } }