Skip to content

[iceberg] Omit unknown null counts and bounds in IcebergDataFileMeta #8732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 1 commit into from
Jul 20, 2026
+208 −14
Merged

[iceberg] Omit unknown null counts and bounds in IcebergDataFileMeta #8732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 25 additions & 14 deletions paimon-core/src/main/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,25 +174,36 @@ public static IcebergDataFileMeta create(
}

int idx = indexMap.get(field.name());
nullValueCounts.put(field.id(), stats.nullCounts().getLong(idx));
// an unknown null count must be omitted, not published as 0
if (!stats.nullCounts().isNullAt(idx)) {
nullValueCounts.put(field.id(), stats.nullCounts().getLong(idx));
}

// these types have no bounds; skip before reading the stats slots
DataTypeRoot typeRoot = field.dataType().getTypeRoot();
if (typeRoot == DataTypeRoot.ARRAY
|| typeRoot == DataTypeRoot.MAP
|| typeRoot == DataTypeRoot.ROW
|| typeRoot == DataTypeRoot.MULTISET
|| typeRoot == DataTypeRoot.VARIANT
|| typeRoot == DataTypeRoot.VECTOR
|| typeRoot == DataTypeRoot.BLOB) {
continue;
}

// use the nullable copy of the type, so that an unknown (null) min/max slot
// of a required field reads as null instead of garbage
InternalRow.FieldGetter fieldGetter =
InternalRow.createFieldGetter(field.dataType(), idx);
InternalRow.createFieldGetter(field.dataType().nullable(), idx);
Object minValue = fieldGetter.getFieldOrNull(stats.minValues());
Object maxValue = fieldGetter.getFieldOrNull(stats.maxValues());
if (minValue != null && maxValue != null) {
DataTypeRoot typeRoot = field.dataType().getTypeRoot();
if (typeRoot != DataTypeRoot.ARRAY
&& typeRoot != DataTypeRoot.MAP
&& typeRoot != DataTypeRoot.ROW
&& typeRoot != DataTypeRoot.MULTISET) {
lowerBounds.put(
field.id(),
IcebergConversions.toByteBuffer(field.dataType(), minValue).array());
upperBounds.put(
field.id(),
IcebergConversions.toByteBuffer(field.dataType(), maxValue).array());
}
lowerBounds.put(
field.id(),
IcebergConversions.toByteBuffer(field.dataType(), minValue).array());
upperBounds.put(
field.id(),
IcebergConversions.toByteBuffer(field.dataType(), maxValue).array());
}
}

Expand Down
183 changes: 183 additions & 0 deletions paimon-core/src/test/java/org/apache/paimon/iceberg/manifest/IcebergDataFileMetaTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,22 @@
package org.apache.paimon.iceberg.manifest;

import org.apache.paimon.TestKeyValueGenerator;
import org.apache.paimon.data.BinaryArray;
import org.apache.paimon.data.BinaryArrayWriter;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryRowWriter;
import org.apache.paimon.data.GenericMap;
import org.apache.paimon.iceberg.metadata.IcebergDataField;
import org.apache.paimon.iceberg.metadata.IcebergSchema;
import org.apache.paimon.stats.SimpleStats;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;

import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;

import java.util.Arrays;
import java.util.List;
import java.util.Optional;

Expand All @@ -45,4 +55,177 @@ void testPartitionIsNotNull() {
assertThat(partitionField.get().name()).isEqualTo("partition");
assertThat(partitionField.get().type()).isEqualTo(partitionType.notNull());
}

@Test
@DisplayName("Test unknown null count is omitted instead of published as 0")
void testUnknownNullCountOmitted() {
IcebergSchema icebergSchema =
new IcebergSchema(
0,
Arrays.asList(
new IcebergDataField(1, "a", false, "int", null),
new IcebergDataField(2, "b", false, "int", null)));

BinaryRow values = new BinaryRow(2);
BinaryRowWriter rowWriter = new BinaryRowWriter(values);
rowWriter.setNullAt(0);
rowWriter.setNullAt(1);
rowWriter.complete();

// "a" has an unknown null count, "b" has a known null count of 0
BinaryArray nullCounts = new BinaryArray();
BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 2, 8);
arrayWriter.setNullLong(0);
arrayWriter.writeLong(1, 0L);
arrayWriter.complete();

IcebergDataFileMeta meta =
IcebergDataFileMeta.create(
IcebergDataFileMeta.Content.DATA,
"path",
"parquet",
BinaryRow.EMPTY_ROW,
10,
100,
icebergSchema,
new SimpleStats(values, values, nullCounts),
null);

assertThat(meta.nullValueCounts().size()).isEqualTo(1);
assertThat(((GenericMap) meta.nullValueCounts()).get(1)).isNull();
assertThat(((GenericMap) meta.nullValueCounts()).get(2)).isEqualTo(0L);
}

@Test
@DisplayName("Test required field with unknown stats does not publish garbage bounds")
void testRequiredFieldWithUnknownStats() {
IcebergSchema icebergSchema =
new IcebergSchema(
0,
Arrays.asList(
new IcebergDataField(1, "id", true, "int", null),
new IcebergDataField(2, "cnt", true, "long", null)));

BinaryRow values = new BinaryRow(2);
BinaryRowWriter rowWriter = new BinaryRowWriter(values);
rowWriter.writeInt(0, 1);
rowWriter.setNullAt(1);
rowWriter.complete();

BinaryArray nullCounts = new BinaryArray();
BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 2, 8);
arrayWriter.writeLong(0, 0L);
arrayWriter.setNullLong(1);
arrayWriter.complete();

IcebergDataFileMeta meta =
IcebergDataFileMeta.create(
IcebergDataFileMeta.Content.DATA,
"path",
"parquet",
BinaryRow.EMPTY_ROW,
10,
100,
icebergSchema,
new SimpleStats(values, values, nullCounts),
null);

assertThat(meta.nullValueCounts().size()).isEqualTo(1);
assertThat(((GenericMap) meta.nullValueCounts()).get(1)).isEqualTo(0L);
assertThat(meta.lowerBounds().size()).isEqualTo(1);
assertThat(meta.upperBounds().size()).isEqualTo(1);
}

@Test
@DisplayName("Test stats of a statsColumns subset are read by stats index, not field ordinal")
void testStatsColumnsSubsetAlignment() {
IcebergSchema icebergSchema =
new IcebergSchema(
0,
Arrays.asList(
new IcebergDataField(1, "a", false, "int", null),
new IcebergDataField(2, "b", false, "long", null)));

// stats only cover "b": slot 0 in the stats, ordinal 1 in the schema
BinaryRow values = new BinaryRow(1);
BinaryRowWriter rowWriter = new BinaryRowWriter(values);
rowWriter.writeLong(0, 7L);
rowWriter.complete();

BinaryArray nullCounts = new BinaryArray();
BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 1, 8);
arrayWriter.writeLong(0, 2L);
arrayWriter.complete();

IcebergDataFileMeta meta =
IcebergDataFileMeta.create(
IcebergDataFileMeta.Content.DATA,
"path",
"parquet",
BinaryRow.EMPTY_ROW,
10,
100,
icebergSchema,
new SimpleStats(values, values, nullCounts),
Arrays.asList("b"));

assertThat(meta.nullValueCounts().size()).isEqualTo(1);
assertThat(((GenericMap) meta.nullValueCounts()).get(2)).isEqualTo(2L);
assertThat(meta.lowerBounds().size()).isEqualTo(1);
byte[] expectedLong7 = {7, 0, 0, 0, 0, 0, 0, 0};
assertThat((byte[]) ((GenericMap) meta.lowerBounds()).get(2)).isEqualTo(expectedLong7);
assertThat((byte[]) ((GenericMap) meta.upperBounds()).get(2)).isEqualTo(expectedLong7);
}

@Test
@DisplayName("Test required nested field with unknown stats is skipped before reading slots")
void testRequiredNestedFieldSkipped() {
IcebergSchema icebergSchema =
new IcebergSchema(
0,
Arrays.asList(
new IcebergDataField(1, "id", true, "int", null),
new IcebergDataField(
new DataField(
2,
"arr",
DataTypes.ARRAY(DataTypes.INT()).notNull()))));

BinaryRow minValues = new BinaryRow(2);
BinaryRowWriter minWriter = new BinaryRowWriter(minValues);
minWriter.writeInt(0, 1);
minWriter.setNullAt(1);
minWriter.complete();

BinaryRow maxValues = new BinaryRow(2);
BinaryRowWriter maxWriter = new BinaryRowWriter(maxValues);
maxWriter.writeInt(0, 5);
maxWriter.setNullAt(1);
maxWriter.complete();

BinaryArray nullCounts = new BinaryArray();
BinaryArrayWriter arrayWriter = new BinaryArrayWriter(nullCounts, 2, 8);
arrayWriter.writeLong(0, 0L);
arrayWriter.setNullLong(1);
arrayWriter.complete();

IcebergDataFileMeta meta =
IcebergDataFileMeta.create(
IcebergDataFileMeta.Content.DATA,
"path",
"parquet",
BinaryRow.EMPTY_ROW,
10,
100,
icebergSchema,
new SimpleStats(minValues, maxValues, nullCounts),
null);

assertThat(meta.lowerBounds().size()).isEqualTo(1);
assertThat(meta.upperBounds().size()).isEqualTo(1);
assertThat((byte[]) ((GenericMap) meta.lowerBounds()).get(1))
.isEqualTo(new byte[] {1, 0, 0, 0});
assertThat((byte[]) ((GenericMap) meta.upperBounds()).get(1))
.isEqualTo(new byte[] {5, 0, 0, 0});
}
}
Loading