From d22449b08714592bbdd0f44b3d20e11735291df3 Mon Sep 17 00:00:00 2001 From: Peter-Phi-Tran Date: Sat, 20 Jun 2026 03:15:20 -0500 Subject: [PATCH 1/5] [IcebergIO] Bump Java floor to 17 (prep for Iceberg 1.11.0) Iceberg 1.11.0 is published as Java 17 bytecode (class file version 61); Gradle module metadata declares "org.gradle.jvm.version": 17. The current Java 11 floor on sdks/java/io/iceberg causes Iceberg 1.11.0 artifacts to fail resolution. Raise the floor independently so the version bump is a clean diff. Iceberg 1.10.0 still resolves under Java 17 (10.0 only requires Java 11+). Tracks #38925. --- CHANGES.md | 1 + sdks/java/io/iceberg/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index bd9c6ea246a5..4cd1f08c6b02 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -82,6 +82,7 @@ * (Python) Typehints of dataclass fields are honored during type inferences. To restore the behavior of fallback-to-any, use pipeline option `--exclude_infer_dataclass_field_type` ([#38797](https://github.com/apache/beam/issues/38797)). However fixing forward is recommended. +* (Java) IcebergIO now requires Java 17 at runtime. This raises the floor in preparation for the Iceberg 1.11.0 upgrade, which is published as Java 17 bytecode. Pipelines using `Managed.read(ICEBERG)`/`Managed.write(ICEBERG)` must run on a Java 17+ JVM ([#38925](https://github.com/apache/beam/issues/38925)). ## Deprecations diff --git a/sdks/java/io/iceberg/build.gradle b/sdks/java/io/iceberg/build.gradle index 8142c5f5b90b..7cdd32ed90e4 100644 --- a/sdks/java/io/iceberg/build.gradle +++ b/sdks/java/io/iceberg/build.gradle @@ -23,8 +23,8 @@ import java.util.stream.Collectors plugins { id 'org.apache.beam.module' } applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.io.iceberg', - // iceberg ended support for Java 8 in 1.7.0 - requireJavaVersion: JavaVersion.VERSION_11, + // iceberg ended support for Java 11 in 1.11.0 + requireJavaVersion: JavaVersion.VERSION_17, ) description = "Apache Beam :: SDKs :: Java :: IO :: Iceberg" From e1fdb50141b8c0d2c0720270062b42d402dd8d83 Mon Sep 17 00:00:00 2001 From: Peter-Phi-Tran Date: Mon, 22 Jun 2026 11:39:44 -0500 Subject: [PATCH 2/5] [IcebergIO] Run IcebergIO CI workflows on Java 17 Iceberg 1.11.0 dropped Java 11 support and this branch raised the iceberg module floor to Java 17 (requireJavaVersion VERSION_17). The IO_Iceberg_* workflows call setup-environment-action without a java-version, which defaults to Java 11, so the module would fail to build or be disabled on CI. Pin these workflows to Java 17, matching the Delta and Debezium IO workflows. For #38925 Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/IO_Iceberg_Integration_Tests.yml | 2 ++ .github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml | 2 ++ .../workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml | 2 ++ .github/workflows/IO_Iceberg_Performance_Tests.yml | 2 ++ .github/workflows/IO_Iceberg_Unit_Tests.yml | 2 ++ 5 files changed, 10 insertions(+) diff --git a/.github/workflows/IO_Iceberg_Integration_Tests.yml b/.github/workflows/IO_Iceberg_Integration_Tests.yml index 6341c51124c5..ef6fe3485286 100644 --- a/.github/workflows/IO_Iceberg_Integration_Tests.yml +++ b/.github/workflows/IO_Iceberg_Integration_Tests.yml @@ -72,6 +72,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: Run IcebergIO Integration Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml b/.github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml index 38a3a2514855..ec0659394813 100644 --- a/.github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml +++ b/.github/workflows/IO_Iceberg_Integration_Tests_Dataflow.yml @@ -72,6 +72,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: Run IcebergIO Integration Tests on Dataflow uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml b/.github/workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml index ba72a85204a1..fb4d1a1e4a74 100644 --- a/.github/workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml +++ b/.github/workflows/IO_Iceberg_Managed_Integration_Tests_Dataflow.yml @@ -72,6 +72,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: Run IcebergIO Managed Integration Tests on Dataflow uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/IO_Iceberg_Performance_Tests.yml b/.github/workflows/IO_Iceberg_Performance_Tests.yml index e3ec8b569012..1b6642b06dda 100644 --- a/.github/workflows/IO_Iceberg_Performance_Tests.yml +++ b/.github/workflows/IO_Iceberg_Performance_Tests.yml @@ -72,6 +72,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: Run IcebergIO Performance Test uses: ./.github/actions/gradle-command-self-hosted-action with: diff --git a/.github/workflows/IO_Iceberg_Unit_Tests.yml b/.github/workflows/IO_Iceberg_Unit_Tests.yml index dd8069c74af0..dbf0342ea779 100644 --- a/.github/workflows/IO_Iceberg_Unit_Tests.yml +++ b/.github/workflows/IO_Iceberg_Unit_Tests.yml @@ -91,6 +91,8 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: 17 - name: run IcebergIO build script uses: ./.github/actions/gradle-command-self-hosted-action with: From 4f26daf716caee037533d2617d861a3ced908b20 Mon Sep 17 00:00:00 2001 From: Peter-Phi-Tran Date: Mon, 22 Jun 2026 14:58:10 -0500 Subject: [PATCH 3/5] [IcebergIO] Raise Java 17 floor for IcebergIO's Java 11 dependents Bumping the iceberg module to Java 17 (Iceberg 1.11.0 dropped Java 11) breaks the Java 11 modules that depend on it, because Gradle's JVM-version variant resolution refuses to let a Java 11 consumer depend on a Java 17 library. Raise requireJavaVersion 11 -> 17 in: - sdks/java/extensions/sql/iceberg - examples/java/iceberg The Java IO expansion service (sdks/java/io/expansion-service) also depends on IcebergIO but was already raised to Java 17 on master in #38974 (for Delta Lake), so it needs no change here. For #38925 Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGES.md | 2 +- examples/java/iceberg/build.gradle | 4 ++-- sdks/java/extensions/sql/iceberg/build.gradle | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4cd1f08c6b02..4c4bf3bd12fa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -62,7 +62,6 @@ * Python SDK now supports memory profiling with Memray ([#38853](https://github.com/apache/beam/issues/38853)). * (Python) Added [Qdrant](https://qdrant.tech/) VectorDatabaseWriteConfig implementation ([#38141](https://github.com/apache/beam/issues/38141)). - ## I/Os * Support for reading from Delta Lake added (Java) ([#38551](https://github.com/apache/beam/issues/38551)). @@ -83,6 +82,7 @@ use pipeline option `--exclude_infer_dataclass_field_type` ([#38797](https://github.com/apache/beam/issues/38797)). However fixing forward is recommended. * (Java) IcebergIO now requires Java 17 at runtime. This raises the floor in preparation for the Iceberg 1.11.0 upgrade, which is published as Java 17 bytecode. Pipelines using `Managed.read(ICEBERG)`/`Managed.write(ICEBERG)` must run on a Java 17+ JVM ([#38925](https://github.com/apache/beam/issues/38925)). +* (Java) Beam SQL's IcebergIO extension now requires Java 17, as it depends on the IcebergIO module ([#38925](https://github.com/apache/beam/issues/38925)). ## Deprecations diff --git a/examples/java/iceberg/build.gradle b/examples/java/iceberg/build.gradle index 4d4a1fb44413..2cff15c5fb0d 100644 --- a/examples/java/iceberg/build.gradle +++ b/examples/java/iceberg/build.gradle @@ -26,8 +26,8 @@ plugins { applyJavaNature( exportJavadoc: false, automaticModuleName: 'org.apache.beam.examples.iceberg', - // iceberg requires Java11+ - requireJavaVersion: JavaVersion.VERSION_11 + // iceberg requires Java17+ + requireJavaVersion: JavaVersion.VERSION_17 ) description = "Apache Beam :: Examples :: Java :: Iceberg" diff --git a/sdks/java/extensions/sql/iceberg/build.gradle b/sdks/java/extensions/sql/iceberg/build.gradle index 893a485e7d86..97a128d69e9e 100644 --- a/sdks/java/extensions/sql/iceberg/build.gradle +++ b/sdks/java/extensions/sql/iceberg/build.gradle @@ -22,8 +22,8 @@ plugins { id 'org.apache.beam.module' } applyJavaNature( automaticModuleName: 'org.apache.beam.sdk.extensions.sql.meta.provider.hcatalog', - // iceberg requires Java11+ - requireJavaVersion: JavaVersion.VERSION_11, + // iceberg requires Java17+ + requireJavaVersion: JavaVersion.VERSION_17, ) dependencies { From b4347232ab4aadb057935f2b9e66666724568947 Mon Sep 17 00:00:00 2001 From: Peter-Phi-Tran Date: Wed, 24 Jun 2026 00:31:00 -0500 Subject: [PATCH 4/5] [IcebergIO] Fix Checker skipUses regex dropped on forked Java 17 compile Raising the Java 17 floor routes IcebergIO through a forked compile: on an older host JDK (e.g. Java 11 CI) the module is compiled by a separate Java 17 javac launched via java17Home, and Gradle passes its arguments through an @argfile. The backslash-escaped '\.' in Beam's Checker Framework -AskipUses and -AskipDefs regexes does not survive that @argfile round-trip, so '^org\.slf4j\.Logger.*' becomes a literal-backslash regex that matches nothing and the org.slf4j.Logger nullness suppression is silently dropped. Two latent Logger.info(@Nullable) calls in IcebergIO then hard-error under the NullnessChecker. Use the backslash-free character class '[.]' (semantically identical to '\.') so the regexes survive the @argfile round-trip. This fixes the suppression for every module that forks to a newer JDK, with no behavior change on the in-process compile path. --- .../org/apache/beam/gradle/BeamModulePlugin.groovy | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index b5aef5d23ad4..2f715846a8ae 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1224,20 +1224,26 @@ class BeamModulePlugin implements Plugin { maxHeapSize = '2g' } + // NOTE: Use the character class "[.]" instead of an escaped "\\." to match a literal dot in + // these Checker Framework -AskipDefs/-AskipUses regexes. When a module is compiled on an older + // host JDK and forked to a newer JDK via javaXXHome (e.g. iceberg's requireJavaVersion 17 on a + // Java 11 CI host), Gradle passes the javac arguments through an @argfile. Backslash escapes do + // not survive that round-trip intact, so "\\." becomes a literal-backslash regex that matches + // nothing and the suppression is silently dropped. "[.]" is backslash-free and survives. List skipDefRegexes = [] skipDefRegexes << "AutoValue_.*" skipDefRegexes << "AutoBuilder_.*" skipDefRegexes << "AutoOneOf_.*" - skipDefRegexes << ".*\\.jmh_generated\\..*" + skipDefRegexes << ".*[.]jmh_generated[.].*" skipDefRegexes += configuration.generatedClassPatterns skipDefRegexes += configuration.classesTriggerCheckerBugs.keySet() String skipDefCombinedRegex = skipDefRegexes.collect({ regex -> "(${regex})"}).join("|") List skipUsesRegexes = [] // zstd-jni is not annotated, handles Zstd(De)CompressCtx.loadDict(null) just fine - skipUsesRegexes << "^com\\.github\\.luben\\.zstd\\..*" + skipUsesRegexes << "^com[.]github[.]luben[.]zstd[.].*" // SLF4J logger handles null log message parameters - skipUsesRegexes << "^org\\.slf4j\\.Logger.*" + skipUsesRegexes << "^org[.]slf4j[.]Logger.*" String skipUsesCombinedRegex = skipUsesRegexes.collect({ regex -> "(${regex})"}).join("|") project.apply plugin: 'org.checkerframework' From 2a10ca86c26bc786f8134c2eee0bb842aac00e8f Mon Sep 17 00:00:00 2001 From: Peter-Phi-Tran Date: Wed, 24 Jun 2026 00:31:28 -0500 Subject: [PATCH 5/5] [IcebergIO] Run xlang wrapper validation expansion service on Java 17 The cross-language wrapper validation launches the io expansion-service jar from Python via JAVA_HOME. That jar bundles IcebergIO, now Java 17 bytecode, so launching it on the default Java 11 fails with UnsupportedClassVersionError. createCrossLanguageUsingJavaExpansionTask was missing the JAVA_HOME redirect that #38974 added to the other cross-language task factories. Add it, driven by -PtestJavaVersion (resolving java\Home), and pass -PtestJavaVersion=17 in the Xlang_Generated_Transforms precommit so the expansion service runs on a JDK that can load the bundled Java 17 IOs. --- .../beam_PreCommit_Xlang_Generated_Transforms.yml | 1 + .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml index 2f9741c01a01..aa8f5b3bb3fd 100644 --- a/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml +++ b/.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml @@ -117,4 +117,5 @@ jobs: with: gradle-command: :sdks:python:test-suites:direct:crossLanguageWrapperValidationPreCommit --info arguments: | + -PtestJavaVersion=17 -Pjava17Home=$JAVA_HOME_17_X64 diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 2f715846a8ae..95f8ee232695 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -2721,6 +2721,11 @@ class BeamModulePlugin implements Plugin { def usesDataflowRunner = config.pythonPipelineOptions.contains("--runner=TestDataflowRunner") || config.pythonPipelineOptions.contains("--runner=DataflowRunner") String ver = project.findProperty('testJavaVersion') def javaContainerSuffix = ver ? getSupportedJavaVersion(ver) : getSupportedJavaVersion() + // When a specific test JDK is requested (-PtestJavaVersion), launch the Python-started + // expansion service on it as well (see the exec block below). Some bundled IOs (e.g. + // IcebergIO) are compiled for Java 17, so the expansion service must run on a JDK that can + // load them. Mirrors the JAVA_HOME handling in the other cross-language task factories. + String testJavaHome = ver ? project.findProperty("java${ver}Home") : null // Sets up, collects, and runs Python pipeline tests project.tasks.register(config.name+"PythonUsingJava") { @@ -2750,6 +2755,9 @@ class BeamModulePlugin implements Plugin { project.exec { // environment variable to indicate that jars have been built environment "EXPANSION_JARS", config.expansionProjectPaths + if (testJavaHome) { + environment "JAVA_HOME", testJavaHome + } String additionalDependencyCmd = "" if (config.additionalDeps != null && !config.additionalDeps.isEmpty()){ additionalDependencyCmd = "&& pip install ${config.additionalDeps.join(' ')} "