apache · hemanthumashankar0511 · Jul 24, 2026 · Jul 24, 2026 · Jul 24, 2026 · Jul 24, 2026
diff --git a/itests/pom.xml b/itests/pom.xml
@@ -48,6 +48,7 @@
     <module>qtest-iceberg</module>
     <module>test-docker</module>
     <module>hive-iceberg</module>
+    <module>tez-yarn-it</module>
   </modules>
   <dependencyManagement>
     <dependencies>

diff --git a/itests/tez-yarn-it/pom.xml b/itests/tez-yarn-it/pom.xml
@@ -0,0 +1,171 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hive</groupId>
+    <artifactId>hive-it</artifactId>
+    <version>4.3.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
+  </parent>
+  <artifactId>hive-it-tez-yarn</artifactId>
+  <packaging>jar</packaging>
+  <name>Hive Integration - Tez on YARN localization tests</name>
+  <properties>
+    <hive.path.to.root>../..</hive.path.to.root>
+    <skip.tez.yarn.tests>true</skip.tez.yarn.tests>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>testcontainers</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <!-- Explicit Tez dependencies: Hive's Tez compilation/execution path needs these classes
+         on the in-process HS2 JVM classpath. In the main Hive build they are often profile-gated. -->
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-common</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-dag</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-runtime-library</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-runtime-internals</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>tez-mapreduce</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <!-- Provides org.apache.tez.hadoop.shim.HadoopShimsLoader, which DAGAppMaster.serviceInit()
+         loads at Tez AM startup. Normally transitive via tez-api; declared explicitly so it is
+         reliably on the classpath and gets staged into tez.lib.uris (its jar name has no "tez"). -->
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>hadoop-shim</artifactId>
+      <version>${tez.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-service</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-jdbc</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-it-util</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-antrun-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>generate-custom-hosts-file</id>
+            <!-- Must run after the parent "setup-test-dirs" execution (process-test-resources). -->
+            <phase>test-compile</phase>
+            <goals>
+              <goal>run</goal>
+            </goals>
+            <configuration>
+              <target>
+                <mkdir dir="${test.conf.dir}"/>
+                <copy file="${basedir}/src/test/resources/custom_hosts_file" tofile="${test.conf.dir}/custom_hosts_file" overwrite="true"/>
+                <exec executable="hostname" output="${test.conf.dir}/local_hostname.txt" failonerror="true"/>
+                <loadfile property="hive.tez.yarn.it.local.hostname" srcFile="${test.conf.dir}/local_hostname.txt">
+                  <filterchain>
+                    <striplinebreaks/>
+                  </filterchain>
+                </loadfile>
+                <!--
+                  When -Djdk.net.hosts.file is set, Java hostname resolution uses only that file.
+                  Ensure the local machine hostname is resolvable so Hive can start/compile queries.
+                -->
+                <echo file="${test.conf.dir}/custom_hosts_file" append="true">127.0.0.1 ${hive.tez.yarn.it.local.hostname}</echo>
+              </target>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <skipTests>${skip.tez.yarn.tests}</skipTests>
+          <!-- Resolve docker-network hostnames (namenode, resourcemanager, …) to 127.0.0.1
+               so HDFS URIs embedded in Tez LocalResources are reachable from both the host JVM
+               and inside YARN containers (via the shared docker network alias). -->
+          <argLine>${maven.test.jvm.args} -Xshare:off -Djdk.net.hosts.file=${test.conf.dir}/custom_hosts_file</argLine>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+  <profiles>
+    <profile>
+      <id>tez-yarn</id>
+      <properties>
+        <skip.tez.yarn.tests>false</skip.tez.yarn.tests>
+      </properties>
+    </profile>
+  </profiles>
+</project>
diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Custom Hadoop image for the Tez-on-YARN localization integration tests.
+#
+# The official apache/hadoop:3.4.2 image ships Java 8 for its HDFS/YARN daemons.
+# Hive is compiled for Java 21, so Tez AM and task containers -- which load
+# hive-exec classes -- must run on Java 21.  This image adds a Java 21 JDK at
+# /opt/jdk21 using a multi-stage build from eclipse-temurin:21-jdk; the daemons
+# continue to use the default Java 8.
+FROM --platform=linux/amd64 eclipse-temurin:21-jdk AS jdk21
+FROM apache/hadoop:3.4.2
+
+USER root
+COPY --from=jdk21 /opt/java/openjdk /opt/jdk21
+# Ensure the JDK binaries are executable and verify the copy is functional.
+# Fails at image-build time if the JDK is incompatible with the base image's
+# glibc, surfacing the problem early rather than silently at YARN container
+# launch time (where YARN shows exit code 1 and empty stderr).
+RUN chmod 755 /opt/jdk21/bin/java && /opt/jdk21/bin/java -version
+
+# commons-collections 3.2.2 is needed by the Tez AM's HDFS client path in
+# Tez 0.10.x; it is not bundled in apache/hadoop:3.4.2.
+RUN wget -q https://repo1.maven.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar \
+    -P /opt/hadoop/share/hadoop/common/lib/
+
+USER hadoop
diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CORE-SITE.XML_fs.default.name=hdfs://namenode
+CORE-SITE.XML_fs.defaultFS=hdfs://namenode
+
+HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020
+HDFS-SITE.XML_dfs.replication=1
+# Ensure the host JVM can reach the DataNode via a stable hostname when
+# -Djdk.net.hosts.file is used (and we publish DataNode ports in fixed-port mode).
+HDFS-SITE.XML_dfs.datanode.hostname=datanode
+HDFS-SITE.XML_dfs.client.use.datanode.hostname=true
+
+MAPRED-SITE.XML_mapreduce.framework.name=yarn
+MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
+MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
+MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME
+
+YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager
+YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle
+# Docker cgroup limits trip the default memory checks on NodeManager.
+YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false
+YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false
+YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600
+# Generous liveness expiry to tolerate slow container startup (e.g. under heavy load on CI).
+YARN-SITE.XML_yarn.nm.liveness-monitor.expiry-interval-ms=3600000
+YARN-SITE.XML_yarn.am.liveness-monitor.expiry-interval-ms=3600000
+YARN-SITE.XML_yarn.resourcemanager.nodemanagers.heartbeat-interval-ms=1000
+# Explicit absolute classpath so YARN container launch scripts do not rely on
+# $HADOOP_COMMON_HOME / $HADOOP_HDFS_HOME env-var expansion, which is not
+# available in child container environments (only the NM daemon's own env is).
+# Without this, tez.use.cluster.hadoop-libs=true leaves Hadoop jars off the
+# Tez AM classpath, causing the launch script to fail with exit code 1 and
+# empty stderr (the error fires before the java stderr redirect is active).
+YARN-SITE.XML_yarn.application.classpath=/opt/hadoop/etc/hadoop,/opt/hadoop/share/hadoop/common/*,/opt/hadoop/share/hadoop/common/lib/*,/opt/hadoop/share/hadoop/hdfs/*,/opt/hadoop/share/hadoop/hdfs/lib/*,/opt/hadoop/share/hadoop/yarn/*,/opt/hadoop/share/hadoop/yarn/lib/*,/opt/hadoop/share/hadoop/mapreduce/*,/opt/hadoop/share/hadoop/mapreduce/lib/*
+# Propagate HADOOP_HOME (and friends) into YARN containers so that tez.am.launch.env
+# overrides and any hadoop-config.sh sourcing in launch scripts can resolve correctly.
+YARN-SITE.XML_yarn.nodemanager.env-whitelist=JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME,HADOOP_HOME,PATH,LANG,TZ
+
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=*
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=*
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings=
+CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false