From 22f3eaf42c435d3d3c28ef53ab291d0e501d947b Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Fri, 24 Jul 2026 22:45:59 +0530 Subject: [PATCH 01/18] HIVE-29483: Add Tez-on-YARN resource localization integration test --- itests/pom.xml | 1 + itests/tez-yarn-it/pom.xml | 158 ++++++ .../src/test/docker/hadoop-yarn/Dockerfile | 40 ++ .../src/test/docker/hadoop-yarn/config | 65 +++ .../docker/hadoop-yarn/docker-compose.yml | 71 +++ .../tez/yarn/TestTezYarnLocalization.java | 339 +++++++++++++ .../tez/yarn/TestTezYarnLocalizationBase.java | 131 +++++ .../tez/yarn/TezYarnClusterContainer.java | 460 ++++++++++++++++++ .../tez/yarn/TezYarnClusterContainerTest.java | 108 ++++ .../src/test/resources/custom_hosts_file | 33 ++ .../src/test/resources/hive-site-yarn-it.xml | 63 +++ .../src/test/resources/yarn-site.xml | 33 ++ 12 files changed, 1502 insertions(+) create mode 100644 itests/tez-yarn-it/pom.xml create mode 100644 itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile create mode 100644 itests/tez-yarn-it/src/test/docker/hadoop-yarn/config create mode 100644 itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml create mode 100644 itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java create mode 100644 itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java create mode 100644 itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java create mode 100644 itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainerTest.java create mode 100644 itests/tez-yarn-it/src/test/resources/custom_hosts_file create mode 100644 itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml create mode 100644 itests/tez-yarn-it/src/test/resources/yarn-site.xml diff --git a/itests/pom.xml b/itests/pom.xml index 566bd2fde3dd..951d947ff4ca 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -48,6 +48,7 @@ qtest-iceberg test-docker hive-iceberg + tez-yarn-it diff --git a/itests/tez-yarn-it/pom.xml b/itests/tez-yarn-it/pom.xml new file mode 100644 index 000000000000..88f852b2d5b0 --- /dev/null +++ b/itests/tez-yarn-it/pom.xml @@ -0,0 +1,158 @@ + + + + 4.0.0 + + org.apache.hive + hive-it + 4.3.0-SNAPSHOT + ../pom.xml + + hive-it-tez-yarn + jar + Hive Integration - Tez on YARN localization tests + + ../.. + true + + + + org.testcontainers + testcontainers + test + + + + org.apache.tez + tez-api + test + + + org.apache.tez + tez-common + test + + + org.apache.tez + tez-dag + test + + + org.apache.tez + tez-runtime-library + test + + + org.apache.tez + tez-runtime-internals + test + + + org.apache.tez + tez-mapreduce + test + + + org.apache.hive + hive-service + test + + + org.apache.hive + hive-jdbc + test + + + org.apache.hive + hive-exec + test + + + org.apache.hive + hive-it-util + test + + + org.apache.hadoop + hadoop-common + test + + + org.apache.hadoop + hadoop-mapreduce-client-core + test + + + junit + junit + test + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-custom-hosts-file + + test-compile + + run + + + + + + + + + + + + + 127.0.0.1 ${hive.tez.yarn.it.local.hostname} + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skip.tez.yarn.tests} + + ${maven.test.jvm.args} -Xshare:off -Djdk.net.hosts.file=${test.conf.dir}/custom_hosts_file + + + + + + + tez-yarn + + false + + + + diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile new file mode 100644 index 000000000000..ac2f824f9f31 --- /dev/null +++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Custom Hadoop image for the Tez-on-YARN localization integration tests. +# +# The official apache/hadoop:3.4.2 image ships Java 8 for its HDFS/YARN daemons. +# Hive is compiled for Java 21, so Tez AM and task containers -- which load +# hive-exec classes -- must run on Java 21. This image adds a Java 21 JDK at +# /opt/jdk21 using a multi-stage build from eclipse-temurin:21-jdk; the daemons +# continue to use the default Java 8. +FROM eclipse-temurin:21-jdk AS jdk21 +FROM apache/hadoop:3.4.2 + +USER root +COPY --from=jdk21 /opt/java/openjdk /opt/jdk21 +# Ensure the JDK binaries are executable and verify the copy is functional. +# Fails at image-build time if the JDK is incompatible with the base image's +# glibc, surfacing the problem early rather than silently at YARN container +# launch time (where YARN shows exit code 1 and empty stderr). +RUN chmod 755 /opt/jdk21/bin/java && /opt/jdk21/bin/java -version + +# commons-collections 3.2.2 is needed by the Tez AM's HDFS client path in +# Tez 0.10.x; it is not bundled in apache/hadoop:3.4.2. +RUN wget -q https://repo1.maven.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar \ + -P /opt/hadoop/share/hadoop/common/lib/ + +USER hadoop diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config new file mode 100644 index 000000000000..a61614b76146 --- /dev/null +++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode + +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +# Ensure the host JVM can reach the DataNode via a stable hostname when +# -Djdk.net.hosts.file is used (and we publish DataNode ports in fixed-port mode). +HDFS-SITE.XML_dfs.datanode.hostname=datanode +HDFS-SITE.XML_dfs.client.use.datanode.hostname=true + +MAPRED-SITE.XML_mapreduce.framework.name=yarn +MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME + +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +# Docker cgroup limits trip the default memory checks on NodeManager. +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +# Generous liveness expiry to tolerate slow container startup (e.g. under heavy load on CI). +YARN-SITE.XML_yarn.nm.liveness-monitor.expiry-interval-ms=3600000 +YARN-SITE.XML_yarn.am.liveness-monitor.expiry-interval-ms=3600000 +YARN-SITE.XML_yarn.resourcemanager.nodemanagers.heartbeat-interval-ms=1000 +# Explicit absolute classpath so YARN container launch scripts do not rely on +# $HADOOP_COMMON_HOME / $HADOOP_HDFS_HOME env-var expansion, which is not +# available in child container environments (only the NM daemon's own env is). +# Without this, tez.use.cluster.hadoop-libs=true leaves Hadoop jars off the +# Tez AM classpath, causing the launch script to fail with exit code 1 and +# empty stderr (the error fires before the java stderr redirect is active). +YARN-SITE.XML_yarn.application.classpath=/opt/hadoop/etc/hadoop,/opt/hadoop/share/hadoop/common/*,/opt/hadoop/share/hadoop/common/lib/*,/opt/hadoop/share/hadoop/hdfs/*,/opt/hadoop/share/hadoop/hdfs/lib/*,/opt/hadoop/share/hadoop/yarn/*,/opt/hadoop/share/hadoop/yarn/lib/*,/opt/hadoop/share/hadoop/mapreduce/*,/opt/hadoop/share/hadoop/mapreduce/lib/* +# Propagate HADOOP_HOME (and friends) into YARN containers so that tez.am.launch.env +# overrides and any hadoop-config.sh sourcing in launch scripts can resolve correctly. +YARN-SITE.XML_yarn.nodemanager.env-whitelist=JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME,HADOOP_HOME,PATH,LANG,TZ + +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml new file mode 100644 index 000000000000..72775ddfa1fe --- /dev/null +++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2" +services: + namenode: + image: apache/hadoop:3.4.2 + hostname: namenode + command: ["hdfs", "namenode"] + env_file: + - ./config + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-hadoop/dfs/name" + ports: + - "9870:9870" + - "8020:8020" + networks: + - hive-yarn-it + + datanode: + image: apache/hadoop:3.4.2 + command: ["hdfs", "datanode"] + env_file: + - ./config + depends_on: + - namenode + networks: + - hive-yarn-it + + resourcemanager: + image: apache/hadoop:3.4.2 + hostname: resourcemanager + command: ["yarn", "resourcemanager"] + env_file: + - ./config + ports: + - "8088:8088" + - "8032:8032" + depends_on: + - namenode + networks: + - hive-yarn-it + + nodemanager: + image: apache/hadoop:3.4.2 + command: ["yarn", "nodemanager"] + env_file: + - ./config + ports: + - "8042:8042" + depends_on: + - resourcemanager + networks: + - hive-yarn-it + +networks: + hive-yarn-it: + name: hive-yarn-it diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java new file mode 100644 index 000000000000..a128d1192230 --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -0,0 +1,339 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; + +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * End-to-end integration test for HIVE-29483: verifies that {@code hive-exec.jar} + * is correctly localized by Tez on YARN. + * + *

The test starts a real Hadoop YARN+HDFS cluster in Docker containers + * (fixed host ports so the same URIs work from both the host JVM and inside + * YARN containers), stages Tez framework jars to HDFS, brings up an in-process + * HiveServer2 pointed at that cluster, and then runs an actual Tez DAG via JDBC. + * + *

If {@code hive-exec.jar} were missing from {@code commonLocalResources} + * (i.e. if {@code TezSessionState.appJarLr} were removed), the Tez task + * containers would fail with {@code ClassNotFoundException} for Hive executor + * classes and this test would fail. + * + *

Prerequisites

+ *
    + *
  • Docker must be running on the host.
    • + *
  • Host ports 8020, 9870, 8032, 8088 must be free.
    • + *
  • The test JVM must resolve {@code namenode} and {@code resourcemanager} + * to {@code 127.0.0.1}. When run via Maven this is automatic because + * Surefire passes {@code -Djdk.net.hosts.file=custom_hosts_file}.
    • + *
+ * + *

Running locally

+ * 
+ * mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \
+ *     -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar
+ *
+ */ +public class TestTezYarnLocalization { + + private static final Logger LOG = LoggerFactory.getLogger(TestTezYarnLocalization.class); + + private static final String HDFS_BASE = "hdfs://namenode:8020"; + private static final String HDFS_WAREHOUSE = HDFS_BASE + "/tmp/hive-tez-loc/warehouse"; + private static final String HDFS_SCRATCH = HDFS_BASE + "/tmp/hive-tez-loc/scratch"; + private static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(true); + cluster.start(); + + GenericContainer nn = cluster.namenodeContainer(); + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/warehouse"); + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/scratch"); + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/user-install"); + nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", HDFS_ROOT); + + String tezLibUris = cluster.uploadTezLibsToHdfs(); + LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); + + Path localScratch = Files.createTempDirectory("hive-tez-loc-"); + HiveConf conf = buildHiveConf(tezLibUris, localScratch); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + waitForJdbc(hs2Port); + LOG.info("HiveServer2 is ready on port {}", hs2Port); + } + + @AfterClass + public static void stopAll() { + // Always dump NM diagnostics before tearing down containers — logs disappear on stop(). + dumpNodeManagerDiagnostics(); + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + } + + /** + * Positive test case for HIVE-29483. + * + *

Runs a Tez DAG via JDBC and asserts that: + *

    + *
  1. The query returns the expected result (primary proof that + * {@code hive-exec.jar} was localized and Hive executor classes + * were available inside YARN task containers).
    2. + *
  2. A Tez YARN application is visible in the ResourceManager + * (secondary confirmation that the DAG actually ran on YARN).
    3. + *
+ */ + @Test + public void testQuerySucceedsWithAppJar() throws Exception { + String url = jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + try (Statement stmt = conn.createStatement()) { + + stmt.execute("CREATE TABLE IF NOT EXISTS tez_loc_test (id INT) STORED AS ORC"); + stmt.execute("INSERT INTO tez_loc_test VALUES (1), (2), (3)"); + + // COUNT(*) forces a Tez reduce task. If hive-exec.jar is not localized the task + // container fails with ClassNotFoundException before this assertion is reached. + try (ResultSet rs = stmt.executeQuery("SELECT count(*) FROM tez_loc_test")) { + Assert.assertTrue("Result set must contain at least one row", rs.next()); + long count = rs.getLong(1); + Assert.assertEquals( + "SELECT count(*) FROM tez_loc_test should return 3 (hive-exec.jar was localized)", + 3L, count); + LOG.info("Tez query succeeded: count(*) = {}", count); + } + } + } + + verifyTezYarnAppExists(); + } + + /** + * Dumps NodeManager container launch scripts and log files when a Tez job fails. + * + *

A failing Tez AM often shows an empty {@code stderr} in the YARN diagnostic, which is + * misleading: Tez logs through log4j to the container's {@code syslog} file, not {@code stderr}. + * This helper dumps four sources so CI can pinpoint the real cause without SSH into a container: + *

    + *
  • {@code launch_container.sh} — the exact classpath / env-var setup
    • + *
  • container {@code stderr} — JVM-level errors (empty when the failure is logged via log4j)
    • + *
  • container {@code syslog} — the Tez AM's log4j output (where NoClassDefFoundError etc. lands)
    • + *
  • the NodeManager daemon log — pre-exec launch-script failures that bypass container stderr
    • + *
+ */ + private static void dumpNodeManagerDiagnostics() { + if (cluster == null) { + return; + } + // Print to System.out (not SLF4J): with -Dsurefire.useFile=false Surefire streams the + // forked JVM's stdout into the build log, whereas Hive routes LOG.* to target/tmp/log/hive.log + // which is invisible in CI. This runs from @AfterClass BEFORE the containers are stopped, + // so the NM container is still alive and its log dirs still exist (delete-debug-delay=600s). + System.out.println("########## BEGIN NodeManager diagnostics ##########"); + try { + // launch_container.sh reveals the EXACT java command, JAVA_HOME and full classpath the + // AM was launched with -- decisive when the AM exits 1 with empty stderr/prelaunch.err. + dumpNmCommand("launch_container.sh (AM launch command + classpath)", + "find /tmp -name 'launch_container.sh' 2>/dev/null | head -3 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + // syslog = Tez AM log4j output (NoClassDefFoundError etc. land here, not in stderr). + dumpNmCommand("container syslog (Tez AM log4j output)", + "find /tmp -name 'syslog' -path '*/container_*/*' 2>/dev/null | head -5 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + // stderr / prelaunch.err (usually empty, but confirm). + dumpNmCommand("container stderr + prelaunch.err", + "find /tmp \\( -name 'stderr' -o -name 'prelaunch.err' \\) -path '*/container_*/*' 2>/dev/null | head -10 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + // NodeManager daemon log: pre-exec launch-script failures that bypass container stderr. + dumpNmCommand("NodeManager daemon log (tail 200)", + "find /opt/hadoop/logs -name '*.log' 2>/dev/null | head -2 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; tail -200 {}' 2>/dev/null || true"); + } catch (Exception e) { + System.out.println("Could not dump NodeManager diagnostics: " + e); + } + System.out.println("########## END NodeManager diagnostics ##########"); + System.out.flush(); + } + + private static void dumpNmCommand(String label, String bashCommand) { + try { + GenericContainer.ExecResult r = + cluster.nodeManagerContainer().execInContainer("bash", "-c", bashCommand); + String out = r.getStdout(); + System.out.println("===== NM: " + label + " ====="); + System.out.println(out.isEmpty() ? "(no output found)" : out); + } catch (Exception e) { + System.out.println("===== NM: " + label + " (dump failed: " + e + ") ====="); + } + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) throws Exception { + HiveConf conf = new HiveConf(); + + URL hiveSite = TestTezYarnLocalization.class.getClassLoader().getResource("hive-site-yarn-it.xml"); + URL yarnSite = TestTezYarnLocalization.class.getClassLoader().getResource("yarn-site.xml"); + if (hiveSite != null) { conf.addResource(hiveSite); } + if (yarnSite != null) { conf.addResource(yarnSite); } + + conf.set("fs.defaultFS", HDFS_BASE); + // Force the HDFS client to use DataNode hostnames (resolvable via custom_hosts_file) + // rather than docker-internal IPs that are unreachable from the host JVM. + conf.setBoolean("dfs.client.use.datanode.hostname", true); + conf.set("hive.metastore.warehouse.dir", HDFS_WAREHOUSE); + conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, HDFS_SCRATCH); + conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname, localScratch.toAbsolutePath().toString()); + // hive-exec.jar is uploaded to this HDFS path by DagUtils when hive.jar.directory is unset. + conf.setVar(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR, HDFS_ROOT + "/user-install"); + + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); + + conf.set("yarn.resourcemanager.hostname", "resourcemanager"); + conf.set("yarn.resourcemanager.address", "resourcemanager:8032"); + conf.set("yarn.resourcemanager.webapp.address", "resourcemanager:8088"); + + // tez.lib.uris provides Tez framework jars to YARN containers. tez.use.cluster.hadoop-libs + // tells Tez to source Hadoop jars from the cluster node (/opt/hadoop) rather than requiring + // them in tez.lib.uris, which would require staging the full Hadoop distribution to HDFS. + conf.set("tez.lib.uris", tezLibUris); + conf.setBoolean("tez.use.cluster.hadoop-libs", true); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS, false); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 0); + + // The Hadoop image daemons use Java 8 but hive-exec requires Java 21; point Tez AM/task + // container environments at the Java 21 runtime added by the custom Dockerfile. + // HADOOP_HOME must be set explicitly: the apache/hadoop image does not export it into YARN + // container environments, so tez.use.cluster.hadoop-libs's "${HADOOP_HOME}/bin/hadoop classpath" + // call would silently produce an empty classpath without it. + String containerEnv = "JAVA_HOME=" + TezYarnClusterContainer.CONTAINER_JAVA_21_HOME + + ",HADOOP_HOME=/opt/hadoop" + + ",HADOOP_MAPRED_HOME=/opt/hadoop"; + conf.set("tez.am.launch.env", containerEnv); + conf.set("tez.task.launch.env", containerEnv); + conf.set("yarn.app.mapreduce.am.env", containerEnv); + conf.set("mapreduce.map.env", containerEnv); + conf.set("mapreduce.reduce.env", containerEnv); + + hs2Port = findFreePort(); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, "localhost"); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort()); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, "NOSASL"); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + + return conf; + } + + /** + * Checks that at least one Tez YARN application is registered with the + * ResourceManager. If the check cannot be performed (e.g. due to a + * container exec error), a warning is logged and the test continues — + * the primary assertion is the query result check in the calling test. + */ + private static void verifyTezYarnAppExists() { + try { + GenericContainer rm = cluster.resourceManagerContainer(); + GenericContainer.ExecResult result = rm.execInContainer( + "yarn", "application", "-list", "-appTypes", "TEZ", "-appStates", "ALL"); + String out = result.getStdout(); + LOG.info("YARN application list (TEZ, ALL states):\n{}", out); + + Pattern appIdPattern = Pattern.compile("(application_\\d+_\\d+)"); + Matcher matcher = appIdPattern.matcher(out); + boolean found = false; + while (matcher.find()) { + LOG.info("Found Tez YARN application: {}", matcher.group(1)); + found = true; + } + + Assert.assertTrue( + "At least one Tez YARN application must be visible in the ResourceManager after running a Tez query", + found); + + } catch (Exception e) { + LOG.warn("Could not verify Tez YARN application existence via RM exec; " + + "primary query-result assertion already passed. Cause: {}", e.getMessage()); + } + } + + private static void waitForJdbc(int port) throws InterruptedException { + String url = jdbcUrl(port); + long deadline = System.currentTimeMillis() + 120_000; + while (System.currentTimeMillis() < deadline) { + try (Connection c = DriverManager.getConnection(url, "hive", "")) { + return; + } catch (Exception ignored) { + Thread.sleep(2000); + } + } + throw new IllegalStateException( + "HiveServer2 JDBC endpoint not reachable on port " + port + " after 120s"); + } + + private static String jdbcUrl(int port) { + return "jdbc:hive2://localhost:" + port + "/default;auth=noSasl"; + } + + private static int findFreePort() throws Exception { + try (ServerSocket s = new ServerSocket(0)) { + s.setReuseAddress(true); + return s.getLocalPort(); + } + } +} diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java new file mode 100644 index 000000000000..ff6292172dc4 --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; + +public class TestTezYarnLocalizationBase { + + protected static TezYarnClusterContainer cluster; + protected static HiveServer2 hs2; + protected static int hs2Port; + protected static String hdfsUri; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(); + cluster.start(); + + hdfsUri = cluster.getHdfsUri(); + + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/hive-29483/warehouse"); + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/hive-29483/scratch"); + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp/hive-29483"); + + Path localScratch = Files.createTempDirectory("hive-29483-local-"); + + HiveConf conf = new HiveConf(); + URL hiveSite = TestTezYarnLocalizationBase.class.getClassLoader().getResource("hive-site-yarn-it.xml"); + URL yarnSite = TestTezYarnLocalizationBase.class.getClassLoader().getResource("yarn-site.xml"); + if (hiveSite != null) conf.addResource(hiveSite); + if (yarnSite != null) conf.addResource(yarnSite); + + conf.set("fs.defaultFS", hdfsUri); + conf.set("hive.metastore.warehouse.dir", hdfsUri + "/tmp/hive-29483/warehouse"); + conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, hdfsUri + "/tmp/hive-29483/scratch"); + conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname, localScratch.toAbsolutePath().toString()); + + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); + + conf.set("yarn.resourcemanager.address", cluster.getResourceManagerAddress()); + conf.set("yarn.resourcemanager.webapp.address", cluster.getResourceManagerWebAppAddress()); + + hs2Port = findFreePort(); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, "localhost"); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort()); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, "NOSASL"); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + waitForJdbc(hs2Port); + } + + @AfterClass + public static void stopAll() { + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + } + + @Test + public void testJdbcSessionOpen() throws Exception { + String url = jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + Assert.assertNotNull("JDBC connection must not be null", conn); + } + } + + private static void waitForJdbc(int port) throws InterruptedException { + String url = jdbcUrl(port); + long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + try (Connection c = DriverManager.getConnection(url, "hive", "")) { + return; + } catch (Exception ignored) { + Thread.sleep(2000); + } + } + throw new IllegalStateException("HiveServer2 JDBC endpoint not reachable on port " + port + " after 60s"); + } + + private static String jdbcUrl(int port) { + // HS2 is configured with HIVE_SERVER2_AUTHENTICATION=NOSASL, which requires the client to + // set auth=noSasl explicitly. + return "jdbc:hive2://localhost:" + port + "/default;auth=noSasl"; + } + + private static int findFreePort() throws Exception { + try (ServerSocket s = new ServerSocket(0)) { + s.setReuseAddress(true); + return s.getLocalPort(); + } + } +} diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java new file mode 100644 index 000000000000..0f46b4dcf960 --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java @@ -0,0 +1,460 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.FixedHostPortGenericContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +/** + * Manages a pseudo-distributed HDFS+YARN cluster (NameNode, DataNode, + * ResourceManager, NodeManager) using a custom image built from + * apache/hadoop:3.4.2 (see {@code src/test/docker/hadoop-yarn/Dockerfile}). + * The image adds a Java 21 runtime at {@link #CONTAINER_JAVA_21_HOME} so that + * Tez AM/task containers (which load hive-exec compiled for Java 21) can run, + * while the HDFS/YARN daemons keep the image default Java 8. Intended for + * integration tests that need real YARN container scheduling and HDFS-backed + * resource localization. + * + *

All four containers share a single Docker network and are configured via + * the same environment-variable convention used by the official image (the + * env var key encodes the target XML file and property name, e.g. + * {@code CORE-SITE.XML_fs.defaultFS=hdfs://namenode}). + * + *

Two modes are supported: + *

    + *
  • Dynamic-port mode (default, {@code new TezYarnClusterContainer()}): + * Testcontainers assigns random host ports. Use for tests where only the + * host JVM accesses HDFS/YARN (e.g. JDBC connectivity smoke tests).
    • + *
  • Fixed-port mode ({@code new TezYarnClusterContainer(true)}): + * Host ports match container ports (NN: 8020/9870, RM: 8032/8088). + * The docker-network hostnames {@code namenode} and {@code resourcemanager} + * must resolve to {@code 127.0.0.1} on the host, e.g. via + * {@code -Djdk.net.hosts.file=custom_hosts_file}. Use for tests where + * YARN containers also need to reach HDFS by the same URI as the host JVM + * (real Tez-on-YARN localization tests).
    • + *
+ */ +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + /** + * Path (inside the containers) to the Java 21 runtime added by the custom Dockerfile. + * The HDFS/YARN daemons keep the image default Java 8, but the Tez AM/task containers + * (which load hive-exec compiled for Java 21) must be pointed here via {@code JAVA_HOME}. + */ + public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; + + // Custom image: apache/hadoop:3.4.2 + Corretto 21 at /opt/jdk21 (see src/test/docker/hadoop-yarn/Dockerfile). + private static final String HADOOP_IMAGE = buildHadoopImage(); + private static final Duration STARTUP_TIMEOUT = Duration.ofMinutes(3); + private static final Map COMMON_ENV = loadCommonEnv(); + + // Fixed host ports used when fixedPorts=true. Must match container-internal ports. + private static final int NN_RPC_PORT = 8020; + private static final int NN_HTTP_PORT = 9870; + private static final int RM_RPC_PORT = 8032; + private static final int RM_HTTP_PORT = 8088; + // DataNode ports (Hadoop 3.x defaults). Host JVM must reach 9866 for HDFS writes. + private static final int DN_HTTP_PORT = 9864; + private static final int DN_XFER_PORT = 9866; + + private final Network network; + private final GenericContainer namenode; + private final GenericContainer datanode; + private final GenericContainer resourcemanager; + private final GenericContainer nodemanager; + private final boolean fixedPorts; + + /** Dynamic-port mode; backward-compatible with existing tests. */ + public TezYarnClusterContainer() { + this(false); + } + + /** + * @param fixedPorts when {@code true}, maps host ports 8020/9870/8032/8088 + * to the matching container ports so that the same hostnames + * and ports work from both the host JVM and inside containers. + */ + public TezYarnClusterContainer(boolean fixedPorts) { + this.fixedPorts = fixedPorts; + network = Network.newNetwork(); + + if (fixedPorts) { + namenode = new FixedHostPortGenericContainer<>(HADOOP_IMAGE) + .withFixedExposedPort(NN_RPC_PORT, NN_RPC_PORT) + .withFixedExposedPort(NN_HTTP_PORT, NN_HTTP_PORT) + .withNetwork(network) + .withNetworkAliases("namenode") + .withCommand("hdfs", "namenode") + .withEnv(COMMON_ENV) + .withEnv("ENSURE_NAMENODE_DIR", "/tmp/hadoop-hadoop/dfs/name") + .waitingFor(Wait.forHttp("/").forPort(NN_HTTP_PORT).withStartupTimeout(STARTUP_TIMEOUT)); + + resourcemanager = new FixedHostPortGenericContainer<>(HADOOP_IMAGE) + .withFixedExposedPort(RM_RPC_PORT, RM_RPC_PORT) + .withFixedExposedPort(RM_HTTP_PORT, RM_HTTP_PORT) + .withNetwork(network) + .withNetworkAliases("resourcemanager") + .withCommand("yarn", "resourcemanager") + .withEnv(COMMON_ENV) + .waitingFor(Wait.forHttp("/ws/v1/cluster/info").forPort(RM_HTTP_PORT).withStartupTimeout(STARTUP_TIMEOUT)); + + datanode = new FixedHostPortGenericContainer<>(HADOOP_IMAGE) + .withFixedExposedPort(DN_XFER_PORT, DN_XFER_PORT) + .withFixedExposedPort(DN_HTTP_PORT, DN_HTTP_PORT) + .withNetwork(network) + .withNetworkAliases("datanode") + .withCommand("hdfs", "datanode") + .withEnv(COMMON_ENV); + } else { + namenode = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("namenode") + .withCommand("hdfs", "namenode") + .withEnv(COMMON_ENV) + .withEnv("ENSURE_NAMENODE_DIR", "/tmp/hadoop-hadoop/dfs/name") + .withExposedPorts(NN_HTTP_PORT, NN_RPC_PORT) + .waitingFor(Wait.forHttp("/").forPort(NN_HTTP_PORT).withStartupTimeout(STARTUP_TIMEOUT)); + + resourcemanager = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("resourcemanager") + .withCommand("yarn", "resourcemanager") + .withEnv(COMMON_ENV) + .withExposedPorts(RM_HTTP_PORT, RM_RPC_PORT) + .waitingFor(Wait.forHttp("/ws/v1/cluster/info").forPort(RM_HTTP_PORT).withStartupTimeout(STARTUP_TIMEOUT)); + + datanode = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("datanode") + .withCommand("hdfs", "datanode") + .withEnv(COMMON_ENV); + } + + nodemanager = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("nodemanager") + .withCommand("yarn", "nodemanager") + .withEnv(COMMON_ENV); + } + + /** + * Starts NN → DN → RM → NM in order and then polls until at least one + * NodeManager has registered with the ResourceManager. + */ + public void start() { + namenode.start(); + datanode.start(); + resourcemanager.start(); + nodemanager.start(); + waitForNodeManagerRegistration(); + verifyJava21InNodeManager(); + } + + /** + * Verifies that {@link #CONTAINER_JAVA_21_HOME}/bin/java is functional inside the + * NodeManager container. Logs a clear warning if not — this will cause all Tez AM and + * task containers to fail with exit code 1 and empty stderr (the JVM never starts, so + * nothing is written to the YARN container's stderr file before the launch script exits). + */ + private void verifyJava21InNodeManager() { + try { + GenericContainer.ExecResult r = nodemanager.execInContainer( + CONTAINER_JAVA_21_HOME + "/bin/java", "-version"); + if (r.getExitCode() == 0) { + LOG.info("Java 21 is functional in NodeManager ({}): {}", + CONTAINER_JAVA_21_HOME, r.getStderr().trim()); + } else { + LOG.warn("Java 21 check FAILED in NodeManager (exit {}). " + + "Tez AM/task containers will fail at launch time. " + + "stderr: {}", r.getExitCode(), r.getStderr()); + } + } catch (Exception e) { + LOG.warn("Could not verify Java 21 in NodeManager container", e); + } + } + + public void stop() { + nodemanager.stop(); + resourcemanager.stop(); + datanode.stop(); + namenode.stop(); + network.close(); + } + + /** + * Returns the HDFS URI for use in {@code HiveConf} / {@code Configuration}. + * + *

In fixed-port mode, returns {@code hdfs://namenode:8020} — a URI that + * works from both the host JVM (via the custom hosts file) and from inside + * YARN containers (via the docker network alias). In dynamic-port mode, + * returns {@code hdfs://localhost:}. + */ + public String getHdfsUri() { + if (fixedPorts) { + return "hdfs://namenode:" + NN_RPC_PORT; + } + return "hdfs://" + namenode.getHost() + ":" + namenode.getMappedPort(NN_RPC_PORT); + } + + /** + * Returns {@code host:port} of the ResourceManager RPC address, suitable for + * {@code yarn.resourcemanager.address}. + */ + public String getResourceManagerAddress() { + if (fixedPorts) { + return "resourcemanager:" + RM_RPC_PORT; + } + return resourcemanager.getHost() + ":" + resourcemanager.getMappedPort(RM_RPC_PORT); + } + + /** + * Returns {@code host:port} of the ResourceManager web-application address, + * suitable for {@code yarn.resourcemanager.webapp.address}. + */ + public String getResourceManagerWebAppAddress() { + if (fixedPorts) { + return "resourcemanager:" + RM_HTTP_PORT; + } + return resourcemanager.getHost() + ":" + resourcemanager.getMappedPort(RM_HTTP_PORT); + } + + /** + * Copies a local jar into the HDFS directory {@code /tmp/hive-29483-jars} and returns + * the resulting HDFS path. Used to seed localized resources in integration tests. + */ + public String uploadJarToHdfs(Path localJarPath) throws IOException, InterruptedException { + String fileName = localJarPath.getFileName().toString(); + String containerTmp = "/tmp/" + fileName; + String hdfsDir = "/tmp/hive-29483-jars"; + String hdfsPath = hdfsDir + "/" + fileName; + + // Ensure the file is readable by the container's default user (hadoop). + namenode.copyFileToContainer(MountableFile.forHostPath(localJarPath, 0644), containerTmp); + + GenericContainer.ExecResult mkdir = namenode.execInContainer("hdfs", "dfs", "-mkdir", "-p", hdfsDir); + requireSuccess(mkdir, "hdfs dfs -mkdir -p " + hdfsDir); + + GenericContainer.ExecResult put = namenode.execInContainer("hdfs", "dfs", "-put", "-f", containerTmp, hdfsPath); + requireSuccess(put, "hdfs dfs -put -f " + containerTmp + " " + hdfsPath); + + return hdfsPath; + } + + /** + * Discovers Tez jars on the test JVM classpath, uploads them to + * {@code /tmp/hive-29483/tez-libs/} on HDFS, and returns a comma-separated + * list of fully-qualified HDFS URIs suitable for the {@code tez.lib.uris} + * configuration property. + * + *

Only available in fixed-port mode; the returned URIs use the stable + * hostname {@code namenode} which must be resolvable from both the host JVM + * and YARN containers. + * + * @throws IllegalStateException if no Tez jars are found on the classpath + */ + public String uploadTezLibsToHdfs() throws IOException, InterruptedException { + String hdfsDir = "/tmp/hive-29483/tez-libs"; + GenericContainer.ExecResult mkdir = namenode.execInContainer("hdfs", "dfs", "-mkdir", "-p", hdfsDir); + requireSuccess(mkdir, "hdfs dfs -mkdir -p " + hdfsDir); + + Set tezJars = findTezJarsFromClasspath(); + List hdfsUris = new ArrayList<>(); + for (Path jarPath : tezJars) { + String jarName = jarPath.getFileName().toString(); + String containerTmp = "/tmp/" + jarName; + String hdfsPath = hdfsDir + "/" + jarName; + + namenode.copyFileToContainer(MountableFile.forHostPath(jarPath, 0644), containerTmp); + GenericContainer.ExecResult put = namenode.execInContainer( + "hdfs", "dfs", "-put", "-f", containerTmp, hdfsPath); + requireSuccess(put, "hdfs dfs -put -f " + containerTmp + " " + hdfsPath); + + hdfsUris.add("hdfs://namenode:" + NN_RPC_PORT + hdfsPath); + } + + if (hdfsUris.isEmpty()) { + throw new IllegalStateException( + "No Tez jars (filenames containing 'tez') were found on the test classpath. " + + "Ensure tez-api, tez-dag, tez-runtime-library, etc. are transitive dependencies."); + } + return String.join(",", hdfsUris); + } + + /** + * Discovers Tez jars from the running JVM's classpath using two complementary strategies: + *

    + *
  1. Reflection on known Tez probe classes — works regardless of how Surefire + * passes the classpath (direct {@code -cp} or manifest-only argfile JAR).
    2. + *
  2. Scanning {@link ManagementFactory#getRuntimeMXBean() RuntimeMXBean#getClassPath()} + * for entries whose filename contains {@code "tez"} — a safety net when the + * reflection approach misses a jar (e.g. classes not loaded yet).
    3. + *
+ * Only regular {@code .jar} files whose filename contains {@code "tez"} and does + * not end with {@code "-tests.jar"} are returned. + */ + private static Set findTezJarsFromClasspath() { + Set jars = new LinkedHashSet<>(); + + // Strategy 1: reflect on known Tez probe classes to get their source jar. + // Use Class.forName to avoid compile-time imports of Tez internal modules. + String[] probeClassNames = { + "org.apache.tez.dag.api.TezConfiguration", // tez-api + "org.apache.tez.common.TezConverterUtils", // tez-common + "org.apache.tez.dag.app.DAGAppMaster", // tez-dag (the Tez AM) + "org.apache.tez.mapreduce.hadoop.MRHelpers", // tez-mapreduce + "org.apache.tez.runtime.LogicalIOProcessorRuntimeTask", // tez-runtime-internals + "org.apache.tez.runtime.library.api.KeyValueReader" // tez-runtime-library + }; + for (String className : probeClassNames) { + try { + Class cls = Class.forName(className); + CodeSource cs = cls.getProtectionDomain().getCodeSource(); + if (cs != null && cs.getLocation() != null) { + String path = cs.getLocation().getPath(); + if (path.endsWith(".jar")) { + Path p = Paths.get(path); + String name = p.getFileName().toString(); + if (name.contains("tez") && !name.endsWith("-tests.jar") && Files.isRegularFile(p)) { + jars.add(p); + } + } + } + } catch (ClassNotFoundException | SecurityException ignored) { + // Class not on classpath; skip. + } + } + + // Strategy 2: scan the JVM classpath string for jars with "tez" in the name. + String cp = ManagementFactory.getRuntimeMXBean().getClassPath(); + for (String entry : cp.split(File.pathSeparator)) { + if (!entry.endsWith(".jar")) { + continue; + } + Path p = Paths.get(entry); + String name = p.getFileName().toString(); + if (name.contains("tez") && !name.endsWith("-tests.jar") && Files.isRegularFile(p)) { + jars.add(p); + } + } + + return jars; + } + + // Package-private: only test classes in this package need direct exec access. + GenericContainer namenodeContainer() { + return namenode; + } + + GenericContainer resourceManagerContainer() { + return resourcemanager; + } + + GenericContainer nodeManagerContainer() { + return nodemanager; + } + + private void waitForNodeManagerRegistration() { + long deadline = System.currentTimeMillis() + Duration.ofMinutes(2).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + GenericContainer.ExecResult result = resourcemanager.execInContainer("yarn", "node", "-list"); + String out = result.getStdout(); + if (out.contains("Total Nodes:") && !out.contains("Total Nodes:0")) { + return; + } + Thread.sleep(3000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for NodeManager registration"); + } catch (Exception ignored) { + try { Thread.sleep(3000); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } + } + } + throw new IllegalStateException("NodeManager did not register with ResourceManager within 2 minutes"); + } + + private static void requireSuccess(GenericContainer.ExecResult result, String cmd) { + if (result.getExitCode() != 0) { + throw new IllegalStateException("Command failed (" + cmd + ")\nstdout:\n" + + result.getStdout() + "\nstderr:\n" + result.getStderr()); + } + } + + private static Map loadCommonEnv() { + Map env = new LinkedHashMap<>(); + String basedir = System.getProperty("basedir", "."); + Path configPath = Paths.get(basedir, "src/test/docker/hadoop-yarn/config"); + try (Stream lines = Files.lines(configPath, StandardCharsets.UTF_8)) { + lines.map(String::trim) + .filter(l -> !l.isEmpty()) + .filter(l -> !l.startsWith("#")) + .forEach(l -> { + int idx = l.indexOf('='); + if (idx < 0) { + throw new IllegalArgumentException("Invalid config line (missing '='): " + l); + } + String key = l.substring(0, idx).trim(); + String value = l.substring(idx + 1); + env.put(key, value); + }); + } catch (IOException e) { + throw new IllegalStateException("Failed to load Hadoop docker config from " + configPath, e); + } + return env; + } + + /** + * Builds (or reuses a cached build of) the custom Hadoop image that layers a Java 21 + * runtime on top of apache/hadoop:3.4.2. Testcontainers caches the build by content + * hash, so repeated runs reuse the previously built image. + * + * @return the resulting local image tag + */ + private static String buildHadoopImage() { + String basedir = System.getProperty("basedir", "."); + Path dockerfile = Paths.get(basedir, "src/test/docker/hadoop-yarn/Dockerfile"); + return new ImageFromDockerfile("hive-it-hadoop-jdk21", false) + .withDockerfile(dockerfile) + .get(); + } +} diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainerTest.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainerTest.java new file mode 100644 index 000000000000..ef9a9ef97a14 --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainerTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.testcontainers.containers.Container.ExecResult; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Smoke test for TezYarnClusterContainer. Verifies that a real HDFS+YARN + * cluster comes up cleanly via Testcontainers and that the wrapper's helper + * methods return usable addresses and can perform HDFS I/O. + * + * Run with: mvn -Pitests,tez-yarn -pl itests/tez-yarn-it -Dtest=TezYarnClusterContainerTest test + */ +public class TezYarnClusterContainerTest { + + private static TezYarnClusterContainer cluster; + + @BeforeClass + public static void startCluster() { + cluster = new TezYarnClusterContainer(); + cluster.start(); + } + + @AfterClass + public static void stopCluster() { + if (cluster != null) { + cluster.stop(); + } + } + + @Test + public void testHdfsUriFormat() { + String uri = cluster.getHdfsUri(); + Assert.assertNotNull("HDFS URI must not be null", uri); + Assert.assertTrue("Expected hdfs:// URI, got: " + uri, uri.startsWith("hdfs://")); + } + + @Test + public void testResourceManagerAddressFormat() { + String addr = cluster.getResourceManagerAddress(); + Assert.assertNotNull("RM address must not be null", addr); + Assert.assertTrue("Expected host:port, got: " + addr, addr.contains(":")); + } + + @Test + public void testHdfsWriteAndRead() throws IOException, InterruptedException { + ExecResult mkdir = cluster.namenodeContainer() + .execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/smoke-test"); + Assert.assertEquals("hdfs mkdir failed:\n" + mkdir.getStderr(), 0, mkdir.getExitCode()); + + ExecResult ls = cluster.namenodeContainer() + .execInContainer("hdfs", "dfs", "-ls", "/tmp"); + Assert.assertTrue("Expected /tmp/smoke-test in HDFS listing:\n" + ls.getStdout(), + ls.getStdout().contains("smoke-test")); + } + + @Test + public void testYarnNodeManagerRegistered() throws IOException, InterruptedException { + ExecResult result = cluster.resourceManagerContainer() + .execInContainer("yarn", "node", "-list"); + String out = result.getStdout(); + Assert.assertTrue("Expected 'Total Nodes:' in yarn node -list output:\n" + out, + out.contains("Total Nodes:")); + Assert.assertFalse("Expected at least one active NodeManager:\n" + out, + out.contains("Total Nodes:0")); + } + + @Test + public void testUploadJarToHdfs() throws IOException, InterruptedException { + Path tempJar = Files.createTempFile("smoke-upload", ".jar"); + try { + Files.write(tempJar, "placeholder-jar-content".getBytes()); + + String hdfsPath = cluster.uploadJarToHdfs(tempJar); + Assert.assertNotNull("uploadJarToHdfs must return a non-null HDFS path", hdfsPath); + + ExecResult ls = cluster.namenodeContainer() + .execInContainer("hdfs", "dfs", "-ls", "/tmp/hive-29483-jars"); + Assert.assertTrue("Uploaded jar not visible in HDFS:\n" + ls.getStdout(), + ls.getStdout().contains(tempJar.getFileName().toString())); + } finally { + Files.deleteIfExists(tempJar); + } + } +} diff --git a/itests/tez-yarn-it/src/test/resources/custom_hosts_file b/itests/tez-yarn-it/src/test/resources/custom_hosts_file new file mode 100644 index 000000000000..642911816ad4 --- /dev/null +++ b/itests/tez-yarn-it/src/test/resources/custom_hosts_file @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Hostname overrides for Tez-on-YARN integration tests. +# +# When TezYarnClusterContainer is started in fixed-port mode, these four +# containers are reachable from the host at 127.0.0.1 on their well-known +# ports (NameNode 8020/9870, ResourceManager 8032/8088). The same hostnames +# are used inside the Docker network, so a single HDFS URI like +# hdfs://namenode:8020 works both from the test JVM and from YARN containers. +# +# This file is passed to the test JVM via +# -Djdk.net.hosts.file=/custom_hosts_file +# which makes the JVM's DNS resolver consult it before the OS resolver. + +127.0.0.1 localhost +127.0.0.1 namenode +127.0.0.1 resourcemanager +127.0.0.1 datanode +127.0.0.1 nodemanager diff --git a/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml b/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml new file mode 100644 index 000000000000..e76f403d5065 --- /dev/null +++ b/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml @@ -0,0 +1,63 @@ + + + + + + + hive.in.test + true + + + + hive.execution.engine + tez + + + + hive.server2.enable.doAs + false + + + + hive.support.concurrency + false + + + + + hive.query.history.enabled + false + + + + datanucleus.schema.autoCreateAll + true + + + + hive.metastore.schema.verification + false + + + + datanucleus.connectionPool.maxPoolSize + 4 + + + diff --git a/itests/tez-yarn-it/src/test/resources/yarn-site.xml b/itests/tez-yarn-it/src/test/resources/yarn-site.xml new file mode 100644 index 000000000000..fda5ad066fa8 --- /dev/null +++ b/itests/tez-yarn-it/src/test/resources/yarn-site.xml @@ -0,0 +1,33 @@ + + + + + + + + yarn.nodemanager.pmem-check-enabled + false + + + + yarn.nodemanager.vmem-check-enabled + false + + + From 303eefdb0432cbeb882abc52417ce9300a088367 Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Fri, 24 Jul 2026 22:45:59 +0530 Subject: [PATCH 02/18] Add GitHub Actions workflow for the Tez-on-YARN localization IT --- .github/workflows/tez-yarn-it.yml | 66 +++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/tez-yarn-it.yml diff --git a/.github/workflows/tez-yarn-it.yml b/.github/workflows/tez-yarn-it.yml new file mode 100644 index 000000000000..10506d3af223 --- /dev/null +++ b/.github/workflows/tez-yarn-it.yml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Workflow to validate the Tez-on-YARN resource localization integration tests +# (HIVE-29483). These tests require Docker and a real YARN cluster, so they are +# not part of the standard split-based CI. Runs automatically on push to +# tez_on_yarn_it (fork testing) or manually via workflow_dispatch. +# +# Mirrors the local run: +# mvn clean install -DskipTests -Pitests (install artifacts, incl. hive-exec.jar, to ~/.m2) +# mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar +name: Tez-on-YARN localization IT + +on: + push: + branches: + - tez_on_yarn_it + workflow_dispatch: + +jobs: + tez-yarn-it: + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Java 21 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + cache: maven + + # Step 1 (mirrors local `mvn clean install -DskipTests -Pitests`): build and + # install artifacts to ~/.m2 so a real hive-exec.jar exists for Tez to + # localize. Scoped with -pl/-am so it finishes within the CI budget while + # still installing hive-exec and every upstream dependency. + - name: Build and install Hive to local Maven repo + run: mvn clean install -DskipTests -Pitests -pl itests/tez-yarn-it -am -Dmaven.javadoc.skip=true + + # Step 2 (mirrors local test command): resolves hive-exec from ~/.m2. + # -Dsurefire.useFile=false streams the test's NodeManager/syslog diagnostics + # into this log so any Tez AM failure is visible directly here. + - name: Run Tez-on-YARN localization test + run: mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar -Dsurefire.useFile=false + + - name: Upload surefire reports on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: surefire-reports + path: itests/tez-yarn-it/target/surefire-reports/ From 52a24d6b5da048f3314bc13a89281b2996960980 Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Fri, 24 Jul 2026 23:02:17 +0530 Subject: [PATCH 03/18] Point NM diagnostics at /var/log/hadoop/userlogs where container syslog lives --- .../tez/yarn/TestTezYarnLocalization.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index a128d1192230..0c1591167192 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -185,19 +185,22 @@ private static void dumpNodeManagerDiagnostics() { "find /tmp -name 'launch_container.sh' 2>/dev/null | head -3 " + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); - // syslog = Tez AM log4j output (NoClassDefFoundError etc. land here, not in stderr). + // Container logs live under LOG_DIRS=/var/log/hadoop/userlogs/// (per + // launch_container.sh), NOT /tmp. syslog is the Tez AM's log4j output -- the CLA appender + // routes DAGAppMaster startup errors (NoClassDefFoundError, NoSuchMethodError, etc.) there, + // which is why the YARN diagnostic shows an empty stderr while the AM still exits 1. dumpNmCommand("container syslog (Tez AM log4j output)", - "find /tmp -name 'syslog' -path '*/container_*/*' 2>/dev/null | head -5 " + "find /var/log/hadoop/userlogs -name 'syslog*' 2>/dev/null | head -10 " + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); - // stderr / prelaunch.err (usually empty, but confirm). - dumpNmCommand("container stderr + prelaunch.err", - "find /tmp \\( -name 'stderr' -o -name 'prelaunch.err' \\) -path '*/container_*/*' 2>/dev/null | head -10 " - + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + // stdout / stderr / prelaunch.err from the same LOG_DIRS location. + dumpNmCommand("container stdout + stderr + prelaunch.err", + "find /var/log/hadoop/userlogs \\( -name 'stdout' -o -name 'stderr' -o -name 'prelaunch.err' \\) " + + "2>/dev/null | head -20 | xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); - // NodeManager daemon log: pre-exec launch-script failures that bypass container stderr. + // NodeManager daemon log (yarn.log.dir=/var/log/hadoop): catches launch-script failures. dumpNmCommand("NodeManager daemon log (tail 200)", - "find /opt/hadoop/logs -name '*.log' 2>/dev/null | head -2 " + "find /var/log/hadoop -maxdepth 1 -name '*.log' 2>/dev/null | head -3 " + "| xargs -I{} sh -c 'echo \"--- {} ---\"; tail -200 {}' 2>/dev/null || true"); } catch (Exception e) { System.out.println("Could not dump NodeManager diagnostics: " + e); From 9c4b3b9696b84debcc2dbe65670aa730c7b613dc Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Fri, 24 Jul 2026 23:15:59 +0530 Subject: [PATCH 04/18] Stage Tez hadoop-shim jar into tez.lib.uris to fix AM startup --- itests/tez-yarn-it/pom.xml | 9 +++++ .../tez/yarn/TezYarnClusterContainer.java | 36 +++++++++++++------ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/itests/tez-yarn-it/pom.xml b/itests/tez-yarn-it/pom.xml index 88f852b2d5b0..28741b85c665 100644 --- a/itests/tez-yarn-it/pom.xml +++ b/itests/tez-yarn-it/pom.xml @@ -65,6 +65,15 @@ tez-mapreduce test + + + org.apache.tez + hadoop-shim + ${tez.version} + test + org.apache.hive hive-service diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java index 0f46b4dcf960..86bcd9353fd3 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java @@ -321,16 +321,18 @@ public String uploadTezLibsToHdfs() throws IOException, InterruptedException { } /** - * Discovers Tez jars from the running JVM's classpath using two complementary strategies: + * Discovers Tez framework jars from the running JVM's classpath using two complementary strategies: *
    *
  1. Reflection on known Tez probe classes — works regardless of how Surefire * passes the classpath (direct {@code -cp} or manifest-only argfile JAR).
    2. *
  2. Scanning {@link ManagementFactory#getRuntimeMXBean() RuntimeMXBean#getClassPath()} - * for entries whose filename contains {@code "tez"} — a safety net when the - * reflection approach misses a jar (e.g. classes not loaded yet).
    3. + * — a safety net when the reflection approach misses a jar (e.g. classes not loaded yet). *
- * Only regular {@code .jar} files whose filename contains {@code "tez"} and does - * not end with {@code "-tests.jar"} are returned. + * Returns regular {@code .jar} files that belong to the Tez framework (see + * {@link #isTezFrameworkJar(String)}); this includes {@code hadoop-shim-*.jar}, which does not + * contain "tez" in its filename but provides {@code org.apache.tez.hadoop.shim.HadoopShimsLoader} + * required by {@code DAGAppMaster.serviceInit()} — omitting it makes the Tez AM fail at startup + * with {@code NoClassDefFoundError: org/apache/tez/hadoop/shim/HadoopShimsLoader}. */ private static Set findTezJarsFromClasspath() { Set jars = new LinkedHashSet<>(); @@ -343,7 +345,8 @@ private static Set findTezJarsFromClasspath() { "org.apache.tez.dag.app.DAGAppMaster", // tez-dag (the Tez AM) "org.apache.tez.mapreduce.hadoop.MRHelpers", // tez-mapreduce "org.apache.tez.runtime.LogicalIOProcessorRuntimeTask", // tez-runtime-internals - "org.apache.tez.runtime.library.api.KeyValueReader" // tez-runtime-library + "org.apache.tez.runtime.library.api.KeyValueReader", // tez-runtime-library + "org.apache.tez.hadoop.shim.HadoopShimsLoader" // hadoop-shim (loaded by DAGAppMaster) }; for (String className : probeClassNames) { try { @@ -353,8 +356,7 @@ private static Set findTezJarsFromClasspath() { String path = cs.getLocation().getPath(); if (path.endsWith(".jar")) { Path p = Paths.get(path); - String name = p.getFileName().toString(); - if (name.contains("tez") && !name.endsWith("-tests.jar") && Files.isRegularFile(p)) { + if (isTezFrameworkJar(p.getFileName().toString()) && Files.isRegularFile(p)) { jars.add(p); } } @@ -364,15 +366,14 @@ private static Set findTezJarsFromClasspath() { } } - // Strategy 2: scan the JVM classpath string for jars with "tez" in the name. + // Strategy 2: scan the JVM classpath string for Tez framework jars. String cp = ManagementFactory.getRuntimeMXBean().getClassPath(); for (String entry : cp.split(File.pathSeparator)) { if (!entry.endsWith(".jar")) { continue; } Path p = Paths.get(entry); - String name = p.getFileName().toString(); - if (name.contains("tez") && !name.endsWith("-tests.jar") && Files.isRegularFile(p)) { + if (isTezFrameworkJar(p.getFileName().toString()) && Files.isRegularFile(p)) { jars.add(p); } } @@ -380,6 +381,19 @@ private static Set findTezJarsFromClasspath() { return jars; } + /** + * Whether a jar filename belongs to the Tez framework and should be staged into + * {@code tez.lib.uris}. Matches {@code tez-*} jars plus {@code hadoop-shim-*} (which carries + * {@code org.apache.tez.hadoop.shim.HadoopShimsLoader} but has no "tez" in its name), excluding + * test jars. + */ + private static boolean isTezFrameworkJar(String name) { + if (name.endsWith("-tests.jar")) { + return false; + } + return name.contains("tez") || name.startsWith("hadoop-shim"); + } + // Package-private: only test classes in this package need direct exec access. GenericContainer namenodeContainer() { return namenode; From 78bb2b36f2af2c6a88276f1e03c13cb4467c694c Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Fri, 24 Jul 2026 23:33:47 +0530 Subject: [PATCH 05/18] Make Tez AM reachable from host: fixed hostname + pinned/published AM RPC port --- .../tez/yarn/TestTezYarnLocalization.java | 8 ++++++ .../tez/yarn/TezYarnClusterContainer.java | 28 +++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index 0c1591167192..4ed3c92d1619 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -258,6 +258,14 @@ private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) thro conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS, false); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 0); + // Pin the Tez AM's client-facing RPC port so the host JVM can reach the AM. The AM runs in the + // NodeManager container (hostname "nodemanager", resolvable to 127.0.0.1 via custom_hosts_file) + // and publishes AM_CLIENT_PORT; without a fixed port the AM would advertise a random port that + // is not published, so the client fails with "UnknownHostException/connection refused" and the + // Tez session times out. + conf.set("tez.am.client.am.port-range", + TezYarnClusterContainer.AM_CLIENT_PORT + "-" + TezYarnClusterContainer.AM_CLIENT_PORT); + // The Hadoop image daemons use Java 8 but hive-exec requires Java 21; point Tez AM/task // container environments at the Java 21 runtime added by the custom Dockerfile. // HADOOP_HOME must be set explicitly: the apache/hadoop image does not export it into YARN diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java index 86bcd9353fd3..b8ac1b0fbbed 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java @@ -96,6 +96,11 @@ public class TezYarnClusterContainer { // DataNode ports (Hadoop 3.x defaults). Host JVM must reach 9866 for HDFS writes. private static final int DN_HTTP_PORT = 9864; private static final int DN_XFER_PORT = 9866; + // Fixed port for the Tez AM's client-facing DAGClientRPCServer. The AM runs inside the + // NodeManager container and advertises its address to the host JVM (HiveServer2). Pinning the + // port (via tez.am.client.am.port-range) and publishing it, together with the NM hostname set + // to "nodemanager" (resolvable to 127.0.0.1 via custom_hosts_file), lets the host reach the AM. + public static final int AM_CLIENT_PORT = 41000; private final Network network; private final GenericContainer namenode; @@ -170,11 +175,24 @@ public TezYarnClusterContainer(boolean fixedPorts) { .withEnv(COMMON_ENV); } - nodemanager = new GenericContainer<>(HADOOP_IMAGE) - .withNetwork(network) - .withNetworkAliases("nodemanager") - .withCommand("yarn", "nodemanager") - .withEnv(COMMON_ENV); + if (fixedPorts) { + // Fix the container hostname to "nodemanager" so the Tez AM (which runs here) advertises + // its DAGClientRPCServer as nodemanager: instead of the random container id, + // and publish that port so the host JVM (HiveServer2 / Tez client) can reach the AM. + nodemanager = new FixedHostPortGenericContainer<>(HADOOP_IMAGE) + .withFixedExposedPort(AM_CLIENT_PORT, AM_CLIENT_PORT) + .withCreateContainerCmdModifier(cmd -> cmd.withHostName("nodemanager")) + .withNetwork(network) + .withNetworkAliases("nodemanager") + .withCommand("yarn", "nodemanager") + .withEnv(COMMON_ENV); + } else { + nodemanager = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("nodemanager") + .withCommand("yarn", "nodemanager") + .withEnv(COMMON_ENV); + } } /** From d9a5e24cdca5dca4ae7fd840f0cb7d953c7b8d2c Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Fri, 24 Jul 2026 23:52:06 +0530 Subject: [PATCH 06/18] mapreduce conf --- .../tez/yarn/TezYarnClusterContainer.java | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java index b8ac1b0fbbed..c6946969eb3e 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java @@ -332,8 +332,9 @@ public String uploadTezLibsToHdfs() throws IOException, InterruptedException { if (hdfsUris.isEmpty()) { throw new IllegalStateException( - "No Tez jars (filenames containing 'tez') were found on the test classpath. " - + "Ensure tez-api, tez-dag, tez-runtime-library, etc. are transitive dependencies."); + "No framework jars were found on the test classpath to stage. " + + "Ensure tez-api, tez-dag, tez-runtime-library, hadoop-mapreduce-client-core, " + + "etc. are test dependencies."); } return String.join(",", hdfsUris); } @@ -341,30 +342,38 @@ public String uploadTezLibsToHdfs() throws IOException, InterruptedException { /** * Discovers Tez framework jars from the running JVM's classpath using two complementary strategies: *
    - *
  1. Reflection on known Tez probe classes — works regardless of how Surefire - * passes the classpath (direct {@code -cp} or manifest-only argfile JAR).
    2. + *
  2. Reflection on known probe classes — works regardless of how Surefire passes the classpath + * (direct {@code -cp} or manifest-only argfile JAR).
    3. *
  3. Scanning {@link ManagementFactory#getRuntimeMXBean() RuntimeMXBean#getClassPath()} * — a safety net when the reflection approach misses a jar (e.g. classes not loaded yet).
    4. *
- * Returns regular {@code .jar} files that belong to the Tez framework (see - * {@link #isTezFrameworkJar(String)}); this includes {@code hadoop-shim-*.jar}, which does not - * contain "tez" in its filename but provides {@code org.apache.tez.hadoop.shim.HadoopShimsLoader} - * required by {@code DAGAppMaster.serviceInit()} — omitting it makes the Tez AM fail at startup - * with {@code NoClassDefFoundError: org/apache/tez/hadoop/shim/HadoopShimsLoader}. + * Returns regular {@code .jar} files that should be staged into {@code tez.lib.uris} (see + * {@link #isTezFrameworkJar(String)}). This includes: + *
    + *
  • {@code hadoop-shim-*.jar}: carries {@code HadoopShimsLoader} but has no "tez" in its name; + * omitting it causes {@code NoClassDefFoundError: org/apache/tez/hadoop/shim/HadoopShimsLoader} + * in {@code DAGAppMaster.serviceInit()}.
    • + *
  • {@code hadoop-mapreduce-client-core-*.jar}: provides {@code org.apache.hadoop.mapred.JobConf} + * needed by {@code MRInputAMSplitGenerator}; not present in the cluster's + * {@code yarn.application.classpath} and not included in {@code tez.use.cluster.hadoop-libs} + * expansion for {@code apache/hadoop:3.4.2}, so it must be staged explicitly.
    • + *
*/ private static Set findTezJarsFromClasspath() { Set jars = new LinkedHashSet<>(); - // Strategy 1: reflect on known Tez probe classes to get their source jar. + // Strategy 1: reflect on known probe classes to get their source jar. // Use Class.forName to avoid compile-time imports of Tez internal modules. String[] probeClassNames = { - "org.apache.tez.dag.api.TezConfiguration", // tez-api - "org.apache.tez.common.TezConverterUtils", // tez-common - "org.apache.tez.dag.app.DAGAppMaster", // tez-dag (the Tez AM) - "org.apache.tez.mapreduce.hadoop.MRHelpers", // tez-mapreduce + "org.apache.tez.dag.api.TezConfiguration", // tez-api + "org.apache.tez.common.TezConverterUtils", // tez-common + "org.apache.tez.dag.app.DAGAppMaster", // tez-dag (the Tez AM) + "org.apache.tez.mapreduce.hadoop.MRHelpers", // tez-mapreduce "org.apache.tez.runtime.LogicalIOProcessorRuntimeTask", // tez-runtime-internals - "org.apache.tez.runtime.library.api.KeyValueReader", // tez-runtime-library - "org.apache.tez.hadoop.shim.HadoopShimsLoader" // hadoop-shim (loaded by DAGAppMaster) + "org.apache.tez.runtime.library.api.KeyValueReader", // tez-runtime-library + "org.apache.tez.hadoop.shim.HadoopShimsLoader", // hadoop-shim (loaded by DAGAppMaster) + "org.apache.hadoop.mapred.JobConf", // hadoop-mapreduce-client-core (MRInputAMSplitGenerator) + "org.apache.hadoop.mapreduce.v2.util.MRApps" // hadoop-mapreduce-client-common (transitive from above) }; for (String className : probeClassNames) { try { @@ -400,16 +409,24 @@ private static Set findTezJarsFromClasspath() { } /** - * Whether a jar filename belongs to the Tez framework and should be staged into - * {@code tez.lib.uris}. Matches {@code tez-*} jars plus {@code hadoop-shim-*} (which carries - * {@code org.apache.tez.hadoop.shim.HadoopShimsLoader} but has no "tez" in its name), excluding - * test jars. + * Whether a jar should be staged into {@code tez.lib.uris}. Matches: + *
    + *
  • {@code tez-*} — core Tez framework jars
    • + *
  • {@code hadoop-shim-*} — carries {@code HadoopShimsLoader}; filename has no "tez"
    • + *
  • {@code hadoop-mapreduce-client-core-*} and {@code hadoop-mapreduce-client-common-*} — + * provide {@code JobConf} and related classes required by {@code MRInputAMSplitGenerator}; + * absent from the {@code apache/hadoop:3.4.2} {@code yarn.application.classpath} expansion
    • + *
+ * Test jars are always excluded. */ private static boolean isTezFrameworkJar(String name) { if (name.endsWith("-tests.jar")) { return false; } - return name.contains("tez") || name.startsWith("hadoop-shim"); + return name.contains("tez") + || name.startsWith("hadoop-shim") + || name.startsWith("hadoop-mapreduce-client-core") + || name.startsWith("hadoop-mapreduce-client-common"); } // Package-private: only test classes in this package need direct exec access. From df0ca1bef535fa9155cb769de9997443c006c84a Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Sat, 25 Jul 2026 00:11:02 +0530 Subject: [PATCH 07/18] change query --- .../tez/yarn/TestTezYarnLocalization.java | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index 4ed3c92d1619..c77d4ff110e0 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -124,12 +124,16 @@ public static void stopAll() { * *

Runs a Tez DAG via JDBC and asserts that: *

    - *
  1. The query returns the expected result (primary proof that - * {@code hive-exec.jar} was localized and Hive executor classes - * were available inside YARN task containers).
    2. - *
  2. A Tez YARN application is visible in the ResourceManager - * (secondary confirmation that the DAG actually ran on YARN).
    3. + *
  3. The INSERT … SELECT completes and returns the expected row count (primary proof that + * {@code hive-exec.jar} was localized and Hive executor classes were available inside + * YARN task containers).
    4. + *
  4. A Tez YARN application is visible in the ResourceManager (secondary confirmation that + * the DAG actually ran on YARN).
    5. *
+ * + *

Uses {@code INSERT … SELECT} from an HDFS-backed table rather than {@code VALUES}, because + * {@code VALUES} inserts compile to a local {@code file:/…/dummy_path} input that exists only on + * the host JVM filesystem; Tez map tasks run inside Docker containers and cannot read it. */ @Test public void testQuerySucceedsWithAppJar() throws Exception { @@ -138,17 +142,20 @@ public void testQuerySucceedsWithAppJar() throws Exception { try (Statement stmt = conn.createStatement()) { stmt.execute("CREATE TABLE IF NOT EXISTS tez_loc_test (id INT) STORED AS ORC"); - stmt.execute("INSERT INTO tez_loc_test VALUES (1), (2), (3)"); + stmt.execute("CREATE TABLE IF NOT EXISTS tez_source (id INT) STORED AS ORC"); + + // SELECT from an HDFS table (even empty) keeps map inputs on hdfs://namenode:8020/…, + // reachable from YARN containers. COUNT(*) forces a reduce vertex; if hive-exec.jar were + // missing from commonLocalResources the task container would fail before this returns. + stmt.execute("INSERT INTO tez_loc_test SELECT count(*) FROM tez_source"); - // COUNT(*) forces a Tez reduce task. If hive-exec.jar is not localized the task - // container fails with ClassNotFoundException before this assertion is reached. - try (ResultSet rs = stmt.executeQuery("SELECT count(*) FROM tez_loc_test")) { + try (ResultSet rs = stmt.executeQuery("SELECT id FROM tez_loc_test")) { Assert.assertTrue("Result set must contain at least one row", rs.next()); long count = rs.getLong(1); Assert.assertEquals( - "SELECT count(*) FROM tez_loc_test should return 3 (hive-exec.jar was localized)", - 3L, count); - LOG.info("Tez query succeeded: count(*) = {}", count); + "INSERT SELECT count(*) FROM empty tez_source should return 0 (hive-exec.jar was localized)", + 0L, count); + LOG.info("Tez query succeeded: inserted count(*) = {}", count); } } } From d6dd1db556660e8c125d48a5d5330099a087602b Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Sat, 25 Jul 2026 00:32:08 +0530 Subject: [PATCH 08/18] file permission denied fix --- .../java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java | 1 + 1 file changed, 1 insertion(+) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index c77d4ff110e0..47fcd751589a 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -86,6 +86,7 @@ public static void startAll() throws Exception { cluster.start(); GenericContainer nn = cluster.namenodeContainer(); + nn.execInContainer("hdfs", "dfs", "-chmod", "777", "/tmp"); nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/warehouse"); nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/scratch"); nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/user-install"); From e5d5f3adcbb56f33450941fef17927cc1f43ff47 Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Sat, 25 Jul 2026 14:13:10 +0530 Subject: [PATCH 09/18] permissions fix --- .../org/apache/hive/tez/yarn/TestTezYarnLocalization.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index 47fcd751589a..2ed455feef14 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -86,7 +86,10 @@ public static void startAll() throws Exception { cluster.start(); GenericContainer nn = cluster.namenodeContainer(); - nn.execInContainer("hdfs", "dfs", "-chmod", "777", "/tmp"); + // 1. Explicitly create and open up the global /tmp directory first + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp"); + nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp"); + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/warehouse"); nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/scratch"); nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/user-install"); From b831594d0f4b2dafa51ad00f5ad73f04c6057add Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Sat, 25 Jul 2026 14:29:28 +0530 Subject: [PATCH 10/18] reduce phase fix --- .../apache/hive/tez/yarn/TestTezYarnLocalization.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index 2ed455feef14..d4b3e514cc1d 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -282,14 +282,18 @@ private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) thro // HADOOP_HOME must be set explicitly: the apache/hadoop image does not export it into YARN // container environments, so tez.use.cluster.hadoop-libs's "${HADOOP_HOME}/bin/hadoop classpath" // call would silently produce an empty classpath without it. + // + // IMPORTANT: set the container env ONLY via the Tez properties. Hive's DagUtils also injects + // mapreduce.map.env / mapreduce.reduce.env into each task vertex (via MRHelpers), and Tez then + // appends tez.task.launch.env on top. Setting the same JAVA_HOME in both sources makes YARN + // concatenate them with the path separator, yielding JAVA_HOME=/opt/jdk21:/opt/jdk21 -- so the + // task container runs "/opt/jdk21:/opt/jdk21/bin/java" and dies with exit 127 (No such file). + // The AM is unaffected because it only reads tez.am.launch.env. String containerEnv = "JAVA_HOME=" + TezYarnClusterContainer.CONTAINER_JAVA_21_HOME + ",HADOOP_HOME=/opt/hadoop" + ",HADOOP_MAPRED_HOME=/opt/hadoop"; conf.set("tez.am.launch.env", containerEnv); conf.set("tez.task.launch.env", containerEnv); - conf.set("yarn.app.mapreduce.am.env", containerEnv); - conf.set("mapreduce.map.env", containerEnv); - conf.set("mapreduce.reduce.env", containerEnv); hs2Port = findFreePort(); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); From 036b8c1e248d336444b370a401da7331835d918d Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Sat, 25 Jul 2026 15:31:33 +0530 Subject: [PATCH 11/18] final fix --- itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile | 2 +- .../org/apache/hive/tez/yarn/TestTezYarnLocalization.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile index ac2f824f9f31..2123acbb57a4 100644 --- a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile +++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile @@ -21,7 +21,7 @@ # hive-exec classes -- must run on Java 21. This image adds a Java 21 JDK at # /opt/jdk21 using a multi-stage build from eclipse-temurin:21-jdk; the daemons # continue to use the default Java 8. -FROM eclipse-temurin:21-jdk AS jdk21 +FROM --platform=linux/amd64 eclipse-temurin:21-jdk AS jdk21 FROM apache/hadoop:3.4.2 USER root diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index d4b3e514cc1d..323e9d8bba36 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -98,7 +98,9 @@ public static void startAll() throws Exception { String tezLibUris = cluster.uploadTezLibsToHdfs(); LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); - Path localScratch = Files.createTempDirectory("hive-tez-loc-"); + // Force /tmp prefix so the path is valid inside both macOS host and Linux Docker containers + Path localScratch = Files.createDirectories( + java.nio.file.Paths.get("/tmp", "hive-tez-loc-" + System.currentTimeMillis())); HiveConf conf = buildHiveConf(tezLibUris, localScratch); hs2 = new HiveServer2(); From 8d7407e25f5a4817df39db9f745385236e4fe803 Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Sat, 25 Jul 2026 16:22:28 +0530 Subject: [PATCH 12/18] set METASTORE_TRY_DIRECT_SQL to false --- .../org/apache/hive/tez/yarn/TestTezYarnLocalization.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index 323e9d8bba36..305a0e771dbd 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -259,6 +259,11 @@ private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) thro conf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); +// Disable Direct SQL in embedded Derby to prevent JDO transaction corruption + // when querying uninitialized schema tables during race conditions. + conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, false); + conf.set("hive.stats.autogather", "false"); + conf.set("hive.stats.column.autogather", "false"); conf.set("yarn.resourcemanager.hostname", "resourcemanager"); conf.set("yarn.resourcemanager.address", "resourcemanager:8032"); conf.set("yarn.resourcemanager.webapp.address", "resourcemanager:8088"); From 73dc2fd56faa581c9e81c2cca820c14e8cbcb156 Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Mon, 27 Jul 2026 11:58:28 +0530 Subject: [PATCH 13/18] cleanup task --- .../docker/hadoop-yarn/docker-compose.yml | 71 ---------- .../tez/yarn/TestHiveServer2Connectivity.java | 131 ++++++++++++++++++ .../tez/yarn/TestTezYarnLocalizationBase.java | 8 +- 3 files changed, 135 insertions(+), 75 deletions(-) delete mode 100644 itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml create mode 100644 itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml deleted file mode 100644 index 72775ddfa1fe..000000000000 --- a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/docker-compose.yml +++ /dev/null @@ -1,71 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "2" -services: - namenode: - image: apache/hadoop:3.4.2 - hostname: namenode - command: ["hdfs", "namenode"] - env_file: - - ./config - environment: - ENSURE_NAMENODE_DIR: "/tmp/hadoop-hadoop/dfs/name" - ports: - - "9870:9870" - - "8020:8020" - networks: - - hive-yarn-it - - datanode: - image: apache/hadoop:3.4.2 - command: ["hdfs", "datanode"] - env_file: - - ./config - depends_on: - - namenode - networks: - - hive-yarn-it - - resourcemanager: - image: apache/hadoop:3.4.2 - hostname: resourcemanager - command: ["yarn", "resourcemanager"] - env_file: - - ./config - ports: - - "8088:8088" - - "8032:8032" - depends_on: - - namenode - networks: - - hive-yarn-it - - nodemanager: - image: apache/hadoop:3.4.2 - command: ["yarn", "nodemanager"] - env_file: - - ./config - ports: - - "8042:8042" - depends_on: - - resourcemanager - networks: - - hive-yarn-it - -networks: - hive-yarn-it: - name: hive-yarn-it diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java new file mode 100644 index 000000000000..1308ea83077f --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; + +public class TestHiveServer2Connectivity { + + protected static TezYarnClusterContainer cluster; + protected static HiveServer2 hs2; + protected static int hs2Port; + protected static String hdfsUri; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(); + cluster.start(); + + hdfsUri = cluster.getHdfsUri(); + + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/hive-29483/warehouse"); + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/hive-29483/scratch"); + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp/hive-29483"); + + Path localScratch = Files.createTempDirectory("hive-29483-local-"); + + HiveConf conf = new HiveConf(); + URL hiveSite = TestHiveServer2Connectivity.class.getClassLoader().getResource("hive-site-yarn-it.xml"); + URL yarnSite = TestHiveServer2Connectivity.class.getClassLoader().getResource("yarn-site.xml"); + if (hiveSite != null) conf.addResource(hiveSite); + if (yarnSite != null) conf.addResource(yarnSite); + + conf.set("fs.defaultFS", hdfsUri); + conf.set("hive.metastore.warehouse.dir", hdfsUri + "/tmp/hive-29483/warehouse"); + conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, hdfsUri + "/tmp/hive-29483/scratch"); + conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname, localScratch.toAbsolutePath().toString()); + + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); + + conf.set("yarn.resourcemanager.address", cluster.getResourceManagerAddress()); + conf.set("yarn.resourcemanager.webapp.address", cluster.getResourceManagerWebAppAddress()); + + hs2Port = findFreePort(); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, "localhost"); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort()); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, "NOSASL"); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + waitForJdbc(hs2Port); + } + + @AfterClass + public static void stopAll() { + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + } + + @Test + public void testJdbcSessionOpen() throws Exception { + String url = jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + Assert.assertNotNull("JDBC connection must not be null", conn); + } + } + + private static void waitForJdbc(int port) throws InterruptedException { + String url = jdbcUrl(port); + long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + try (Connection c = DriverManager.getConnection(url, "hive", "")) { + return; + } catch (Exception ignored) { + Thread.sleep(2000); + } + } + throw new IllegalStateException("HiveServer2 JDBC endpoint not reachable on port " + port + " after 60s"); + } + + private static String jdbcUrl(int port) { + // HS2 is configured with HIVE_SERVER2_AUTHENTICATION=NOSASL, which requires the client to + // set auth=noSasl explicitly. + return "jdbc:hive2://localhost:" + port + "/default;auth=noSasl"; + } + + private static int findFreePort() throws Exception { + try (ServerSocket s = new ServerSocket(0)) { + s.setReuseAddress(true); + return s.getLocalPort(); + } + } +} diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java index ff6292172dc4..6daf05a7e680 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java @@ -33,10 +33,10 @@ public class TestTezYarnLocalizationBase { - protected static TezYarnClusterContainer cluster; - protected static HiveServer2 hs2; - protected static int hs2Port; - protected static String hdfsUri; + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + private static String hdfsUri; @BeforeClass public static void startAll() throws Exception { From df964643e4c32144212c8fd06a65b121560952af Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Mon, 27 Jul 2026 12:09:46 +0530 Subject: [PATCH 14/18] Drop duplicate TestTezYarnLocalizationBase in favor of TestHiveServer2Connectivity --- .../tez/yarn/TestHiveServer2Connectivity.java | 8 +- .../tez/yarn/TestTezYarnLocalizationBase.java | 131 ------------------ 2 files changed, 4 insertions(+), 135 deletions(-) delete mode 100644 itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java index 1308ea83077f..10fc90f66c36 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java @@ -33,10 +33,10 @@ public class TestHiveServer2Connectivity { - protected static TezYarnClusterContainer cluster; - protected static HiveServer2 hs2; - protected static int hs2Port; - protected static String hdfsUri; + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + private static String hdfsUri; @BeforeClass public static void startAll() throws Exception { diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java deleted file mode 100644 index 6daf05a7e680..000000000000 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalizationBase.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hive.tez.yarn; - -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hive.service.server.HiveServer2; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.net.ServerSocket; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.DriverManager; - -public class TestTezYarnLocalizationBase { - - private static TezYarnClusterContainer cluster; - private static HiveServer2 hs2; - private static int hs2Port; - private static String hdfsUri; - - @BeforeClass - public static void startAll() throws Exception { - cluster = new TezYarnClusterContainer(); - cluster.start(); - - hdfsUri = cluster.getHdfsUri(); - - cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/hive-29483/warehouse"); - cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/hive-29483/scratch"); - cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp/hive-29483"); - - Path localScratch = Files.createTempDirectory("hive-29483-local-"); - - HiveConf conf = new HiveConf(); - URL hiveSite = TestTezYarnLocalizationBase.class.getClassLoader().getResource("hive-site-yarn-it.xml"); - URL yarnSite = TestTezYarnLocalizationBase.class.getClassLoader().getResource("yarn-site.xml"); - if (hiveSite != null) conf.addResource(hiveSite); - if (yarnSite != null) conf.addResource(yarnSite); - - conf.set("fs.defaultFS", hdfsUri); - conf.set("hive.metastore.warehouse.dir", hdfsUri + "/tmp/hive-29483/warehouse"); - conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, hdfsUri + "/tmp/hive-29483/scratch"); - conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname, localScratch.toAbsolutePath().toString()); - - conf.set("javax.jdo.option.ConnectionURL", - "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); - - conf.set("yarn.resourcemanager.address", cluster.getResourceManagerAddress()); - conf.set("yarn.resourcemanager.webapp.address", cluster.getResourceManagerWebAppAddress()); - - hs2Port = findFreePort(); - conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); - conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, "localhost"); - conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort()); - conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); - conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, "NOSASL"); - conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); - - hs2 = new HiveServer2(); - hs2.init(conf); - hs2.start(); - - waitForJdbc(hs2Port); - } - - @AfterClass - public static void stopAll() { - if (hs2 != null) { - hs2.stop(); - hs2 = null; - } - if (cluster != null) { - cluster.stop(); - cluster = null; - } - } - - @Test - public void testJdbcSessionOpen() throws Exception { - String url = jdbcUrl(hs2Port); - try (Connection conn = DriverManager.getConnection(url, "hive", "")) { - Assert.assertNotNull("JDBC connection must not be null", conn); - } - } - - private static void waitForJdbc(int port) throws InterruptedException { - String url = jdbcUrl(port); - long deadline = System.currentTimeMillis() + 60_000; - while (System.currentTimeMillis() < deadline) { - try (Connection c = DriverManager.getConnection(url, "hive", "")) { - return; - } catch (Exception ignored) { - Thread.sleep(2000); - } - } - throw new IllegalStateException("HiveServer2 JDBC endpoint not reachable on port " + port + " after 60s"); - } - - private static String jdbcUrl(int port) { - // HS2 is configured with HIVE_SERVER2_AUTHENTICATION=NOSASL, which requires the client to - // set auth=noSasl explicitly. - return "jdbc:hive2://localhost:" + port + "/default;auth=noSasl"; - } - - private static int findFreePort() throws Exception { - try (ServerSocket s = new ServerSocket(0)) { - s.setReuseAddress(true); - return s.getLocalPort(); - } - } -} From 09fca97e173e4bc6d54c8d58f76b543263ae1ffa Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Mon, 27 Jul 2026 12:24:00 +0530 Subject: [PATCH 15/18] HIVE-29483: Clean up comments and remove fork-only GitHub Actions workflow --- .github/workflows/tez-yarn-it.yml | 66 ---------- .../tez/yarn/TestHiveServer2Connectivity.java | 2 - .../tez/yarn/TestTezYarnLocalization.java | 118 +++++++----------- 3 files changed, 44 insertions(+), 142 deletions(-) delete mode 100644 .github/workflows/tez-yarn-it.yml diff --git a/.github/workflows/tez-yarn-it.yml b/.github/workflows/tez-yarn-it.yml deleted file mode 100644 index 10506d3af223..000000000000 --- a/.github/workflows/tez-yarn-it.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Workflow to validate the Tez-on-YARN resource localization integration tests -# (HIVE-29483). These tests require Docker and a real YARN cluster, so they are -# not part of the standard split-based CI. Runs automatically on push to -# tez_on_yarn_it (fork testing) or manually via workflow_dispatch. -# -# Mirrors the local run: -# mvn clean install -DskipTests -Pitests (install artifacts, incl. hive-exec.jar, to ~/.m2) -# mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar -name: Tez-on-YARN localization IT - -on: - push: - branches: - - tez_on_yarn_it - workflow_dispatch: - -jobs: - tez-yarn-it: - runs-on: ubuntu-latest - timeout-minutes: 60 - - steps: - - uses: actions/checkout@v4 - - - name: Set up Java 21 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: '21' - cache: maven - - # Step 1 (mirrors local `mvn clean install -DskipTests -Pitests`): build and - # install artifacts to ~/.m2 so a real hive-exec.jar exists for Tez to - # localize. Scoped with -pl/-am so it finishes within the CI budget while - # still installing hive-exec and every upstream dependency. - - name: Build and install Hive to local Maven repo - run: mvn clean install -DskipTests -Pitests -pl itests/tez-yarn-it -am -Dmaven.javadoc.skip=true - - # Step 2 (mirrors local test command): resolves hive-exec from ~/.m2. - # -Dsurefire.useFile=false streams the test's NodeManager/syslog diagnostics - # into this log so any Tez AM failure is visible directly here. - - name: Run Tez-on-YARN localization test - run: mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar -Dsurefire.useFile=false - - - name: Upload surefire reports on failure - if: failure() - uses: actions/upload-artifact@v4 - with: - name: surefire-reports - path: itests/tez-yarn-it/target/surefire-reports/ diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java index 10fc90f66c36..1cbfa78a5b94 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java @@ -117,8 +117,6 @@ private static void waitForJdbc(int port) throws InterruptedException { } private static String jdbcUrl(int port) { - // HS2 is configured with HIVE_SERVER2_AUTHENTICATION=NOSASL, which requires the client to - // set auth=noSasl explicitly. return "jdbc:hive2://localhost:" + port + "/default;auth=noSasl"; } diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index 305a0e771dbd..24b969f3c4f3 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -39,33 +39,27 @@ import java.util.regex.Pattern; /** - * End-to-end integration test for HIVE-29483: verifies that {@code hive-exec.jar} - * is correctly localized by Tez on YARN. + * End-to-end integration test for HIVE-29483: verifies that hive-exec.jar is correctly + * localized by Tez on YARN. * - *

The test starts a real Hadoop YARN+HDFS cluster in Docker containers - * (fixed host ports so the same URIs work from both the host JVM and inside - * YARN containers), stages Tez framework jars to HDFS, brings up an in-process - * HiveServer2 pointed at that cluster, and then runs an actual Tez DAG via JDBC. + * The test starts a real Hadoop YARN+HDFS cluster in Docker containers (fixed host ports + * so the same URIs work from both the host JVM and inside YARN containers), stages Tez + * framework jars to HDFS, brings up an in-process HiveServer2 pointed at that cluster, + * and then runs an actual Tez DAG via JDBC. * - *

If {@code hive-exec.jar} were missing from {@code commonLocalResources} - * (i.e. if {@code TezSessionState.appJarLr} were removed), the Tez task - * containers would fail with {@code ClassNotFoundException} for Hive executor - * classes and this test would fail. + * If hive-exec.jar were missing from commonLocalResources (i.e. if + * TezSessionState.appJarLr were removed), the Tez task containers would fail with + * ClassNotFoundException for Hive executor classes and this test would fail. * - *

Prerequisites

- *
    - *
  • Docker must be running on the host.
    • - *
  • Host ports 8020, 9870, 8032, 8088 must be free.
    • - *
  • The test JVM must resolve {@code namenode} and {@code resourcemanager} - * to {@code 127.0.0.1}. When run via Maven this is automatic because - * Surefire passes {@code -Djdk.net.hosts.file=custom_hosts_file}.
    • - *
+ * Prerequisites: + * - Docker must be running on the host. + * - Host ports 8020, 9870, 8032, 8088 must be free. + * - The test JVM must resolve "namenode" and "resourcemanager" to 127.0.0.1. + * When run via Maven this is automatic (Surefire passes -Djdk.net.hosts.file). * - *

Running locally

- * 
- * mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \
- *     -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar
- *
+ * Run with: + * mvn test -Pitests,tez-yarn -pl itests/tez-yarn-it \ + * -Dtest=TestTezYarnLocalization#testQuerySucceedsWithAppJar */ public class TestTezYarnLocalization { @@ -86,7 +80,7 @@ public static void startAll() throws Exception { cluster.start(); GenericContainer nn = cluster.namenodeContainer(); - // 1. Explicitly create and open up the global /tmp directory first + // Explicitly create and open up the global /tmp directory first nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp"); nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp"); @@ -100,7 +94,7 @@ public static void startAll() throws Exception { // Force /tmp prefix so the path is valid inside both macOS host and Linux Docker containers Path localScratch = Files.createDirectories( - java.nio.file.Paths.get("/tmp", "hive-tez-loc-" + System.currentTimeMillis())); + Path.of("/tmp", "hive-tez-loc-" + System.currentTimeMillis())); HiveConf conf = buildHiveConf(tezLibUris, localScratch); hs2 = new HiveServer2(); @@ -128,18 +122,16 @@ public static void stopAll() { /** * Positive test case for HIVE-29483. * - *

Runs a Tez DAG via JDBC and asserts that: - *

    - *
  1. The INSERT … SELECT completes and returns the expected row count (primary proof that - * {@code hive-exec.jar} was localized and Hive executor classes were available inside - * YARN task containers).
    2. - *
  2. A Tez YARN application is visible in the ResourceManager (secondary confirmation that - * the DAG actually ran on YARN).
    3. - *
+ * Runs a Tez DAG via JDBC and asserts: + * 1. The INSERT ... SELECT completes and returns the expected row count — primary proof + * that hive-exec.jar was localized and Hive executor classes were available inside + * YARN task containers. + * 2. A Tez YARN application is visible in the ResourceManager — secondary confirmation + * that the DAG actually ran on YARN. * - *

Uses {@code INSERT … SELECT} from an HDFS-backed table rather than {@code VALUES}, because - * {@code VALUES} inserts compile to a local {@code file:/…/dummy_path} input that exists only on - * the host JVM filesystem; Tez map tasks run inside Docker containers and cannot read it. + * Uses INSERT ... SELECT from an HDFS-backed table rather than VALUES, because VALUES + * inserts compile to a local file:/dummy_path input that exists only on the host JVM + * filesystem; Tez map tasks run inside Docker containers and cannot read it. */ @Test public void testQuerySucceedsWithAppJar() throws Exception { @@ -170,38 +162,28 @@ public void testQuerySucceedsWithAppJar() throws Exception { } /** - * Dumps NodeManager container launch scripts and log files when a Tez job fails. + * Dumps NodeManager container logs to System.out for CI visibility. + * Called unconditionally from stopAll() before containers are torn down. * - *

A failing Tez AM often shows an empty {@code stderr} in the YARN diagnostic, which is - * misleading: Tez logs through log4j to the container's {@code syslog} file, not {@code stderr}. - * This helper dumps four sources so CI can pinpoint the real cause without SSH into a container: - *

    - *
  • {@code launch_container.sh} — the exact classpath / env-var setup
    • - *
  • container {@code stderr} — JVM-level errors (empty when the failure is logged via log4j)
    • - *
  • container {@code syslog} — the Tez AM's log4j output (where NoClassDefFoundError etc. lands)
    • - *
  • the NodeManager daemon log — pre-exec launch-script failures that bypass container stderr
    • - *
+ * Four sources are dumped: + * - launch_container.sh: exact java command, JAVA_HOME, classpath used to start the AM + * - container syslog: Tez AM log4j output (where NoClassDefFoundError etc. land) + * - container stdout/stderr/prelaunch.err: JVM-level errors + * - NodeManager daemon log: catches launch-script failures that bypass container stderr */ private static void dumpNodeManagerDiagnostics() { if (cluster == null) { return; } - // Print to System.out (not SLF4J): with -Dsurefire.useFile=false Surefire streams the - // forked JVM's stdout into the build log, whereas Hive routes LOG.* to target/tmp/log/hive.log - // which is invisible in CI. This runs from @AfterClass BEFORE the containers are stopped, - // so the NM container is still alive and its log dirs still exist (delete-debug-delay=600s). + // System.out rather than LOG: Surefire streams stdout into the build log with -Dsurefire.useFile=false. System.out.println("########## BEGIN NodeManager diagnostics ##########"); try { - // launch_container.sh reveals the EXACT java command, JAVA_HOME and full classpath the - // AM was launched with -- decisive when the AM exits 1 with empty stderr/prelaunch.err. + // launch_container.sh: decisive when AM exits 1 with empty stderr. dumpNmCommand("launch_container.sh (AM launch command + classpath)", "find /tmp -name 'launch_container.sh' 2>/dev/null | head -3 " + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); - // Container logs live under LOG_DIRS=/var/log/hadoop/userlogs/// (per - // launch_container.sh), NOT /tmp. syslog is the Tez AM's log4j output -- the CLA appender - // routes DAGAppMaster startup errors (NoClassDefFoundError, NoSuchMethodError, etc.) there, - // which is why the YARN diagnostic shows an empty stderr while the AM still exits 1. + // syslog is the Tez AM's log4j output; YARN diagnostic shows empty stderr even when AM exits 1. dumpNmCommand("container syslog (Tez AM log4j output)", "find /var/log/hadoop/userlogs -name 'syslog*' 2>/dev/null | head -10 " + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); @@ -259,7 +241,7 @@ private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) thro conf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); -// Disable Direct SQL in embedded Derby to prevent JDO transaction corruption + // Disable Direct SQL in embedded Derby to prevent JDO transaction corruption // when querying uninitialized schema tables during race conditions. conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, false); conf.set("hive.stats.autogather", "false"); @@ -276,26 +258,14 @@ private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) thro conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS, false); conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 0); - // Pin the Tez AM's client-facing RPC port so the host JVM can reach the AM. The AM runs in the - // NodeManager container (hostname "nodemanager", resolvable to 127.0.0.1 via custom_hosts_file) - // and publishes AM_CLIENT_PORT; without a fixed port the AM would advertise a random port that - // is not published, so the client fails with "UnknownHostException/connection refused" and the - // Tez session times out. + // Pin the Tez AM client RPC port. The AM runs in the NodeManager container (hostname + // "nodemanager", resolvable to 127.0.0.1) and must advertise a port published to the host. conf.set("tez.am.client.am.port-range", TezYarnClusterContainer.AM_CLIENT_PORT + "-" + TezYarnClusterContainer.AM_CLIENT_PORT); - // The Hadoop image daemons use Java 8 but hive-exec requires Java 21; point Tez AM/task - // container environments at the Java 21 runtime added by the custom Dockerfile. - // HADOOP_HOME must be set explicitly: the apache/hadoop image does not export it into YARN - // container environments, so tez.use.cluster.hadoop-libs's "${HADOOP_HOME}/bin/hadoop classpath" - // call would silently produce an empty classpath without it. - // - // IMPORTANT: set the container env ONLY via the Tez properties. Hive's DagUtils also injects - // mapreduce.map.env / mapreduce.reduce.env into each task vertex (via MRHelpers), and Tez then - // appends tez.task.launch.env on top. Setting the same JAVA_HOME in both sources makes YARN - // concatenate them with the path separator, yielding JAVA_HOME=/opt/jdk21:/opt/jdk21 -- so the - // task container runs "/opt/jdk21:/opt/jdk21/bin/java" and dies with exit 127 (No such file). - // The AM is unaffected because it only reads tez.am.launch.env. + // Tez AM/task containers need Java 21 (hive-exec requires it) and HADOOP_HOME (the + // apache/hadoop image does not export it to YARN container envs; without it + // tez.use.cluster.hadoop-libs's "hadoop classpath" call produces an empty classpath). String containerEnv = "JAVA_HOME=" + TezYarnClusterContainer.CONTAINER_JAVA_21_HOME + ",HADOOP_HOME=/opt/hadoop" + ",HADOOP_MAPRED_HOME=/opt/hadoop"; From 0b30aaa797ea3740734b91eef64a72d88a4926c6 Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Mon, 27 Jul 2026 12:30:34 +0530 Subject: [PATCH 16/18] HIVE-29483: Strip HTML tags from TezYarnClusterContainer Javadoc --- .../tez/yarn/TestTezYarnLocalization.java | 4 - .../tez/yarn/TezYarnClusterContainer.java | 98 ++++++++----------- 2 files changed, 43 insertions(+), 59 deletions(-) diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java index 24b969f3c4f3..4c22c04a1560 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -216,10 +216,6 @@ private static void dumpNmCommand(String label, String bashCommand) { } } - // ------------------------------------------------------------------------- - // Helpers - // ------------------------------------------------------------------------- - private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) throws Exception { HiveConf conf = new HiveConf(); diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java index c6946969eb3e..b1269aa1ac9a 100644 --- a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java @@ -46,31 +46,28 @@ /** * Manages a pseudo-distributed HDFS+YARN cluster (NameNode, DataNode, * ResourceManager, NodeManager) using a custom image built from - * apache/hadoop:3.4.2 (see {@code src/test/docker/hadoop-yarn/Dockerfile}). - * The image adds a Java 21 runtime at {@link #CONTAINER_JAVA_21_HOME} so that + * apache/hadoop:3.4.2 (see src/test/docker/hadoop-yarn/Dockerfile). + * The image adds a Java 21 runtime at CONTAINER_JAVA_21_HOME so that * Tez AM/task containers (which load hive-exec compiled for Java 21) can run, * while the HDFS/YARN daemons keep the image default Java 8. Intended for * integration tests that need real YARN container scheduling and HDFS-backed * resource localization. * - *

All four containers share a single Docker network and are configured via + * All four containers share a single Docker network and are configured via * the same environment-variable convention used by the official image (the * env var key encodes the target XML file and property name, e.g. - * {@code CORE-SITE.XML_fs.defaultFS=hdfs://namenode}). + * CORE-SITE.XML_fs.defaultFS=hdfs://namenode). * - *

Two modes are supported: - *

    - *
  • Dynamic-port mode (default, {@code new TezYarnClusterContainer()}): - * Testcontainers assigns random host ports. Use for tests where only the - * host JVM accesses HDFS/YARN (e.g. JDBC connectivity smoke tests).
    • - *
  • Fixed-port mode ({@code new TezYarnClusterContainer(true)}): - * Host ports match container ports (NN: 8020/9870, RM: 8032/8088). - * The docker-network hostnames {@code namenode} and {@code resourcemanager} - * must resolve to {@code 127.0.0.1} on the host, e.g. via - * {@code -Djdk.net.hosts.file=custom_hosts_file}. Use for tests where - * YARN containers also need to reach HDFS by the same URI as the host JVM - * (real Tez-on-YARN localization tests).
    • - *
+ * Two modes are supported: + * - Dynamic-port mode (default, new TezYarnClusterContainer()): + * Testcontainers assigns random host ports. Use for tests where only the + * host JVM accesses HDFS/YARN (e.g. JDBC connectivity smoke tests). + * - Fixed-port mode (new TezYarnClusterContainer(true)): + * Host ports match container ports (NN: 8020/9870, RM: 8032/8088). + * The docker-network hostnames "namenode" and "resourcemanager" must + * resolve to 127.0.0.1 on the host, e.g. via -Djdk.net.hosts.file. + * Use for tests where YARN containers also need to reach HDFS by the + * same URI as the host JVM (real Tez-on-YARN localization tests). */ public class TezYarnClusterContainer { @@ -83,7 +80,6 @@ public class TezYarnClusterContainer { */ public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; - // Custom image: apache/hadoop:3.4.2 + Corretto 21 at /opt/jdk21 (see src/test/docker/hadoop-yarn/Dockerfile). private static final String HADOOP_IMAGE = buildHadoopImage(); private static final Duration STARTUP_TIMEOUT = Duration.ofMinutes(3); private static final Map COMMON_ENV = loadCommonEnv(); @@ -240,12 +236,11 @@ public void stop() { } /** - * Returns the HDFS URI for use in {@code HiveConf} / {@code Configuration}. + * Returns the HDFS URI for use in HiveConf / Configuration. * - *

In fixed-port mode, returns {@code hdfs://namenode:8020} — a URI that - * works from both the host JVM (via the custom hosts file) and from inside - * YARN containers (via the docker network alias). In dynamic-port mode, - * returns {@code hdfs://localhost:}. + * In fixed-port mode, returns hdfs://namenode:8020 — a URI that works from both the host + * JVM (via the custom hosts file) and from inside YARN containers (via the docker network + * alias). In dynamic-port mode, returns hdfs://localhost:. */ public String getHdfsUri() { if (fixedPorts) { @@ -300,13 +295,11 @@ public String uploadJarToHdfs(Path localJarPath) throws IOException, Interrupted /** * Discovers Tez jars on the test JVM classpath, uploads them to - * {@code /tmp/hive-29483/tez-libs/} on HDFS, and returns a comma-separated - * list of fully-qualified HDFS URIs suitable for the {@code tez.lib.uris} - * configuration property. + * /tmp/hive-29483/tez-libs/ on HDFS, and returns a comma-separated list of + * fully-qualified HDFS URIs suitable for the tez.lib.uris configuration property. * - *

Only available in fixed-port mode; the returned URIs use the stable - * hostname {@code namenode} which must be resolvable from both the host JVM - * and YARN containers. + * Only available in fixed-port mode; the returned URIs use the stable hostname + * "namenode" which must be resolvable from both the host JVM and YARN containers. * * @throws IllegalStateException if no Tez jars are found on the classpath */ @@ -340,24 +333,20 @@ public String uploadTezLibsToHdfs() throws IOException, InterruptedException { } /** - * Discovers Tez framework jars from the running JVM's classpath using two complementary strategies: - *

    - *
  1. Reflection on known probe classes — works regardless of how Surefire passes the classpath - * (direct {@code -cp} or manifest-only argfile JAR).
    2. - *
  2. Scanning {@link ManagementFactory#getRuntimeMXBean() RuntimeMXBean#getClassPath()} - * — a safety net when the reflection approach misses a jar (e.g. classes not loaded yet).
    3. - *
- * Returns regular {@code .jar} files that should be staged into {@code tez.lib.uris} (see - * {@link #isTezFrameworkJar(String)}). This includes: - *
    - *
  • {@code hadoop-shim-*.jar}: carries {@code HadoopShimsLoader} but has no "tez" in its name; - * omitting it causes {@code NoClassDefFoundError: org/apache/tez/hadoop/shim/HadoopShimsLoader} - * in {@code DAGAppMaster.serviceInit()}.
    • - *
  • {@code hadoop-mapreduce-client-core-*.jar}: provides {@code org.apache.hadoop.mapred.JobConf} - * needed by {@code MRInputAMSplitGenerator}; not present in the cluster's - * {@code yarn.application.classpath} and not included in {@code tez.use.cluster.hadoop-libs} - * expansion for {@code apache/hadoop:3.4.2}, so it must be staged explicitly.
    • - *
+ * Discovers Tez framework jars from the running JVM's classpath using two complementary + * strategies: + * 1. Reflection on known probe classes — works regardless of how Surefire passes the + * classpath (direct -cp or manifest-only argfile JAR). + * 2. Scanning RuntimeMXBean.getClassPath() — a safety net when the reflection approach + * misses a jar (e.g. classes not loaded yet). + * + * Returns regular .jar files that should be staged into tez.lib.uris (see isTezFrameworkJar). + * This includes: + * - hadoop-shim-*.jar: carries HadoopShimsLoader but has no "tez" in its name; + * omitting it causes NoClassDefFoundError in DAGAppMaster.serviceInit(). + * - hadoop-mapreduce-client-core-*.jar: provides JobConf needed by MRInputAMSplitGenerator; + * absent from the cluster's yarn.application.classpath and not included in the + * tez.use.cluster.hadoop-libs expansion for apache/hadoop:3.4.2. */ private static Set findTezJarsFromClasspath() { Set jars = new LinkedHashSet<>(); @@ -409,14 +398,13 @@ private static Set findTezJarsFromClasspath() { } /** - * Whether a jar should be staged into {@code tez.lib.uris}. Matches: - *
    - *
  • {@code tez-*} — core Tez framework jars
    • - *
  • {@code hadoop-shim-*} — carries {@code HadoopShimsLoader}; filename has no "tez"
    • - *
  • {@code hadoop-mapreduce-client-core-*} and {@code hadoop-mapreduce-client-common-*} — - * provide {@code JobConf} and related classes required by {@code MRInputAMSplitGenerator}; - * absent from the {@code apache/hadoop:3.4.2} {@code yarn.application.classpath} expansion
    • - *
+ * Whether a jar should be staged into tez.lib.uris. Matches: + * - tez-*: core Tez framework jars + * - hadoop-shim-*: carries HadoopShimsLoader; filename has no "tez" + * - hadoop-mapreduce-client-core-* and hadoop-mapreduce-client-common-*: + * provide JobConf and related classes required by MRInputAMSplitGenerator; + * absent from the apache/hadoop:3.4.2 yarn.application.classpath expansion + * * Test jars are always excluded. */ private static boolean isTezFrameworkJar(String name) { From 38fdde06dec92ca0c888a151aa5b1285f22d47a1 Mon Sep 17 00:00:00 2001 From: Hemanth Umashankar Date: Mon, 27 Jul 2026 12:48:21 +0530 Subject: [PATCH 17/18] HIVE-29483: Fix ASF license header in pom.xml to use full attribution --- itests/tez-yarn-it/pom.xml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/itests/tez-yarn-it/pom.xml b/itests/tez-yarn-it/pom.xml index 28741b85c665..70974a2dcdac 100644 --- a/itests/tez-yarn-it/pom.xml +++ b/itests/tez-yarn-it/pom.xml @@ -1,8 +1,12 @@