diff --git a/itests/pom.xml b/itests/pom.xml index 566bd2fde3dd..951d947ff4ca 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -48,6 +48,7 @@ qtest-iceberg test-docker hive-iceberg + tez-yarn-it diff --git a/itests/tez-yarn-it/pom.xml b/itests/tez-yarn-it/pom.xml new file mode 100644 index 000000000000..70974a2dcdac --- /dev/null +++ b/itests/tez-yarn-it/pom.xml @@ -0,0 +1,171 @@ + + + + 4.0.0 + + org.apache.hive + hive-it + 4.3.0-SNAPSHOT + ../pom.xml + + hive-it-tez-yarn + jar + Hive Integration - Tez on YARN localization tests + + ../.. + true + + + + org.testcontainers + testcontainers + test + + + + org.apache.tez + tez-api + test + + + org.apache.tez + tez-common + test + + + org.apache.tez + tez-dag + test + + + org.apache.tez + tez-runtime-library + test + + + org.apache.tez + tez-runtime-internals + test + + + org.apache.tez + tez-mapreduce + test + + + + org.apache.tez + hadoop-shim + ${tez.version} + test + + + org.apache.hive + hive-service + test + + + org.apache.hive + hive-jdbc + test + + + org.apache.hive + hive-exec + test + + + org.apache.hive + hive-it-util + test + + + org.apache.hadoop + hadoop-common + test + + + org.apache.hadoop + hadoop-mapreduce-client-core + test + + + junit + junit + test + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + generate-custom-hosts-file + + test-compile + + run + + + + + + + + + + + + + 127.0.0.1 ${hive.tez.yarn.it.local.hostname} + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skip.tez.yarn.tests} + + ${maven.test.jvm.args} -Xshare:off -Djdk.net.hosts.file=${test.conf.dir}/custom_hosts_file + + + + + + + tez-yarn + + false + + + + diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile new file mode 100644 index 000000000000..2123acbb57a4 --- /dev/null +++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/Dockerfile @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Custom Hadoop image for the Tez-on-YARN localization integration tests. +# +# The official apache/hadoop:3.4.2 image ships Java 8 for its HDFS/YARN daemons. +# Hive is compiled for Java 21, so Tez AM and task containers -- which load +# hive-exec classes -- must run on Java 21. This image adds a Java 21 JDK at +# /opt/jdk21 using a multi-stage build from eclipse-temurin:21-jdk; the daemons +# continue to use the default Java 8. +FROM --platform=linux/amd64 eclipse-temurin:21-jdk AS jdk21 +FROM apache/hadoop:3.4.2 + +USER root +COPY --from=jdk21 /opt/java/openjdk /opt/jdk21 +# Ensure the JDK binaries are executable and verify the copy is functional. +# Fails at image-build time if the JDK is incompatible with the base image's +# glibc, surfacing the problem early rather than silently at YARN container +# launch time (where YARN shows exit code 1 and empty stderr). +RUN chmod 755 /opt/jdk21/bin/java && /opt/jdk21/bin/java -version + +# commons-collections 3.2.2 is needed by the Tez AM's HDFS client path in +# Tez 0.10.x; it is not bundled in apache/hadoop:3.4.2. +RUN wget -q https://repo1.maven.org/maven2/commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar \ + -P /opt/hadoop/share/hadoop/common/lib/ + +USER hadoop diff --git a/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config new file mode 100644 index 000000000000..a61614b76146 --- /dev/null +++ b/itests/tez-yarn-it/src/test/docker/hadoop-yarn/config @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode + +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +# Ensure the host JVM can reach the DataNode via a stable hostname when +# -Djdk.net.hosts.file is used (and we publish DataNode ports in fixed-port mode). +HDFS-SITE.XML_dfs.datanode.hostname=datanode +HDFS-SITE.XML_dfs.client.use.datanode.hostname=true + +MAPRED-SITE.XML_mapreduce.framework.name=yarn +MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME + +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +# Docker cgroup limits trip the default memory checks on NodeManager. +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +# Generous liveness expiry to tolerate slow container startup (e.g. under heavy load on CI). +YARN-SITE.XML_yarn.nm.liveness-monitor.expiry-interval-ms=3600000 +YARN-SITE.XML_yarn.am.liveness-monitor.expiry-interval-ms=3600000 +YARN-SITE.XML_yarn.resourcemanager.nodemanagers.heartbeat-interval-ms=1000 +# Explicit absolute classpath so YARN container launch scripts do not rely on +# $HADOOP_COMMON_HOME / $HADOOP_HDFS_HOME env-var expansion, which is not +# available in child container environments (only the NM daemon's own env is). +# Without this, tez.use.cluster.hadoop-libs=true leaves Hadoop jars off the +# Tez AM classpath, causing the launch script to fail with exit code 1 and +# empty stderr (the error fires before the java stderr redirect is active). +YARN-SITE.XML_yarn.application.classpath=/opt/hadoop/etc/hadoop,/opt/hadoop/share/hadoop/common/*,/opt/hadoop/share/hadoop/common/lib/*,/opt/hadoop/share/hadoop/hdfs/*,/opt/hadoop/share/hadoop/hdfs/lib/*,/opt/hadoop/share/hadoop/yarn/*,/opt/hadoop/share/hadoop/yarn/lib/*,/opt/hadoop/share/hadoop/mapreduce/*,/opt/hadoop/share/hadoop/mapreduce/lib/* +# Propagate HADOOP_HOME (and friends) into YARN containers so that tez.am.launch.env +# overrides and any hadoop-config.sh sourcing in launch scripts can resolve correctly. +YARN-SITE.XML_yarn.nodemanager.env-whitelist=JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME,HADOOP_HOME,PATH,LANG,TZ + +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java new file mode 100644 index 000000000000..1cbfa78a5b94 --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestHiveServer2Connectivity.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; + +public class TestHiveServer2Connectivity { + + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + private static String hdfsUri; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(); + cluster.start(); + + hdfsUri = cluster.getHdfsUri(); + + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/hive-29483/warehouse"); + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/hive-29483/scratch"); + cluster.namenodeContainer().execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp/hive-29483"); + + Path localScratch = Files.createTempDirectory("hive-29483-local-"); + + HiveConf conf = new HiveConf(); + URL hiveSite = TestHiveServer2Connectivity.class.getClassLoader().getResource("hive-site-yarn-it.xml"); + URL yarnSite = TestHiveServer2Connectivity.class.getClassLoader().getResource("yarn-site.xml"); + if (hiveSite != null) conf.addResource(hiveSite); + if (yarnSite != null) conf.addResource(yarnSite); + + conf.set("fs.defaultFS", hdfsUri); + conf.set("hive.metastore.warehouse.dir", hdfsUri + "/tmp/hive-29483/warehouse"); + conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, hdfsUri + "/tmp/hive-29483/scratch"); + conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname, localScratch.toAbsolutePath().toString()); + + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); + + conf.set("yarn.resourcemanager.address", cluster.getResourceManagerAddress()); + conf.set("yarn.resourcemanager.webapp.address", cluster.getResourceManagerWebAppAddress()); + + hs2Port = findFreePort(); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, "localhost"); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort()); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, "NOSASL"); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + waitForJdbc(hs2Port); + } + + @AfterClass + public static void stopAll() { + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + } + + @Test + public void testJdbcSessionOpen() throws Exception { + String url = jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + Assert.assertNotNull("JDBC connection must not be null", conn); + } + } + + private static void waitForJdbc(int port) throws InterruptedException { + String url = jdbcUrl(port); + long deadline = System.currentTimeMillis() + 60_000; + while (System.currentTimeMillis() < deadline) { + try (Connection c = DriverManager.getConnection(url, "hive", "")) { + return; + } catch (Exception ignored) { + Thread.sleep(2000); + } + } + throw new IllegalStateException("HiveServer2 JDBC endpoint not reachable on port " + port + " after 60s"); + } + + private static String jdbcUrl(int port) { + return "jdbc:hive2://localhost:" + port + "/default;auth=noSasl"; + } + + private static int findFreePort() throws Exception { + try (ServerSocket s = new ServerSocket(0)) { + s.setReuseAddress(true); + return s.getLocalPort(); + } + } +} diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnClusterContainer.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnClusterContainer.java new file mode 100644 index 000000000000..928788fecf6b --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnClusterContainer.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.testcontainers.containers.Container.ExecResult; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class TestTezYarnClusterContainer { + + private static TezYarnClusterContainer cluster; + + @BeforeClass + public static void startCluster() { + cluster = new TezYarnClusterContainer(); + cluster.start(); + } + + @AfterClass + public static void stopCluster() { + if (cluster != null) { + cluster.stop(); + } + } + + @Test + public void testHdfsUriFormat() { + String uri = cluster.getHdfsUri(); + Assert.assertNotNull("HDFS URI must not be null", uri); + Assert.assertTrue("Expected hdfs:// URI, got: " + uri, uri.startsWith("hdfs://")); + } + + @Test + public void testResourceManagerAddressFormat() { + String addr = cluster.getResourceManagerAddress(); + Assert.assertNotNull("RM address must not be null", addr); + Assert.assertTrue("Expected host:port, got: " + addr, addr.contains(":")); + } + + @Test + public void testHdfsWriteAndRead() throws IOException, InterruptedException { + ExecResult mkdir = cluster.namenodeContainer() + .execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp/smoke-test"); + Assert.assertEquals("hdfs mkdir failed:\n" + mkdir.getStderr(), 0, mkdir.getExitCode()); + + ExecResult ls = cluster.namenodeContainer() + .execInContainer("hdfs", "dfs", "-ls", "/tmp"); + Assert.assertTrue("Expected /tmp/smoke-test in HDFS listing:\n" + ls.getStdout(), + ls.getStdout().contains("smoke-test")); + } + + @Test + public void testYarnNodeManagerRegistered() throws IOException, InterruptedException { + ExecResult result = cluster.resourceManagerContainer() + .execInContainer("yarn", "node", "-list"); + String out = result.getStdout(); + Assert.assertTrue("Expected 'Total Nodes:' in yarn node -list output:\n" + out, + out.contains("Total Nodes:")); + Assert.assertFalse("Expected at least one active NodeManager:\n" + out, + out.contains("Total Nodes:0")); + } + + @Test + public void testUploadJarToHdfs() throws IOException, InterruptedException { + Path tempJar = Files.createTempFile("smoke-upload", ".jar"); + try { + Files.write(tempJar, "placeholder-jar-content".getBytes()); + + String hdfsPath = cluster.uploadJarToHdfs(tempJar); + Assert.assertNotNull("uploadJarToHdfs must return a non-null HDFS path", hdfsPath); + + ExecResult ls = cluster.namenodeContainer() + .execInContainer("hdfs", "dfs", "-ls", "/tmp/hive-29483-jars"); + Assert.assertTrue("Uploaded jar not visible in HDFS:\n" + ls.getStdout(), + ls.getStdout().contains(tempJar.getFileName().toString())); + } finally { + Files.deleteIfExists(tempJar); + } + } +} diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java new file mode 100644 index 000000000000..8a4f082569b4 --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TestTezYarnLocalization.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; + +import java.net.ServerSocket; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class TestTezYarnLocalization { + + private static final Logger LOG = LoggerFactory.getLogger(TestTezYarnLocalization.class); + + private static final String HDFS_BASE = "hdfs://namenode:8020"; + private static final String HDFS_WAREHOUSE = HDFS_BASE + "/tmp/hive-tez-loc/warehouse"; + private static final String HDFS_SCRATCH = HDFS_BASE + "/tmp/hive-tez-loc/scratch"; + private static final String HDFS_ROOT = "/tmp/hive-tez-loc"; + + private static TezYarnClusterContainer cluster; + private static HiveServer2 hs2; + private static int hs2Port; + + @BeforeClass + public static void startAll() throws Exception { + cluster = new TezYarnClusterContainer(true); + cluster.start(); + + GenericContainer nn = cluster.namenodeContainer(); + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", "/tmp"); + nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", "/tmp"); + + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/warehouse"); + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/scratch"); + nn.execInContainer("hdfs", "dfs", "-mkdir", "-p", HDFS_ROOT + "/user-install"); + nn.execInContainer("hdfs", "dfs", "-chmod", "-R", "777", HDFS_ROOT); + + String tezLibUris = cluster.uploadTezLibsToHdfs(); + LOG.info("Staged Tez libs to HDFS: {}", tezLibUris); + + Path localScratch = Files.createDirectories( + Path.of("/tmp", "hive-tez-loc-" + System.currentTimeMillis())); + HiveConf conf = buildHiveConf(tezLibUris, localScratch); + + hs2 = new HiveServer2(); + hs2.init(conf); + hs2.start(); + + waitForJdbc(hs2Port); + LOG.info("HiveServer2 is ready on port {}", hs2Port); + } + + @AfterClass + public static void stopAll() { + dumpNodeManagerDiagnostics(); + if (hs2 != null) { + hs2.stop(); + hs2 = null; + } + if (cluster != null) { + cluster.stop(); + cluster = null; + } + } + + @Test + public void testQuerySucceedsWithAppJar() throws Exception { + String url = jdbcUrl(hs2Port); + try (Connection conn = DriverManager.getConnection(url, "hive", "")) { + try (Statement stmt = conn.createStatement()) { + + stmt.execute("CREATE TABLE IF NOT EXISTS tez_loc_test (id INT) STORED AS ORC"); + stmt.execute("CREATE TABLE IF NOT EXISTS tez_source (id INT) STORED AS ORC"); + + stmt.execute("INSERT INTO tez_loc_test SELECT count(*) FROM tez_source"); + + try (ResultSet rs = stmt.executeQuery("SELECT id FROM tez_loc_test")) { + Assert.assertTrue("Result set must contain at least one row", rs.next()); + long count = rs.getLong(1); + Assert.assertEquals( + "INSERT SELECT count(*) FROM empty tez_source should return 0 (hive-exec.jar was localized)", + 0L, count); + LOG.info("Tez query succeeded: inserted count(*) = {}", count); + } + } + } + + verifyTezYarnAppExists(); + } + + /** Prints Tez AM and NodeManager logs to the Surefire *-output.txt file at teardown. */ + private static void dumpNodeManagerDiagnostics() { + if (cluster == null) { + return; + } + System.out.println("########## BEGIN NodeManager diagnostics ##########"); + try { + dumpNmCommand("launch_container.sh (AM launch command + classpath)", + "find /tmp -name 'launch_container.sh' 2>/dev/null | head -3 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("container syslog (Tez AM log4j output)", + "find /var/log/hadoop/userlogs -name 'syslog*' 2>/dev/null | head -10 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("container stdout + stderr + prelaunch.err", + "find /var/log/hadoop/userlogs \\( -name 'stdout' -o -name 'stderr' -o -name 'prelaunch.err' \\) " + + "2>/dev/null | head -20 | xargs -I{} sh -c 'echo \"--- {} ---\"; cat {}' 2>/dev/null || true"); + + dumpNmCommand("NodeManager daemon log (tail 200)", + "find /var/log/hadoop -maxdepth 1 -name '*.log' 2>/dev/null | head -3 " + + "| xargs -I{} sh -c 'echo \"--- {} ---\"; tail -200 {}' 2>/dev/null || true"); + } catch (Exception e) { + System.out.println("Could not dump NodeManager diagnostics: " + e); + } + System.out.println("########## END NodeManager diagnostics ##########"); + System.out.flush(); + } + + private static void dumpNmCommand(String label, String bashCommand) { + try { + GenericContainer.ExecResult r = + cluster.nodeManagerContainer().execInContainer("bash", "-c", bashCommand); + String out = r.getStdout(); + System.out.println("===== NM: " + label + " ====="); + System.out.println(out.isEmpty() ? "(no output found)" : out); + } catch (Exception e) { + System.out.println("===== NM: " + label + " (dump failed: " + e + ") ====="); + } + } + + private static HiveConf buildHiveConf(String tezLibUris, Path localScratch) throws Exception { + HiveConf conf = new HiveConf(); + + URL hiveSite = TestTezYarnLocalization.class.getClassLoader().getResource("hive-site-yarn-it.xml"); + URL yarnSite = TestTezYarnLocalization.class.getClassLoader().getResource("yarn-site.xml"); + if (hiveSite != null) { conf.addResource(hiveSite); } + if (yarnSite != null) { conf.addResource(yarnSite); } + + conf.set("fs.defaultFS", HDFS_BASE); + conf.setBoolean("dfs.client.use.datanode.hostname", true); + conf.set("hive.metastore.warehouse.dir", HDFS_WAREHOUSE); + conf.set(HiveConf.ConfVars.SCRATCH_DIR.varname, HDFS_SCRATCH); + conf.set(HiveConf.ConfVars.LOCAL_SCRATCH_DIR.varname, localScratch.toAbsolutePath().toString()); + conf.setVar(HiveConf.ConfVars.HIVE_USER_INSTALL_DIR, HDFS_ROOT + "/user-install"); + + conf.set("javax.jdo.option.ConnectionURL", + "jdbc:derby:" + localScratch.resolve("metastore_db").toAbsolutePath() + ";create=true"); + + conf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL, false); + conf.set("hive.stats.autogather", "false"); + conf.set("hive.stats.column.autogather", "false"); + conf.set("yarn.resourcemanager.hostname", "resourcemanager"); + conf.set("yarn.resourcemanager.address", "resourcemanager:8032"); + conf.set("yarn.resourcemanager.webapp.address", "resourcemanager:8088"); + + conf.set("tez.lib.uris", tezLibUris); + conf.setBoolean("tez.use.cluster.hadoop-libs", true); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS, false); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE, 0); + + conf.set("tez.am.client.am.port-range", + TezYarnClusterContainer.AM_CLIENT_PORT + "-" + TezYarnClusterContainer.AM_CLIENT_PORT); + + String containerEnv = "JAVA_HOME=" + TezYarnClusterContainer.CONTAINER_JAVA_21_HOME + + ",HADOOP_HOME=/opt/hadoop" + + ",HADOOP_MAPRED_HOME=/opt/hadoop"; + conf.set("tez.am.launch.env", containerEnv); + conf.set("tez.task.launch.env", containerEnv); + + hs2Port = findFreePort(); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, hs2Port); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, "localhost"); + conf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_WEBUI_PORT, findFreePort()); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); + conf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, "NOSASL"); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + + return conf; + } + + private static void verifyTezYarnAppExists() { + try { + GenericContainer rm = cluster.resourceManagerContainer(); + GenericContainer.ExecResult result = rm.execInContainer( + "yarn", "application", "-list", "-appTypes", "TEZ", "-appStates", "ALL"); + String out = result.getStdout(); + LOG.info("YARN application list (TEZ, ALL states):\n{}", out); + + Pattern appIdPattern = Pattern.compile("(application_\\d+_\\d+)"); + Matcher matcher = appIdPattern.matcher(out); + boolean found = false; + while (matcher.find()) { + LOG.info("Found Tez YARN application: {}", matcher.group(1)); + found = true; + } + + Assert.assertTrue( + "At least one Tez YARN application must be visible in the ResourceManager after running a Tez query", + found); + + } catch (Exception e) { + LOG.warn("Could not verify Tez YARN application existence via RM exec; " + + "primary query-result assertion already passed. Cause: {}", e.getMessage()); + } + } + + private static void waitForJdbc(int port) throws InterruptedException { + String url = jdbcUrl(port); + long deadline = System.currentTimeMillis() + 120_000; + while (System.currentTimeMillis() < deadline) { + try (Connection c = DriverManager.getConnection(url, "hive", "")) { + return; + } catch (Exception ignored) { + Thread.sleep(2000); + } + } + throw new IllegalStateException( + "HiveServer2 JDBC endpoint not reachable on port " + port + " after 120s"); + } + + private static String jdbcUrl(int port) { + return "jdbc:hive2://localhost:" + port + "/default;auth=noSasl"; + } + + private static int findFreePort() throws Exception { + try (ServerSocket s = new ServerSocket(0)) { + s.setReuseAddress(true); + return s.getLocalPort(); + } + } +} diff --git a/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java new file mode 100644 index 000000000000..59f5f4c3a803 --- /dev/null +++ b/itests/tez-yarn-it/src/test/java/org/apache/hive/tez/yarn/TezYarnClusterContainer.java @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.tez.yarn; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.FixedHostPortGenericContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.CodeSource; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +public class TezYarnClusterContainer { + + private static final Logger LOG = LoggerFactory.getLogger(TezYarnClusterContainer.class); + + /** Path to the Java 21 runtime inside containers for Tez AM/task launch environments. */ + public static final String CONTAINER_JAVA_21_HOME = "/opt/jdk21"; + + private static final String HADOOP_IMAGE = buildHadoopImage(); + private static final Duration STARTUP_TIMEOUT = Duration.ofMinutes(3); + private static final Map COMMON_ENV = loadCommonEnv(); + + private static final int NN_RPC_PORT = 8020; + private static final int NN_HTTP_PORT = 9870; + private static final int RM_RPC_PORT = 8032; + private static final int RM_HTTP_PORT = 8088; + private static final int DN_HTTP_PORT = 9864; + private static final int DN_XFER_PORT = 9866; + // Tez AM client RPC port, published by the NM container so the host JVM can reach the AM. + public static final int AM_CLIENT_PORT = 41000; + + private final Network network; + private final GenericContainer namenode; + private final GenericContainer datanode; + private final GenericContainer resourcemanager; + private final GenericContainer nodemanager; + private final boolean fixedPorts; + + public TezYarnClusterContainer() { + this(false); + } + + public TezYarnClusterContainer(boolean fixedPorts) { + this.fixedPorts = fixedPorts; + network = Network.newNetwork(); + + if (fixedPorts) { + namenode = new FixedHostPortGenericContainer<>(HADOOP_IMAGE) + .withFixedExposedPort(NN_RPC_PORT, NN_RPC_PORT) + .withFixedExposedPort(NN_HTTP_PORT, NN_HTTP_PORT) + .withNetwork(network) + .withNetworkAliases("namenode") + .withCommand("hdfs", "namenode") + .withEnv(COMMON_ENV) + .withEnv("ENSURE_NAMENODE_DIR", "/tmp/hadoop-hadoop/dfs/name") + .waitingFor(Wait.forHttp("/").forPort(NN_HTTP_PORT).withStartupTimeout(STARTUP_TIMEOUT)); + + resourcemanager = new FixedHostPortGenericContainer<>(HADOOP_IMAGE) + .withFixedExposedPort(RM_RPC_PORT, RM_RPC_PORT) + .withFixedExposedPort(RM_HTTP_PORT, RM_HTTP_PORT) + .withNetwork(network) + .withNetworkAliases("resourcemanager") + .withCommand("yarn", "resourcemanager") + .withEnv(COMMON_ENV) + .waitingFor(Wait.forHttp("/ws/v1/cluster/info").forPort(RM_HTTP_PORT).withStartupTimeout(STARTUP_TIMEOUT)); + + datanode = new FixedHostPortGenericContainer<>(HADOOP_IMAGE) + .withFixedExposedPort(DN_XFER_PORT, DN_XFER_PORT) + .withFixedExposedPort(DN_HTTP_PORT, DN_HTTP_PORT) + .withNetwork(network) + .withNetworkAliases("datanode") + .withCommand("hdfs", "datanode") + .withEnv(COMMON_ENV); + } else { + namenode = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("namenode") + .withCommand("hdfs", "namenode") + .withEnv(COMMON_ENV) + .withEnv("ENSURE_NAMENODE_DIR", "/tmp/hadoop-hadoop/dfs/name") + .withExposedPorts(NN_HTTP_PORT, NN_RPC_PORT) + .waitingFor(Wait.forHttp("/").forPort(NN_HTTP_PORT).withStartupTimeout(STARTUP_TIMEOUT)); + + resourcemanager = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("resourcemanager") + .withCommand("yarn", "resourcemanager") + .withEnv(COMMON_ENV) + .withExposedPorts(RM_HTTP_PORT, RM_RPC_PORT) + .waitingFor(Wait.forHttp("/ws/v1/cluster/info").forPort(RM_HTTP_PORT).withStartupTimeout(STARTUP_TIMEOUT)); + + datanode = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("datanode") + .withCommand("hdfs", "datanode") + .withEnv(COMMON_ENV); + } + + if (fixedPorts) { + // Fixed hostname "nodemanager" and published AM_CLIENT_PORT so host JVM can reach the Tez AM. + nodemanager = new FixedHostPortGenericContainer<>(HADOOP_IMAGE) + .withFixedExposedPort(AM_CLIENT_PORT, AM_CLIENT_PORT) + .withCreateContainerCmdModifier(cmd -> cmd.withHostName("nodemanager")) + .withNetwork(network) + .withNetworkAliases("nodemanager") + .withCommand("yarn", "nodemanager") + .withEnv(COMMON_ENV); + } else { + nodemanager = new GenericContainer<>(HADOOP_IMAGE) + .withNetwork(network) + .withNetworkAliases("nodemanager") + .withCommand("yarn", "nodemanager") + .withEnv(COMMON_ENV); + } + } + + public void start() { + namenode.start(); + datanode.start(); + resourcemanager.start(); + nodemanager.start(); + waitForNodeManagerRegistration(); + verifyJava21InNodeManager(); + } + + private void verifyJava21InNodeManager() { + try { + GenericContainer.ExecResult r = nodemanager.execInContainer( + CONTAINER_JAVA_21_HOME + "/bin/java", "-version"); + if (r.getExitCode() == 0) { + LOG.info("Java 21 is functional in NodeManager ({}): {}", + CONTAINER_JAVA_21_HOME, r.getStderr().trim()); + } else { + LOG.warn("Java 21 check FAILED in NodeManager (exit {}). " + + "Tez AM/task containers will fail at launch time. " + + "stderr: {}", r.getExitCode(), r.getStderr()); + } + } catch (Exception e) { + LOG.warn("Could not verify Java 21 in NodeManager container", e); + } + } + + public void stop() { + nodemanager.stop(); + resourcemanager.stop(); + datanode.stop(); + namenode.stop(); + network.close(); + } + + public String getHdfsUri() { + if (fixedPorts) { + return "hdfs://namenode:" + NN_RPC_PORT; + } + return "hdfs://" + namenode.getHost() + ":" + namenode.getMappedPort(NN_RPC_PORT); + } + + public String getResourceManagerAddress() { + if (fixedPorts) { + return "resourcemanager:" + RM_RPC_PORT; + } + return resourcemanager.getHost() + ":" + resourcemanager.getMappedPort(RM_RPC_PORT); + } + + public String getResourceManagerWebAppAddress() { + if (fixedPorts) { + return "resourcemanager:" + RM_HTTP_PORT; + } + return resourcemanager.getHost() + ":" + resourcemanager.getMappedPort(RM_HTTP_PORT); + } + + public String uploadJarToHdfs(Path localJarPath) throws IOException, InterruptedException { + String fileName = localJarPath.getFileName().toString(); + String containerTmp = "/tmp/" + fileName; + String hdfsDir = "/tmp/hive-29483-jars"; + String hdfsPath = hdfsDir + "/" + fileName; + + namenode.copyFileToContainer(MountableFile.forHostPath(localJarPath, 0644), containerTmp); + + GenericContainer.ExecResult mkdir = namenode.execInContainer("hdfs", "dfs", "-mkdir", "-p", hdfsDir); + requireSuccess(mkdir, "hdfs dfs -mkdir -p " + hdfsDir); + + GenericContainer.ExecResult put = namenode.execInContainer("hdfs", "dfs", "-put", "-f", containerTmp, hdfsPath); + requireSuccess(put, "hdfs dfs -put -f " + containerTmp + " " + hdfsPath); + + return hdfsPath; + } + + public String uploadTezLibsToHdfs() throws IOException, InterruptedException { + String hdfsDir = "/tmp/hive-29483/tez-libs"; + GenericContainer.ExecResult mkdir = namenode.execInContainer("hdfs", "dfs", "-mkdir", "-p", hdfsDir); + requireSuccess(mkdir, "hdfs dfs -mkdir -p " + hdfsDir); + + Set tezJars = findTezJarsFromClasspath(); + List hdfsUris = new ArrayList<>(); + for (Path jarPath : tezJars) { + String jarName = jarPath.getFileName().toString(); + String containerTmp = "/tmp/" + jarName; + String hdfsPath = hdfsDir + "/" + jarName; + + namenode.copyFileToContainer(MountableFile.forHostPath(jarPath, 0644), containerTmp); + GenericContainer.ExecResult put = namenode.execInContainer( + "hdfs", "dfs", "-put", "-f", containerTmp, hdfsPath); + requireSuccess(put, "hdfs dfs -put -f " + containerTmp + " " + hdfsPath); + + hdfsUris.add("hdfs://namenode:" + NN_RPC_PORT + hdfsPath); + } + + if (hdfsUris.isEmpty()) { + throw new IllegalStateException( + "No framework jars were found on the test classpath to stage. " + + "Ensure tez-api, tez-dag, tez-runtime-library, hadoop-mapreduce-client-core, " + + "etc. are test dependencies."); + } + return String.join(",", hdfsUris); + } + + /** Discovers Tez jars from the classpath via probe-class reflection and string scanning; includes hadoop-shim and MR client jars absent from the cluster classpath. */ + private static Set findTezJarsFromClasspath() { + Set jars = new LinkedHashSet<>(); + + String[] probeClassNames = { + "org.apache.tez.dag.api.TezConfiguration", + "org.apache.tez.common.TezConverterUtils", + "org.apache.tez.dag.app.DAGAppMaster", + "org.apache.tez.mapreduce.hadoop.MRHelpers", + "org.apache.tez.runtime.LogicalIOProcessorRuntimeTask", + "org.apache.tez.runtime.library.api.KeyValueReader", + "org.apache.tez.hadoop.shim.HadoopShimsLoader", + "org.apache.hadoop.mapred.JobConf", + "org.apache.hadoop.mapreduce.v2.util.MRApps" + }; + for (String className : probeClassNames) { + try { + Class cls = Class.forName(className); + CodeSource cs = cls.getProtectionDomain().getCodeSource(); + if (cs != null && cs.getLocation() != null) { + String path = cs.getLocation().getPath(); + if (path.endsWith(".jar")) { + Path p = Paths.get(path); + if (isTezFrameworkJar(p.getFileName().toString()) && Files.isRegularFile(p)) { + jars.add(p); + } + } + } + } catch (ClassNotFoundException | SecurityException ignored) { + } + } + + String cp = ManagementFactory.getRuntimeMXBean().getClassPath(); + for (String entry : cp.split(File.pathSeparator)) { + if (!entry.endsWith(".jar")) { + continue; + } + Path p = Paths.get(entry); + if (isTezFrameworkJar(p.getFileName().toString()) && Files.isRegularFile(p)) { + jars.add(p); + } + } + + return jars; + } + + private static boolean isTezFrameworkJar(String name) { + if (name.endsWith("-tests.jar")) { + return false; + } + return name.contains("tez") + || name.startsWith("hadoop-shim") + || name.startsWith("hadoop-mapreduce-client-core") + || name.startsWith("hadoop-mapreduce-client-common"); + } + + GenericContainer namenodeContainer() { + return namenode; + } + + GenericContainer resourceManagerContainer() { + return resourcemanager; + } + + GenericContainer nodeManagerContainer() { + return nodemanager; + } + + private void waitForNodeManagerRegistration() { + long deadline = System.currentTimeMillis() + Duration.ofMinutes(2).toMillis(); + while (System.currentTimeMillis() < deadline) { + try { + GenericContainer.ExecResult result = resourcemanager.execInContainer("yarn", "node", "-list"); + String out = result.getStdout(); + if (out.contains("Total Nodes:") && !out.contains("Total Nodes:0")) { + return; + } + Thread.sleep(3000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for NodeManager registration"); + } catch (Exception ignored) { + try { Thread.sleep(3000); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } + } + } + throw new IllegalStateException("NodeManager did not register with ResourceManager within 2 minutes"); + } + + private static void requireSuccess(GenericContainer.ExecResult result, String cmd) { + if (result.getExitCode() != 0) { + throw new IllegalStateException("Command failed (" + cmd + ")\nstdout:\n" + + result.getStdout() + "\nstderr:\n" + result.getStderr()); + } + } + + private static Map loadCommonEnv() { + Map env = new LinkedHashMap<>(); + String basedir = System.getProperty("basedir", "."); + Path configPath = Paths.get(basedir, "src/test/docker/hadoop-yarn/config"); + try (Stream lines = Files.lines(configPath, StandardCharsets.UTF_8)) { + lines.map(String::trim) + .filter(l -> !l.isEmpty()) + .filter(l -> !l.startsWith("#")) + .forEach(l -> { + int idx = l.indexOf('='); + if (idx < 0) { + throw new IllegalArgumentException("Invalid config line (missing '='): " + l); + } + String key = l.substring(0, idx).trim(); + String value = l.substring(idx + 1); + env.put(key, value); + }); + } catch (IOException e) { + throw new IllegalStateException("Failed to load Hadoop docker config from " + configPath, e); + } + return env; + } + + private static String buildHadoopImage() { + String basedir = System.getProperty("basedir", "."); + Path dockerfile = Paths.get(basedir, "src/test/docker/hadoop-yarn/Dockerfile"); + return new ImageFromDockerfile("hive-it-hadoop-jdk21", false) + .withDockerfile(dockerfile) + .get(); + } +} diff --git a/itests/tez-yarn-it/src/test/resources/custom_hosts_file b/itests/tez-yarn-it/src/test/resources/custom_hosts_file new file mode 100644 index 000000000000..642911816ad4 --- /dev/null +++ b/itests/tez-yarn-it/src/test/resources/custom_hosts_file @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Hostname overrides for Tez-on-YARN integration tests. +# +# When TezYarnClusterContainer is started in fixed-port mode, these four +# containers are reachable from the host at 127.0.0.1 on their well-known +# ports (NameNode 8020/9870, ResourceManager 8032/8088). The same hostnames +# are used inside the Docker network, so a single HDFS URI like +# hdfs://namenode:8020 works both from the test JVM and from YARN containers. +# +# This file is passed to the test JVM via +# -Djdk.net.hosts.file=/custom_hosts_file +# which makes the JVM's DNS resolver consult it before the OS resolver. + +127.0.0.1 localhost +127.0.0.1 namenode +127.0.0.1 resourcemanager +127.0.0.1 datanode +127.0.0.1 nodemanager diff --git a/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml b/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml new file mode 100644 index 000000000000..e76f403d5065 --- /dev/null +++ b/itests/tez-yarn-it/src/test/resources/hive-site-yarn-it.xml @@ -0,0 +1,63 @@ + + + + + + + hive.in.test + true + + + + hive.execution.engine + tez + + + + hive.server2.enable.doAs + false + + + + hive.support.concurrency + false + + + + + hive.query.history.enabled + false + + + + datanucleus.schema.autoCreateAll + true + + + + hive.metastore.schema.verification + false + + + + datanucleus.connectionPool.maxPoolSize + 4 + + + diff --git a/itests/tez-yarn-it/src/test/resources/yarn-site.xml b/itests/tez-yarn-it/src/test/resources/yarn-site.xml new file mode 100644 index 000000000000..fda5ad066fa8 --- /dev/null +++ b/itests/tez-yarn-it/src/test/resources/yarn-site.xml @@ -0,0 +1,33 @@ + + + + + + + + yarn.nodemanager.pmem-check-enabled + false + + + + yarn.nodemanager.vmem-check-enabled + false + + +