diff --git a/pom.xml b/pom.xml index 00d22c757e..02ca49e270 100644 --- a/pom.xml +++ b/pom.xml @@ -302,7 +302,6 @@ org.apache.hadoop hadoop-aws 3.3.6 - provided org.apache.spark diff --git a/src/main/java/org/rumbledb/cli/JsoniqQueryExecutor.java b/src/main/java/org/rumbledb/cli/JsoniqQueryExecutor.java index 3ee24bafef..ecbf476711 100644 --- a/src/main/java/org/rumbledb/cli/JsoniqQueryExecutor.java +++ b/src/main/java/org/rumbledb/cli/JsoniqQueryExecutor.java @@ -28,6 +28,7 @@ import org.rumbledb.exceptions.ExceptionMetadata; import org.rumbledb.optimizations.Profiler; import org.rumbledb.runtime.functions.input.FileSystemUtil; +import org.slf4j.Logger; import org.rumbledb.serialization.Serializer; import java.io.IOException; @@ -37,11 +38,13 @@ import java.util.List; import java.util.stream.Collectors; +import org.apache.spark.internal.Logging; -public class JsoniqQueryExecutor { +public class JsoniqQueryExecutor implements Logging { private RumbleRuntimeConfiguration configuration; public JsoniqQueryExecutor(RumbleRuntimeConfiguration configuration) { + initializeLogIfNecessary(true, true); this.configuration = configuration; } @@ -178,4 +181,16 @@ public long runInteractive(String query, List resultList) throws IOExcepti return sequence.populateList(resultList, this.configuration.getResultSizeCap()); } + @Override + public Logger org$apache$spark$internal$Logging$$log_() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void org$apache$spark$internal$Logging$$log__$eq(Logger x$1) { + // TODO Auto-generated method stub + + } + } diff --git a/src/main/java/org/rumbledb/cli/Main.java b/src/main/java/org/rumbledb/cli/Main.java index 98504198d3..da6fa0f68b 100644 --- a/src/main/java/org/rumbledb/cli/Main.java +++ b/src/main/java/org/rumbledb/cli/Main.java @@ -112,7 +112,13 @@ private static void handleException(Throwable ex, boolean showErrorInfo) { "⚠️ Java went out of memory." ); System.err.println( - "If running locally, try adding --driver-memory 10G (or any quantity you need) between spark-submit and the RumbleDB jar in the command line to see if it fixes the problem. If running on a cluster, --executor-memory is the way to go." + "If running locally with java -jar, try adding --Xmx10g (or any quantity you need) before the RumbleDB jar in the command line to see if it fixes the problem." + ); + System.err.println( + "If running locally with spark-submit, try adding --driver-memory 10G (or any quantity you need) between spark-submit and the RumbleDB jar in the command line to see if it fixes the problem." + ); + System.err.println( + "If running on a cluster, --executor-memory should be used instead." ); if (showErrorInfo) { ex.printStackTrace(); diff --git a/src/main/java/sparksoniq/spark/SparkSessionManager.java b/src/main/java/sparksoniq/spark/SparkSessionManager.java index f50628563b..d6b54f50e6 100644 --- a/src/main/java/sparksoniq/spark/SparkSessionManager.java +++ b/src/main/java/sparksoniq/spark/SparkSessionManager.java @@ -159,6 +159,26 @@ public SparkSession getOrCreateSession() { private void setDefaultConfiguration() { try { + if (System.getProperty("hadoop.home.dir") == null) { + System.err.println( + "[WARNING] The hadoop home directory was not set. Setting to \"/\"." + ); + System.setProperty("hadoop.home.dir", "/"); + } + String javaVersion = System.getProperty("java.version"); + if (!javaVersion.startsWith("1.8") && !javaVersion.startsWith("11.")) { + System.err.println("[Error] RumbleDB requires Java 8 or Java 11."); + System.err.println("Your Java version: " + System.getProperty("java.version")); + } + + /* + * System.err.println( + * "[INFO] Total available memory: " + (Runtime.getRuntime().maxMemory() / 1000000000) + " GB" + * ); + * System.err.println( + * "[INFO] Total available cores: " + Runtime.getRuntime().availableProcessors() + * ); + */ this.configuration = new SparkConf(); if (this.configuration.get("spark.app.name", "").equals(" +java -jar The examples below assume the jar name is rumbledb.jar. You need to use the actual name of the jar file you downloaded. @@ -21,16 +22,16 @@ for directly running a query from an input file or (with -q) provided directly o It is the default mode. -spark-submit rumbledb.jar run my-query.jq -spark-submit rumbledb.jar run -q '1+1' +java -jar rumbledb.jar run my-query.jq +java -jar rumbledb.jar run -q '1+1' You can specify an output path with -o like so: -spark-submit rumbledb.jar run -q '1+1' -o my-output.txt +java -jar rumbledb.jar run -q '1+1' -o my-output.txt **** serve **** for running as an HTTP server listening on the specified port (-p) and host (-h). -spark-submit rumbledb.jar serve -p 9090 +java -jar rumbledb.jar serve -p 9090 RumbleDB also supports Apache Livy for use in Jupyter notebooks, which may be even more convenient if you are using a cluster. @@ -38,29 +39,14 @@ even more convenient if you are using a cluster. **** repl **** for shell mode. -spark-submit rumbledb.jar repl +java -jar rumbledb.jar repl **** resource use configuration **** For a local use, you can control the number of cores, as well as allocated memory, with: -spark-submit --master local[*] rumbledb.jar repl -spark-submit --master local[*] rumbledb.jar repl -spark-submit --master local[2] rumbledb.jar repl -spark-submit --master local[*] --driver-memory 10G rumbledb.jar repl - -You can use RumbleDB remotely with: -spark-submit --master yarn rumbledb.jar repl - -(Although for clusters provided as a service, --master yarn is often implicit -and unnecessary). - -For remote use (e.g., logged in on the Spark cluster with ssh), you can set the -number of executors, cores and memory, you can use: -spark-submit --executor-cores 3 --executor-memory 5G rumbledb.jar repl -For remote use, you can also use other file system paths such as S3, HDFS, etc: -spark-submit rumbledb.jar run hdfs://server:port/my-query.jq -o hdfs://server:port/my-output.json +java -jar -Xmx10g rumbledb.jar repl More documentation on available CLI parameters is available on https://www.rumbledb.org/