diff --git a/pom.xml b/pom.xml
index 00d22c757e..02ca49e270 100644
--- a/pom.xml
+++ b/pom.xml
@@ -302,7 +302,6 @@
org.apache.hadoop
hadoop-aws
3.3.6
- provided
org.apache.spark
diff --git a/src/main/java/org/rumbledb/cli/JsoniqQueryExecutor.java b/src/main/java/org/rumbledb/cli/JsoniqQueryExecutor.java
index 3ee24bafef..ecbf476711 100644
--- a/src/main/java/org/rumbledb/cli/JsoniqQueryExecutor.java
+++ b/src/main/java/org/rumbledb/cli/JsoniqQueryExecutor.java
@@ -28,6 +28,7 @@
import org.rumbledb.exceptions.ExceptionMetadata;
import org.rumbledb.optimizations.Profiler;
import org.rumbledb.runtime.functions.input.FileSystemUtil;
+import org.slf4j.Logger;
import org.rumbledb.serialization.Serializer;
import java.io.IOException;
@@ -37,11 +38,13 @@
import java.util.List;
import java.util.stream.Collectors;
+import org.apache.spark.internal.Logging;
-public class JsoniqQueryExecutor {
+public class JsoniqQueryExecutor implements Logging {
private RumbleRuntimeConfiguration configuration;
public JsoniqQueryExecutor(RumbleRuntimeConfiguration configuration) {
+ initializeLogIfNecessary(true, true);
this.configuration = configuration;
}
@@ -178,4 +181,16 @@ public long runInteractive(String query, List- resultList) throws IOExcepti
return sequence.populateList(resultList, this.configuration.getResultSizeCap());
}
+ @Override
+ public Logger org$apache$spark$internal$Logging$$log_() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public void org$apache$spark$internal$Logging$$log__$eq(Logger x$1) {
+ // TODO Auto-generated method stub
+
+ }
+
}
diff --git a/src/main/java/org/rumbledb/cli/Main.java b/src/main/java/org/rumbledb/cli/Main.java
index 98504198d3..da6fa0f68b 100644
--- a/src/main/java/org/rumbledb/cli/Main.java
+++ b/src/main/java/org/rumbledb/cli/Main.java
@@ -112,7 +112,13 @@ private static void handleException(Throwable ex, boolean showErrorInfo) {
"⚠️ Java went out of memory."
);
System.err.println(
- "If running locally, try adding --driver-memory 10G (or any quantity you need) between spark-submit and the RumbleDB jar in the command line to see if it fixes the problem. If running on a cluster, --executor-memory is the way to go."
+ "If running locally with java -jar, try adding --Xmx10g (or any quantity you need) before the RumbleDB jar in the command line to see if it fixes the problem."
+ );
+ System.err.println(
+ "If running locally with spark-submit, try adding --driver-memory 10G (or any quantity you need) between spark-submit and the RumbleDB jar in the command line to see if it fixes the problem."
+ );
+ System.err.println(
+ "If running on a cluster, --executor-memory should be used instead."
);
if (showErrorInfo) {
ex.printStackTrace();
diff --git a/src/main/java/sparksoniq/spark/SparkSessionManager.java b/src/main/java/sparksoniq/spark/SparkSessionManager.java
index f50628563b..d6b54f50e6 100644
--- a/src/main/java/sparksoniq/spark/SparkSessionManager.java
+++ b/src/main/java/sparksoniq/spark/SparkSessionManager.java
@@ -159,6 +159,26 @@ public SparkSession getOrCreateSession() {
private void setDefaultConfiguration() {
try {
+ if (System.getProperty("hadoop.home.dir") == null) {
+ System.err.println(
+ "[WARNING] The hadoop home directory was not set. Setting to \"/\"."
+ );
+ System.setProperty("hadoop.home.dir", "/");
+ }
+ String javaVersion = System.getProperty("java.version");
+ if (!javaVersion.startsWith("1.8") && !javaVersion.startsWith("11.")) {
+ System.err.println("[Error] RumbleDB requires Java 8 or Java 11.");
+ System.err.println("Your Java version: " + System.getProperty("java.version"));
+ }
+
+ /*
+ * System.err.println(
+ * "[INFO] Total available memory: " + (Runtime.getRuntime().maxMemory() / 1000000000) + " GB"
+ * );
+ * System.err.println(
+ * "[INFO] Total available cores: " + Runtime.getRuntime().availableProcessors()
+ * );
+ */
this.configuration = new SparkConf();
if (this.configuration.get("spark.app.name", "").equals("
+java -jar
The examples below assume the jar name is rumbledb.jar. You need to use
the actual name of the jar file you downloaded.
@@ -21,16 +22,16 @@ for directly running a query from an input file or (with -q) provided directly o
It is the default mode.
-spark-submit rumbledb.jar run my-query.jq
-spark-submit rumbledb.jar run -q '1+1'
+java -jar rumbledb.jar run my-query.jq
+java -jar rumbledb.jar run -q '1+1'
You can specify an output path with -o like so:
-spark-submit rumbledb.jar run -q '1+1' -o my-output.txt
+java -jar rumbledb.jar run -q '1+1' -o my-output.txt
**** serve ****
for running as an HTTP server listening on the specified port (-p) and host (-h).
-spark-submit rumbledb.jar serve -p 9090
+java -jar rumbledb.jar serve -p 9090
RumbleDB also supports Apache Livy for use in Jupyter notebooks, which may be
even more convenient if you are using a cluster.
@@ -38,29 +39,14 @@ even more convenient if you are using a cluster.
**** repl ****
for shell mode.
-spark-submit rumbledb.jar repl
+java -jar rumbledb.jar repl
**** resource use configuration ****
For a local use, you can control the number of cores, as well as allocated
memory, with:
-spark-submit --master local[*] rumbledb.jar repl
-spark-submit --master local[*] rumbledb.jar repl
-spark-submit --master local[2] rumbledb.jar repl
-spark-submit --master local[*] --driver-memory 10G rumbledb.jar repl
-
-You can use RumbleDB remotely with:
-spark-submit --master yarn rumbledb.jar repl
-
-(Although for clusters provided as a service, --master yarn is often implicit
-and unnecessary).
-
-For remote use (e.g., logged in on the Spark cluster with ssh), you can set the
-number of executors, cores and memory, you can use:
-spark-submit --executor-cores 3 --executor-memory 5G rumbledb.jar repl
-For remote use, you can also use other file system paths such as S3, HDFS, etc:
-spark-submit rumbledb.jar run hdfs://server:port/my-query.jq -o hdfs://server:port/my-output.json
+java -jar -Xmx10g rumbledb.jar repl
More documentation on available CLI parameters is available on https://www.rumbledb.org/