Spark 4.2

pom.xml

@@ -387,38 +387,44 @@
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>4.2.0-preview5</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>4.2.0-preview5</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>3.4.2</version>
+          <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-mllib_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>4.2.0-preview5</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-hive_2.13</artifactId>
-            <version>4.0.1</version>
+            <version>4.2.0-preview5</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-avro_2.13</artifactId>
+            <version>4.2.0-preview5</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-aws</artifactId>
-            <version>3.3.6</version>
+            <version>3.4.3</version>
             <scope>provided</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-avro_2.13</artifactId>
-            <version>4.0.1</version>
-        </dependency>
         <dependency>
             <groupId>org.antlr</groupId>
             <artifactId>antlr4-runtime</artifactId>
@@ -484,7 +490,7 @@
         <dependency>
             <groupId>io.delta</groupId>
             <artifactId>delta-spark_2.13</artifactId>
-            <version>4.0.0</version>
+            <version>4.2.0</version>
         </dependency>
         <dependency>
             <groupId>org.apache.iceberg</groupId>
@@ -514,6 +520,12 @@
             <artifactId>mongo-spark-connector_2.13</artifactId>
             <version>10.5.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>2.24.3</version>
+            <scope>compile</scope>
+        </dependency>
         <dependency>
             <groupId>com.ibm.icu</groupId>
             <artifactId>icu4j</artifactId>

src/main/java/org/rumbledb/types/DerivedAtomicItemType.java

@@ -4,7 +4,7 @@
 import java.util.List;
 import java.util.Set;
 
-import org.apache.commons.collections.ListUtils;
+import org.apache.commons.collections4.ListUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.config.RumbleConfiguration;
 import org.rumbledb.context.DynamicContext;
@@ -191,7 +191,7 @@
         return this.enumeration == null ? this.baseType.getEnumerationFacet() : this.enumeration;
     }
 
     @SuppressWarnings("unchecked")
     @Override
     public List<String> getConstraintsFacet() {
         if (!this.getAllowedFacets().contains(ConstrainingFacetTypes.CONSTRAINTS)) {

src/main/java/org/rumbledb/types/ObjectItemType.java

@@ -11,7 +11,7 @@
 import java.util.Map;
 import java.util.Set;
 
-import org.apache.commons.collections.ListUtils;
+import org.apache.commons.collections4.ListUtils;
 import org.rumbledb.api.Item;
 import org.rumbledb.config.RumbleConfiguration;
 import org.rumbledb.context.DynamicContext;
@@ -186,7 +186,7 @@
         return this.enumeration != null || this.isPrimitive() ? this.enumeration : this.baseType.getEnumerationFacet();
     }
 
     @SuppressWarnings("unchecked")
     @Override
     public List<String> getConstraintsFacet() {
         return this.isPrimitive()

src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-MinMaxScaler.jq

@@ -1,4 +1,4 @@
-(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "scaled" : { } }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "scaled" : [ 0.2, 0.20703124999999992 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "scaled" : [ 0.4, 0.4140625000000001 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "scaled" : [ 0.6000000000000001, 0.6210937500000003 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "scaled" : [ 0.8, 0.79296875 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "scaled" : [ 1, 1 ] })" :)
+(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "scaled" : [ 0, 0 ] }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "scaled" : [ 0.2, 0.20703124999999992 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "scaled" : [ 0.4, 0.4140625000000001 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "scaled" : [ 0.6000000000000001, 0.6210937500000003 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "scaled" : [ 0.8, 0.79296875 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "scaled" : [ 1, 1 ] })" :)
 let $data := annotate(
     json-lines("../../../../queries/rumbleML/sample-ml-data-flat.json"),
     { "label": "integer", "binaryLabel": "integer", "name": "string", "age": "double", "weight": "double", "booleanCol": "boolean", "nullCol": "null", "stringCol": "string", "stringArrayCol": ["string"], "intArrayCol": ["integer"],  "doubleArrayCol": ["double"],  "doubleArrayArrayCol": [["double"]] }

src/test/resources/test_files/RumbleML/RumbleML/EstimatorTests/MLEstimator-PCA.jq

@@ -1,4 +1,4 @@
-(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "pca" : { } }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "pca" : [ 0.2, 0.20703124999999992 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "pca" : [ 0.4, 0.4140625000000001 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "pca" : [ 0.6000000000000001, 0.6210937500000003 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "pca" : [ 0.8, 0.79296875 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "pca" : [ 1, 1 ] })" :)
+(:JIQS: ShouldRun; Output="({ "label" : 0, "name" : "a", "age" : 20, "weight" : 50, "pca" : [ 0, 0 ] }, { "label" : 1, "name" : "b", "age" : 21, "weight" : 55.3, "pca" : [ 0.2, 0.20703124999999992 ] }, { "label" : 2, "name" : "c", "age" : 22, "weight" : 60.6, "pca" : [ 0.4, 0.4140625000000001 ] }, { "label" : 3, "name" : "d", "age" : 23, "weight" : 65.9, "pca" : [ 0.6000000000000001, 0.6210937500000003 ] }, { "label" : 4, "name" : "e", "age" : 24, "weight" : 70.3, "pca" : [ 0.8, 0.79296875 ] }, { "label" : 5, "name" : "f", "age" : 25, "weight" : 75.6, "pca" : [ 1, 1 ] })" :)
 let $data := annotate(
     json-lines("../../../../queries/rumbleML/sample-ml-data-flat.json"),
     { "label": "integer", "binaryLabel": "integer", "name": "string", "age": "double", "weight": "double", "booleanCol": "boolean", "nullCol": "null", "stringCol": "string", "stringArrayCol": ["string"], "intArrayCol": ["integer"],  "doubleArrayCol": ["double"],  "doubleArrayArrayCol": [["double"]] }