[AURON #2386] Resolve deserialized expression classes with an explicit class loader

.github/workflows/tpcds-reusable.yml

@@ -64,6 +64,23 @@ on:
         required: false
         type: string
         default: ''
+      assert-no-classcastexception:
+        description: >-
+          Whether to fail the job if the TPC-DS run logs any ClassCastException. Spark retries
+          absorb task-level deserialization failures, so queries can report PASS while throwing
+          hundreds of exceptions; this asserts on the log instead.
+        required: false
+        type: string
+        default: 'false'
+      jar-on-system-classpath:
+        description: >-
+          Whether to also copy the Auron jar into $SPARK_HOME/jars. When true the jar is
+          loaded by the application class loader; when false it reaches the JVM only through
+          spark-submit --jars, i.e. Spark's MutableURLClassLoader. Some class-loading defects
+          only reproduce in the latter configuration.
+        required: false
+        type: string
+        default: 'true'
       queries:
         description: 'Optional list of queries to run'
         required: false
@@ -257,12 +274,18 @@ jobs:
           path: dev/tpcds_1g
 
       - name: Install Auron JAR
+        env:
+          JAR_ON_SYSTEM_CLASSPATH: ${{ inputs.jar-on-system-classpath }}
         run: |
           ls -la
           jar=$(ls -1 auron-${{ inputs.sparkver }}_${{ inputs.scalaver }}*.jar | head -n1)
           [ -n "$jar" ] || { echo "No jar matched: auron-${{ inputs.sparkver }}_${{ inputs.scalaver }}*.jar"; exit 1; }
           echo "AURON_SPARK_JAR=$jar" >> "$GITHUB_ENV"
-          cp "$jar" spark-bin-${{ inputs.sparkver }}_${{ inputs.scalaver }}/jars/
+          if [ "$JAR_ON_SYSTEM_CLASSPATH" = "false" ]; then
+            echo "Auron reaches the JVM only through spark-submit --jars (MutableURLClassLoader)"
+          else
+            cp "$jar" spark-bin-${{ inputs.sparkver }}_${{ inputs.scalaver }}/jars/
+          fi
 
       - name: Setup Java and Maven cache
         uses: actions/setup-java@v5
@@ -380,13 +403,30 @@ jobs:
           SPARK_HOME: spark-bin-${{ inputs.sparkver }}_${{ inputs.scalaver }}
         run: |
           ls -la
+          set -o pipefail
           dev/auron-it/run-it.sh  \
             ${{ inputs.extrasparkconf }} \
             --type tpcds \
             --data-location dev/tpcds_1g \
             --query-filter ${{ matrix.query }} \
             --result-check \
-            --plan-check
+            --plan-check 2>&1 | tee tpcds-run-${{ matrix.query }}.log
+
+      # Task-level deserialization failures are absorbed by Spark's task retries, so the
+      # queries can still report PASS while throwing hundreds of exceptions. Assert on the
+      # log directly, otherwise a regression of AURON #2386 goes unnoticed.
+      - name: Assert no deserialization ClassCastException
+        if: ${{ inputs.assert-no-classcastexception == 'true' }}
+        env:
+          QUERY_LOG: tpcds-run-${{ matrix.query }}.log
+        run: |
+          count=$(grep -c 'ClassCastException' "$QUERY_LOG" || true)
+          if [ "$count" -gt 0 ]; then
+            echo "::error::$count ClassCastException(s) during expression deserialization"
+            grep -m5 'ClassCastException' "$QUERY_LOG" || true
+            exit 1
+          fi
+          echo "No ClassCastException found."
 
       - name: Upload RSS log
         if: ${{ failure() && (inputs.celebornver != '' || inputs.unifflever != '') }}

.github/workflows/tpcds.yml

@@ -112,3 +112,23 @@ jobs:
       scalaver: '2.13'
       hadoop-profile: 'hadoop3'
       sparktests: 'true'
+
+  test-spark-41-jdk17-scala-2-13-with-bloomfilter-optimizer-enabled:
+    name: Test spark-4.1 JDK17 Scala-2.13 with bloomFilter optimizer enabled
+    uses: ./.github/workflows/tpcds-reusable.yml
+    with:
+      sparkver: spark-4.1
+      javaver: '17'
+      scalaver: '2.13'
+      hadoop-profile: 'hadoop3'
+      sparktests: 'true'
+      # Supply Auron only through spark-submit --jars, so it is defined by Spark's
+      # MutableURLClassLoader rather than the application class loader. The runtime
+      # bloom-filter ScalarSubquery deserialization defect (AURON #2386) cannot reproduce
+      # when the jar is also on $SPARK_HOME/jars.
+      jar-on-system-classpath: 'false'
+      assert-no-classcastexception: 'true'
+      extrasparkconf: >-
+        --conf spark.sql.optimizer.runtime.bloomFilter.enabled=true
+        --conf spark.sql.optimizer.runtime.bloomFilter.applicationSideScanSizeThreshold=1B
+        --conf spark.sql.autoBroadcastJoinThreshold=-1

spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala

@@ -18,8 +18,10 @@ package org.apache.spark.sql.auron
 
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
+import java.io.InputStream
 import java.io.ObjectInputStream
 import java.io.ObjectOutputStream
+import java.io.ObjectStreamClass
 
 import scala.collection.mutable
 import scala.jdk.CollectionConverters._
@@ -1505,24 +1507,63 @@ object NativeConverters extends Logging {
     }
   }
 
+  /**
+   * ObjectInputStream that resolves classes against an explicit class loader.
+   *
+   * The default ObjectInputStream.resolveClass resolves each class through
+   * VM.latestUserDefinedLoader(), which selects a loader from the live call stack rather than the
+   * context class loader. During a nested read the most recent user-defined frame is often a
+   * Spark or Scala class, whose loader cannot see Auron classes when Auron is supplied through
+   * spark.jars and therefore loaded by MutableURLClassLoader. The expression graph then resolves
+   * only partially and an un-readResolve'd DefaultSerializationProxy is assigned into
+   * RDD.dependencies_, raising a ClassCastException. Pinning the loader keeps resolution
+   * independent of the call stack. Spark's own JavaDeserializationStream does the same.
+   */
+  private class AuronObjectInputStream(in: InputStream, loader: ClassLoader)
+      extends ObjectInputStream(in) {
+
+    // scalastyle:off classforname
+    private def load(name: String, cl: ClassLoader): Class[_] = Class.forName(name, false, cl)
+    // scalastyle:on classforname
+
+    // resolveProxyClass is deliberately not overridden: the only non-deprecated way to obtain a
+    // proxy Class is Proxy.getProxyClass, and serialized expressions contain no dynamic proxies.
+    override def resolveClass(desc: ObjectStreamClass): Class[_] = {
+      val name = desc.getName
+      try {
+        load(name, loader)
+      } catch {
+        case _: ClassNotFoundException =>
+          // the loader that defined Auron always resolves Auron's own classes even when the
+          // context loader cannot, and its parent chain still covers Spark and Scala classes
+          try {
+            load(name, getClass.getClassLoader)
+          } catch {
+            case _: ClassNotFoundException => super.resolveClass(desc)
+          }
+      }
+    }
+  }
+
   def deserializeExpression[E <: Expression, S <: Serializable](
       serialized: Array[Byte]): (E with Serializable, S) = {
     Utils.tryWithResource(new ByteArrayInputStream(serialized)) { bis =>
-      Utils.tryWithResource(new ObjectInputStream(bis)) { ois =>
-        def read(): (E with Serializable, S) = {
-          val expr = ois.readObject().asInstanceOf[E with Serializable]
-          val payload = ois.readObject().asInstanceOf[S with Serializable]
-          (expr, payload)
-        }
-        // Spark TaskMetrics#externalAccums is not thread-safe
-        val taskContext = TaskContext.get()
-        if (taskContext != null) {
-          taskContext.taskMetrics().synchronized {
+      Utils.tryWithResource(new AuronObjectInputStream(bis, Utils.getContextOrSparkClassLoader)) {
+        ois =>
+          def read(): (E with Serializable, S) = {
+            val expr = ois.readObject().asInstanceOf[E with Serializable]
+            val payload = ois.readObject().asInstanceOf[S with Serializable]
+            (expr, payload)
+          }
+          // Spark TaskMetrics#externalAccums is not thread-safe
+          val taskContext = TaskContext.get()
+          if (taskContext != null) {
+            taskContext.taskMetrics().synchronized {
+              read()
+            }
+          } else {
             read()
           }
-        } else {
-          read()
-        }
       }
     }
   }