fix: [branch-1.0] backport four bug fixes for 1.0.1

.github/workflows/pr_build_linux.yml

@@ -401,6 +401,7 @@ jobs:
               org.apache.comet.CometCodegenHOFSuite
               org.apache.comet.CometFuzzMathSuite
               org.apache.comet.CometCodegenFuzzSuite
+              org.apache.comet.CometScalaUDFClassLoaderSuite
               org.apache.comet.codegen.CometSpecializedGettersDispatchSuite
               org.apache.comet.CometStringDecodeSuite
               org.apache.comet.CometWidthBucketSuite

.github/workflows/pr_build_macos.yml

@@ -217,6 +217,7 @@ jobs:
               org.apache.comet.CometCodegenHOFSuite
               org.apache.comet.CometFuzzMathSuite
               org.apache.comet.CometCodegenFuzzSuite
+              org.apache.comet.CometScalaUDFClassLoaderSuite
               org.apache.comet.codegen.CometSpecializedGettersDispatchSuite
               org.apache.comet.CometStringDecodeSuite
               org.apache.comet.CometWidthBucketSuite

docs/source/user-guide/latest/compatibility/index.md

@@ -121,10 +121,6 @@ divergence:
   raise raw Arrow errors that bypass `SparkErrorConverter` and surface as `CometNativeException`
   rather than `SparkArithmeticException` with the proper error class and query context
   ([#5072](https://github.com/apache/datafusion-comet/issues/5072)).
-- `next_day` and `make_date` throw at the correct inputs but surface as `CometNativeException`
-  instead of `SparkIllegalArgumentException [ILLEGAL_DAY_OF_WEEK]` /
-  `SparkDateTimeException [DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION]`
-  ([#5073](https://github.com/apache/datafusion-comet/issues/5073)).
 - Spark 4.2 introduced additional ANSI arithmetic overflow behavior differences that Comet does
   not yet track ([#4967](https://github.com/apache/datafusion-comet/issues/4967)).
 

docs/source/user-guide/latest/scala_java_udfs.md

@@ -54,6 +54,7 @@ When a UDF is rejected, the reason surfaces through Comet's standard fallback di
 
 - Non-deterministic expressions referenced from the argument tree (`rand`, `uuid`, `monotonically_increasing_id`) produce per-partition sequences consistent with Spark.
 - `TaskContext.get()` inside the user function returns the driving Spark task's context.
+- The Spark task thread's context ClassLoader is propagated to the thread that runs the user function, so functions defined in jars supplied with `--jars` / `spark.jars` resolve the same way they do under Spark's own execution.
 - The user function must be closure-serializable; the same function that works with Spark's executor execution works here.
 
 ## Known limitations

native/common/src/error.rs

@@ -101,6 +101,12 @@ pub enum SparkError {
     #[error("[DATETIME_OVERFLOW] Datetime arithmetic overflow.")]
     DatetimeOverflow,
 
+    #[error("[ILLEGAL_DAY_OF_WEEK] Illegal input for day of week: {input}.")]
+    IllegalDayOfWeek { input: String },
+
+    #[error("[DATETIME_FIELD_OUT_OF_BOUNDS] {range_message}. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")]
+    DatetimeFieldOutOfBounds { range_message: String },
+
     #[error("[INVALID_ARRAY_INDEX] The index {index_value} is out of bounds. The array has {array_size} elements. Use the SQL function get() to tolerate accessing element at invalid index and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")]
     InvalidArrayIndex { index_value: i32, array_size: i32 },
 
@@ -276,6 +282,8 @@ impl SparkError {
                 "IntervalArithmeticOverflowWithoutSuggestion"
             }
             SparkError::DatetimeOverflow => "DatetimeOverflow",
+            SparkError::IllegalDayOfWeek { .. } => "IllegalDayOfWeek",
+            SparkError::DatetimeFieldOutOfBounds { .. } => "DatetimeFieldOutOfBounds",
             SparkError::InvalidArrayIndex { .. } => "InvalidArrayIndex",
             SparkError::InvalidElementAtIndex { .. } => "InvalidElementAtIndex",
             SparkError::InvalidBitmapPosition { .. } => "InvalidBitmapPosition",
@@ -458,6 +466,16 @@ impl SparkError {
                     "suggestedFunc": suggested_func,
                 })
             }
+            SparkError::IllegalDayOfWeek { input } => {
+                serde_json::json!({
+                    "string": input,
+                })
+            }
+            SparkError::DatetimeFieldOutOfBounds { range_message } => {
+                serde_json::json!({
+                    "rangeMessage": range_message,
+                })
+            }
             SparkError::InvalidFractionOfSecond { value } => {
                 serde_json::json!({
                     "value": value,
@@ -605,11 +623,17 @@ impl SparkError {
             // DateTimeException
             SparkError::InvalidInputInCastToDatetime { .. }
             | SparkError::CannotParseTimestamp { .. }
-            | SparkError::InvalidFractionOfSecond { .. } => "org/apache/spark/SparkDateTimeException",
+            | SparkError::InvalidFractionOfSecond { .. }
+            | SparkError::DatetimeFieldOutOfBounds { .. } => {
+                "org/apache/spark/SparkDateTimeException"
+            }
 
             // IllegalArgumentException
             SparkError::DatatypeCannotOrder { .. }
-            | SparkError::InvalidUtf8String { .. } => "org/apache/spark/SparkIllegalArgumentException",
+            | SparkError::InvalidUtf8String { .. }
+            | SparkError::IllegalDayOfWeek { .. } => {
+                "org/apache/spark/SparkIllegalArgumentException"
+            }
 
             // FileNotFound - will be converted to SparkFileNotFoundException by the shim
             SparkError::FileNotFound { .. } => "org/apache/spark/SparkException",
@@ -693,6 +717,8 @@ impl SparkError {
             // DateTime errors
             SparkError::CannotParseTimestamp { .. } => Some("CANNOT_PARSE_TIMESTAMP"),
             SparkError::InvalidFractionOfSecond { .. } => Some("INVALID_FRACTION_OF_SECOND"),
+            SparkError::IllegalDayOfWeek { .. } => Some("ILLEGAL_DAY_OF_WEEK"),
+            SparkError::DatetimeFieldOutOfBounds { .. } => Some("DATETIME_FIELD_OUT_OF_BOUNDS"),
 
             // String/UTF8 errors
             SparkError::InvalidUtf8String { .. } => Some("INVALID_UTF8_STRING"),

native/common/src/lib.rs

@@ -17,9 +17,11 @@
 
 mod error;
 mod query_context;
+mod schema;
 pub mod tracing;
 mod utils;
 
 pub use error::{decimal_overflow_error, SparkError, SparkErrorWithContext, SparkResult};
 pub use query_context::{create_query_context_map, QueryContext, QueryContextMap};
+pub use schema::{cast_and_stamp_schema, widen_nested_nullability};
 pub use utils::{bytes_to_i128, decode_utf8_spark_lossy};