From f7641bbe5c25944cf3954a22c3f35f27d8ba4fef Mon Sep 17 00:00:00 2001 From: sshpuntoff Date: Mon, 18 May 2026 10:44:48 -0400 Subject: [PATCH] fix: propagate reader options to Hadoop conf in driver paths The V2 scan path already builds its Hadoop Configuration with `newHadoopConfWithOptions(...)`, but driver-side schema inference (`ExcelTable.inferSchema`) and the V1 read/write relations in `DefaultSource` still used plain `newHadoopConf()`. As a result, reader options such as bucket-scoped `fs.s3a.*` credentials passed via `DataFrameReader.options(...)` were ignored on the driver, causing Excel reads to fail during inference before executors got a chance to apply the scoped options. Use `newHadoopConfWithOptions` in all three sites so every Hadoop `Configuration` created by spark-excel sees the option map. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/main/scala/dev/mauch/spark/excel/DefaultSource.scala | 4 ++-- src/main/scala/dev/mauch/spark/excel/v2/ExcelTable.scala | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala b/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala index facbb542..da073623 100644 --- a/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala +++ b/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala @@ -35,7 +35,7 @@ class DefaultSource extends RelationProvider with SchemaRelationProvider with Cr parameters: Map[String, String], schema: StructType ): ExcelRelation = { - val conf = sqlContext.sparkSession.sessionState.newHadoopConf() + val conf = sqlContext.sparkSession.sessionState.newHadoopConfWithOptions(parameters) val wbReader = WorkbookReader(parameters, conf) val dataLocator = DataLocator(parameters) ExcelRelation( @@ -63,7 +63,7 @@ class DefaultSource extends RelationProvider with SchemaRelationProvider with Cr val path = checkParameter(parameters, "path") val header = checkParameter(parameters, "header").toBoolean val filesystemPath = new Path(path) - val conf = sqlContext.sparkSession.sessionState.newHadoopConf() + val conf = sqlContext.sparkSession.sessionState.newHadoopConfWithOptions(parameters) val fs = filesystemPath.getFileSystem(conf) new ExcelFileSaver( fs, diff --git a/src/main/scala/dev/mauch/spark/excel/v2/ExcelTable.scala b/src/main/scala/dev/mauch/spark/excel/v2/ExcelTable.scala index 9e50f0c1..6d1c8aa4 100644 --- a/src/main/scala/dev/mauch/spark/excel/v2/ExcelTable.scala +++ b/src/main/scala/dev/mauch/spark/excel/v2/ExcelTable.scala @@ -72,7 +72,9 @@ case class ExcelTable( /* Actual doing schema inferring */ private def infer(sparkSession: SparkSession, inputPaths: Seq[FileStatus], options: ExcelOptions): StructType = { val excelHelper = ExcelHelper(options) - val conf = sparkSession.sessionState.newHadoopConf() + /* Hadoop Configurations are case sensitive. */ + val caseSensitiveMap = map.asCaseSensitiveMap.asScala.toMap + val conf = sparkSession.sessionState.newHadoopConfWithOptions(caseSensitiveMap) /** Sampling ratio on file level (not row level as in CSV) */ val paths = {