feat: Add usePlainNumberFormatForAllCells option to read numeric cells at full precision regardless of cell format

README.md

@@ -69,6 +69,7 @@ val df = spark.read
     .option("treatEmptyValuesAsNulls", "false") // Optional, default: true
     .option("setErrorCellsToFallbackValues", "true") // Optional, default: false, where errors will be converted to null. If true, any ERROR cell values (e.g. #N/A) will be converted to the zero values of the column's data type.
     .option("usePlainNumberFormat", "false") // Optional, default: false, If true, format the cells without rounding and scientific notations
+    .option("usePlainNumberFormatForAllCells", "false") // Optional, default: false, If true, render every non-date numeric cell without rounding and scientific notations, ignoring the cell's number format; date-formatted cells keep their formatted rendering
     .option("inferSchema", "false") // Optional, default: false
     .option("addColorColumns", "true") // Optional, default: false
     .option("timestampFormat", "MM-dd-yyyy HH:mm:ss") // Optional, default: yyyy-mm-dd hh:mm:ss[.fffffffff]
@@ -96,6 +97,7 @@ val df = spark.read.excel(
     treatEmptyValuesAsNulls = false,  // Optional, default: true
     setErrorCellsToFallbackValues = false, // Optional, default: false, where errors will be converted to null. If true, any ERROR cell values (e.g. #N/A) will be converted to the zero values of the column's data type.
     usePlainNumberFormat = false,  // Optional, default: false. If true, format the cells without rounding and scientific notations
+    usePlainNumberFormatForAllCells = false,  // Optional, default: false. If true, render every non-date numeric cell without rounding and scientific notations, ignoring the cell's number format; date-formatted cells keep their formatted rendering
     inferSchema = false,  // Optional, default: false
     addColorColumns = true,  // Optional, default: false
     timestampFormat = "MM-dd-yyyy HH:mm:ss",  // Optional, default: yyyy-mm-dd hh:mm:ss[.fffffffff]

src/main/scala/dev/mauch/spark/excel/DataColumn.scala

@@ -20,6 +20,7 @@ import org.apache.spark.sql.types._
 
 import java.math.BigDecimal
 import java.sql.{Date, Timestamp}
+import java.text.FieldPosition
 import scala.util.{Failure, Success, Try}
 
 trait DataColumn extends PartialFunction[Seq[Cell], Any] {
@@ -36,11 +37,26 @@ class HeaderDataColumn(
   val columnIndex: Int,
   treatEmptyValuesAsNulls: Boolean,
   usePlainNumberFormat: Boolean,
+  usePlainNumberFormatForAllCells: Boolean,
   parseTimestamp: String => Timestamp,
   parseDate: String => Date,
   setErrorCellsToFallbackValues: Boolean
 ) extends DataColumn {
   def name: String = field.name
+
+  /** Whether this (cached-)numeric cell should be rendered at full precision, ignoring its number format. Date cells
+    * keep their formatted rendering, non-finite values keep POI's display rendering.
+    */
+  private def renderPlainNumber(cell: Cell): Boolean =
+    usePlainNumberFormatForAllCells && !DateUtil.isCellDateFormatted(cell) &&
+      java.lang.Double.isFinite(cell.getNumericCellValue)
+
+  /** Same invocation POI's DataFormatter uses for a registered format, so digits match usePlainNumberFormat's. */
+  private def plainNumberString(cell: Cell): String =
+    PlainNumberFormat
+      .format(BigDecimal.valueOf(cell.getNumericCellValue), new StringBuffer(), new FieldPosition(0))
+      .toString
+
   def extractValue(cell: Cell): Any = {
     val cellType = if (cell.getCellType == CellType.FORMULA) cell.getCachedFormulaResultType else cell.getCellType
     if (cellType == CellType.BLANK) {
@@ -65,11 +81,13 @@ class HeaderDataColumn(
         case CellType.FORMULA =>
           cell.getCachedFormulaResultType match {
             case CellType.STRING => Option(cell.getRichStringCellValue).map(_.getString)
+            case CellType.NUMERIC if renderPlainNumber(cell) => Some(plainNumberString(cell))
             case CellType.NUMERIC => Option(cell.getNumericCellValue).map(_.toString)
             case CellType.BLANK => None
             case _ => Some(dataFormatter.formatCellValue(cell))
           }
         case CellType.BLANK => None
+        case CellType.NUMERIC if renderPlainNumber(cell) => Some(plainNumberString(cell))
         case _ => Some(dataFormatter.formatCellValue(cell))
       }
     def parseNumber(string: Option[String]): Option[Double] = string.filter(_.trim.nonEmpty).map(stringToDouble)

src/main/scala/dev/mauch/spark/excel/DefaultSource.scala

@@ -43,6 +43,7 @@ class DefaultSource extends RelationProvider with SchemaRelationProvider with Cr
       treatEmptyValuesAsNulls = parameters.get("treatEmptyValuesAsNulls").fold(false)(_.toBoolean),
       setErrorCellsToFallbackValues = parameters.get("setErrorCellsToFallbackValues").fold(false)(_.toBoolean),
       usePlainNumberFormat = parameters.get("usePlainNumberFormat").fold(false)(_.toBoolean),
+      usePlainNumberFormatForAllCells = parameters.get("usePlainNumberFormatForAllCells").fold(false)(_.toBoolean),
       userSchema = Option(schema),
       inferSheetSchema = parameters.get("inferSchema").fold(false)(_.toBoolean),
       addColorColumns = parameters.get("addColorColumns").fold(false)(_.toBoolean),

src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala

@@ -33,6 +33,7 @@ case class ExcelRelation(
   header: Boolean,
   treatEmptyValuesAsNulls: Boolean,
   usePlainNumberFormat: Boolean,
+  usePlainNumberFormatForAllCells: Boolean,
   inferSheetSchema: Boolean,
   setErrorCellsToFallbackValues: Boolean,
   addColorColumns: Boolean = true,
@@ -189,6 +190,7 @@ case class ExcelRelation(
         cell.getColumnIndex,
         treatEmptyValuesAsNulls,
         usePlainNumberFormat,
+        usePlainNumberFormatForAllCells,
         timestampParser,
         dateParser,
         setErrorCellsToFallbackValues

src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala

@@ -38,8 +38,9 @@ object PlainNumberFormat extends Format {
       // It's an integer, format without decimal point
       toAppendTo.append(stripped.toBigInteger().toString())
     } else {
-      // It's not an integer, format as plain string
-      toAppendTo.append(bd.toPlainString)
+      // It's not an integer, format the stripped value so no trailing zero from
+      // Double.toString's "d.0E-x" mantissa survives (0.0005 must not become "0.00050")
+      toAppendTo.append(stripped.toPlainString)
     }
   }
 

src/main/scala/dev/mauch/spark/excel/package.scala

@@ -78,6 +78,7 @@ package object excel {
       setErrorCellsToFallbackValues: Boolean = false,
       inferSchema: Boolean = false,
       usePlainNumberFormat: Boolean = false,
+      usePlainNumberFormatForAllCells: Boolean = false,
       addColorColumns: Boolean = false,
       dataAddress: String = null,
       timestampFormat: String = null,
@@ -92,6 +93,7 @@ package object excel {
         "treatEmptyValuesAsNulls" -> treatEmptyValuesAsNulls,
         "setErrorCellsToFallbackValues" -> setErrorCellsToFallbackValues,
         "usePlainNumberFormat" -> usePlainNumberFormat,
+        "usePlainNumberFormatForAllCells" -> usePlainNumberFormatForAllCells,
         "inferSchema" -> inferSchema,
         "addColorColumns" -> addColorColumns,
         "dataAddress" -> dataAddress,

src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala

@@ -22,7 +22,7 @@ import org.apache.hadoop.fs.{FileSystem, Path}
 import org.apache.poi.hssf.usermodel.HSSFWorkbookFactory
 import org.apache.poi.openxml4j.util.ZipInputStreamZipEntrySource
 import org.apache.poi.ss.SpreadsheetVersion
-import org.apache.poi.ss.usermodel.{Cell, CellType, DataFormatter, FormulaError, Workbook, WorkbookFactory}
+import org.apache.poi.ss.usermodel.{Cell, CellType, DataFormatter, DateUtil, FormulaError, Workbook, WorkbookFactory}
 import org.apache.poi.ss.util.{AreaReference, CellReference}
 import org.apache.poi.util.IOUtils
 import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory
@@ -52,8 +52,9 @@ object PlainNumberFormat extends Format {
       // It's an integer, format without decimal point
       toAppendTo.append(stripped.toBigInteger().toString())
     } else {
-      // It's not an integer, format as plain string
-      toAppendTo.append(bd.toPlainString)
+      // It's not an integer, format the stripped value so no trailing zero from
+      // Double.toString's "d.0E-x" mantissa survives (0.0005 must not become "0.00050")
+      toAppendTo.append(stripped.toPlainString)
     }
   }
 
@@ -97,14 +98,39 @@ class ExcelHelper private (options: ExcelOptions) {
          */
         case CellType.ERROR => FormulaError.forInt(cell.getErrorCellValue).getString
         case CellType.STRING => cell.getStringCellValue
+        case CellType.NUMERIC if renderPlainNumber(cell) => plainNumberString(cell)
         case CellType.NUMERIC => cell.getNumericCellValue.toString
 
         /* Get what displayed on the cell, for all other cases */
         case _ => dataFormatter.formatCellValue(cell)
       }
+    case CellType.NUMERIC if renderPlainNumber(cell) => plainNumberString(cell)
     case _ => dataFormatter.formatCellValue(cell)
   }
 
+  /** Whether this (cached-)numeric cell should be rendered at full precision, ignoring its number format. Date cells
+    * keep their formatted rendering, non-finite values keep POI's display rendering.
+    */
+  private def renderPlainNumber(cell: Cell): Boolean =
+    options.usePlainNumberFormatForAllCells && !DateUtil.isCellDateFormatted(cell) &&
+      java.lang.Double.isFinite(cell.getNumericCellValue)
+
+  /** Render a numeric cell at full precision, ignoring its number format. Invokes [[PlainNumberFormat]] with the same
+    * argument POI's DataFormatter passes to a registered format, so the rendering is identical to
+    * usePlainNumberFormat's for General-format cells.
+    */
+  private def plainNumberString(cell: Cell): String =
+    PlainNumberFormat
+      .format(BigDecimal.valueOf(cell.getNumericCellValue), new StringBuffer(), new FieldPosition(0))
+      .toString
+
+  /** Header-cell rendering, honoring usePlainNumberFormatForAllCells so a numeric header is named consistently with its
+    * data cells.
+    */
+  private def headerCellString(cell: Cell): String =
+    if (cell.getCellType == CellType.NUMERIC && renderPlainNumber(cell)) plainNumberString(cell)
+    else dataFormatter.formatCellValue(cell)
+
   /** Get workbook
     *
     * @param conf
@@ -230,14 +256,14 @@ class ExcelHelper private (options: ExcelOptions) {
 
     val dataColumns =
       if (options.header) {
-        val headerNames = firstRow.map(dataFormatter.formatCellValue)
+        val headerNames = firstRow.map(headerCellString)
         val duplicates = {
           val nonNullHeaderNames = headerNames.filter(_ != null)
           nonNullHeaderNames.groupBy(identity).filter(_._2.size > 1).keySet
         }
 
         firstRow.zipWithIndex.map { case (cell, index) =>
-          val value = dataFormatter.formatCellValue(cell)
+          val value = headerCellString(cell)
           val cellType = cell.getCellType
           if (
             cellType == CellType.ERROR || cellType == CellType.BLANK ||

src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala

@@ -106,6 +106,10 @@ trait ExcelOptionsTrait extends Serializable {
   /* If true, format the cells without rounding and scientific notations */
   val usePlainNumberFormat = getBool("usePlainNumberFormat", default = false)
 
+  /* If true, render every non-date numeric cell without rounding and scientific notations,
+     ignoring the cell's number format; date-formatted cells keep their formatted rendering */
+  val usePlainNumberFormatForAllCells = getBool("usePlainNumberFormatForAllCells", default = false)
+
   /* If true, keep undefined (Excel) rows */
   val keepUndefinedRows = getBool("keepUndefinedRows", default = false)
 

src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala

@@ -81,6 +81,36 @@ object PlainNumberReadSuite {
   )
 
   val issue747Data: util.List[Row] = List(Row("9024523", "902"), Row("1020001", "102"), Row("9764342", "L906")).asJava
+
+  // Predefined data for usePlainNumberFormatForAllCells tests
+  val allCellsSchema = StructType(
+    List(
+      StructField("formatted_number", StringType, true),
+      StructField("text_and_number", StringType, true),
+      StructField("general_number", StringType, true),
+      StructField("date_col", StringType, true)
+    )
+  )
+
+  val allCellsPlainData: util.List[Row] = List(
+    Row("84.789", "138,56", "123456789012", "01.07.2026"),
+    Row("3.2886", "5414874004074", "-0.12345678901", "15.01.2023"),
+    Row("0.0005", "0,0005", "123456789012", "31.12.2024") // 0.0005 must not gain a trailing zero; C4 is =123456789012*1
+  ).asJava
+
+  val allCellsDisplayData: util.List[Row] = List(
+    Row("84.79", "138,56", "1.23457E+11", "01.07.2026"), // explicit formats round, General goes scientific
+    Row("3.29", "5414874004074", "-0.123456789", "15.01.2023"),
+    Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") // cached formula results read as Double.toString
+  ).asJava
+
+  // usePlainNumberFormat=true alone: General cells render plain, explicit formats still round,
+  // cached formula results stay Double.toString -- the gap usePlainNumberFormatForAllCells closes
+  val allCellsGeneralPlainData: util.List[Row] = List(
+    Row("84.79", "138,56", "123456789012", "01.07.2026"),
+    Row("3.29", "5414874004074", "-0.12345678901", "15.01.2023"),
+    Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024")
+  ).asJava
 }
 
 class PlainNumberReadSuite extends AnyFunSpec with DataFrameSuiteBase with Matchers {
@@ -149,5 +179,35 @@ class PlainNumberReadSuite extends AnyFunSpec with DataFrameSuiteBase with Match
       // Verify both dataframes should be equal
       assertDataFrameEquals(dfWithPlain, dfWithoutPlain)
     }
+
+    // deliberately goes through the spark.read.excel(...) DSL so the option-key plumbing is exercised too
+    def readAllCellsFixture(usePlainNumberFormat: Boolean, usePlainNumberFormatForAllCells: Boolean): DataFrame = {
+      val url = getClass.getResource("/spreadsheets/plain_number_all_cells.xlsx")
+      spark.read
+        .excel(
+          header = true,
+          usePlainNumberFormat = usePlainNumberFormat,
+          usePlainNumberFormatForAllCells = usePlainNumberFormatForAllCells
+        )
+        .load(url.getPath)
+    }
+
+    it("should render explicitly formatted cells plain when usePlainNumberFormatForAllCells=true") {
+      val df = readAllCellsFixture(usePlainNumberFormat = false, usePlainNumberFormatForAllCells = true)
+      val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema)
+      assertDataFrameEquals(expected, df)
+    }
+
+    it("should keep display rendering when usePlainNumberFormatForAllCells=false") {
+      val df = readAllCellsFixture(usePlainNumberFormat = false, usePlainNumberFormatForAllCells = false)
+      val expected = spark.createDataFrame(allCellsDisplayData, allCellsSchema)
+      assertDataFrameEquals(expected, df)
+    }
+
+    it("should still round explicitly formatted cells when only usePlainNumberFormat=true") {
+      val df = readAllCellsFixture(usePlainNumberFormat = true, usePlainNumberFormatForAllCells = false)
+      val expected = spark.createDataFrame(allCellsGeneralPlainData, allCellsSchema)
+      assertDataFrameEquals(expected, df)
+    }
   }
 }

src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala

@@ -77,6 +77,36 @@ object PlainNumberReadSuite {
   )
 
   val issue747Data: util.List[Row] = List(Row("9024523", "902"), Row("1020001", "102"), Row("9764342", "L906")).asJava
+
+  // Predefined data for usePlainNumberFormatForAllCells tests
+  val allCellsSchema = StructType(
+    List(
+      StructField("formatted_number", StringType, true),
+      StructField("text_and_number", StringType, true),
+      StructField("general_number", StringType, true),
+      StructField("date_col", StringType, true)
+    )
+  )
+
+  val allCellsPlainData: util.List[Row] = List(
+    Row("84.789", "138,56", "123456789012", "01.07.2026"),
+    Row("3.2886", "5414874004074", "-0.12345678901", "15.01.2023"),
+    Row("0.0005", "0,0005", "123456789012", "31.12.2024") // 0.0005 must not gain a trailing zero; C4 is =123456789012*1
+  ).asJava
+
+  val allCellsDisplayData: util.List[Row] = List(
+    Row("84.79", "138,56", "1.23457E+11", "01.07.2026"), // explicit formats round, General goes scientific
+    Row("3.29", "5414874004074", "-0.123456789", "15.01.2023"),
+    Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") // cached formula results read as Double.toString
+  ).asJava
+
+  // usePlainNumberFormat=true alone: General cells render plain, explicit formats still round,
+  // cached formula results stay Double.toString -- the gap usePlainNumberFormatForAllCells closes
+  val allCellsGeneralPlainData: util.List[Row] = List(
+    Row("84.79", "138,56", "123456789012", "01.07.2026"),
+    Row("3.29", "5414874004074", "-0.12345678901", "15.01.2023"),
+    Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024")
+  ).asJava
 }
 
 class PlainNumberReadSuite extends AnyFunSuite with DataFrameSuiteBase with ExcelTestingUtilities {
@@ -181,4 +211,60 @@ class PlainNumberReadSuite extends AnyFunSuite with DataFrameSuiteBase with Exce
     // Verify both dataframes should be equal
     assertDataFrameEquals(dfWithPlain, dfWithoutPlain)
   }
+
+  test("explicitly formatted cells render plain when usePlainNumberFormatForAllCells=true") {
+    val df = readFromResources(
+      spark,
+      path = "plain_number_all_cells.xlsx",
+      options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false)
+    )
+    val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema)
+    assertDataFrameEquals(expected, df)
+  }
+
+  test("explicitly formatted cells render plain when usePlainNumberFormatForAllCells=true and maxRowsInMemory") {
+    val df = readFromResources(
+      spark,
+      path = "plain_number_all_cells.xlsx",
+      options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false, "maxRowsInMemory" -> 1)
+    )
+    val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema)
+    assertDataFrameEquals(expected, df)
+  }
+
+  test("explicitly formatted cells keep display rendering when usePlainNumberFormatForAllCells=false") {
+    val df = readFromResources(
+      spark,
+      path = "plain_number_all_cells.xlsx",
+      options = Map("usePlainNumberFormatForAllCells" -> false, "inferSchema" -> false)
+    )
+    val expected = spark.createDataFrame(allCellsDisplayData, allCellsSchema)
+    assertDataFrameEquals(expected, df)
+  }
+
+  test("explicitly formatted cells still round when only usePlainNumberFormat=true") {
+    val df = readFromResources(
+      spark,
+      path = "plain_number_all_cells.xlsx",
+      options = Map("usePlainNumberFormat" -> true, "inferSchema" -> false)
+    )
+    val expected = spark.createDataFrame(allCellsGeneralPlainData, allCellsSchema)
+    assertDataFrameEquals(expected, df)
+  }
+
+  test("numeric header cells are named consistently with their data cells") {
+    val plain = readFromResources(
+      spark,
+      path = "plain_number_numeric_header.xlsx",
+      options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false)
+    )
+    assert(plain.schema.fieldNames.toSeq == Seq("123456789012", "name"))
+
+    val display = readFromResources(
+      spark,
+      path = "plain_number_numeric_header.xlsx",
+      options = Map("usePlainNumberFormatForAllCells" -> false, "inferSchema" -> false)
+    )
+    assert(display.schema.fieldNames.toSeq == Seq("1.23457E+11", "name"))
+  }
 }