diff --git a/README.md b/README.md index 6652a86d..7b27d380 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ val df = spark.read .option("treatEmptyValuesAsNulls", "false") // Optional, default: true .option("setErrorCellsToFallbackValues", "true") // Optional, default: false, where errors will be converted to null. If true, any ERROR cell values (e.g. #N/A) will be converted to the zero values of the column's data type. .option("usePlainNumberFormat", "false") // Optional, default: false, If true, format the cells without rounding and scientific notations + .option("usePlainNumberFormatForAllCells", "false") // Optional, default: false, If true, render every non-date numeric cell without rounding and scientific notations, ignoring the cell's number format; date-formatted cells keep their formatted rendering .option("inferSchema", "false") // Optional, default: false .option("addColorColumns", "true") // Optional, default: false .option("timestampFormat", "MM-dd-yyyy HH:mm:ss") // Optional, default: yyyy-mm-dd hh:mm:ss[.fffffffff] @@ -96,6 +97,7 @@ val df = spark.read.excel( treatEmptyValuesAsNulls = false, // Optional, default: true setErrorCellsToFallbackValues = false, // Optional, default: false, where errors will be converted to null. If true, any ERROR cell values (e.g. #N/A) will be converted to the zero values of the column's data type. usePlainNumberFormat = false, // Optional, default: false. If true, format the cells without rounding and scientific notations + usePlainNumberFormatForAllCells = false, // Optional, default: false. If true, render every non-date numeric cell without rounding and scientific notations, ignoring the cell's number format; date-formatted cells keep their formatted rendering inferSchema = false, // Optional, default: false addColorColumns = true, // Optional, default: false timestampFormat = "MM-dd-yyyy HH:mm:ss", // Optional, default: yyyy-mm-dd hh:mm:ss[.fffffffff] diff --git a/src/main/scala/dev/mauch/spark/excel/DataColumn.scala b/src/main/scala/dev/mauch/spark/excel/DataColumn.scala index e593fcad..fe8865db 100644 --- a/src/main/scala/dev/mauch/spark/excel/DataColumn.scala +++ b/src/main/scala/dev/mauch/spark/excel/DataColumn.scala @@ -20,6 +20,7 @@ import org.apache.spark.sql.types._ import java.math.BigDecimal import java.sql.{Date, Timestamp} +import java.text.FieldPosition import scala.util.{Failure, Success, Try} trait DataColumn extends PartialFunction[Seq[Cell], Any] { @@ -36,11 +37,26 @@ class HeaderDataColumn( val columnIndex: Int, treatEmptyValuesAsNulls: Boolean, usePlainNumberFormat: Boolean, + usePlainNumberFormatForAllCells: Boolean, parseTimestamp: String => Timestamp, parseDate: String => Date, setErrorCellsToFallbackValues: Boolean ) extends DataColumn { def name: String = field.name + + /** Whether this (cached-)numeric cell should be rendered at full precision, ignoring its number format. Date cells + * keep their formatted rendering, non-finite values keep POI's display rendering. + */ + private def renderPlainNumber(cell: Cell): Boolean = + usePlainNumberFormatForAllCells && !DateUtil.isCellDateFormatted(cell) && + java.lang.Double.isFinite(cell.getNumericCellValue) + + /** Same invocation POI's DataFormatter uses for a registered format, so digits match usePlainNumberFormat's. */ + private def plainNumberString(cell: Cell): String = + PlainNumberFormat + .format(BigDecimal.valueOf(cell.getNumericCellValue), new StringBuffer(), new FieldPosition(0)) + .toString + def extractValue(cell: Cell): Any = { val cellType = if (cell.getCellType == CellType.FORMULA) cell.getCachedFormulaResultType else cell.getCellType if (cellType == CellType.BLANK) { @@ -65,11 +81,13 @@ class HeaderDataColumn( case CellType.FORMULA => cell.getCachedFormulaResultType match { case CellType.STRING => Option(cell.getRichStringCellValue).map(_.getString) + case CellType.NUMERIC if renderPlainNumber(cell) => Some(plainNumberString(cell)) case CellType.NUMERIC => Option(cell.getNumericCellValue).map(_.toString) case CellType.BLANK => None case _ => Some(dataFormatter.formatCellValue(cell)) } case CellType.BLANK => None + case CellType.NUMERIC if renderPlainNumber(cell) => Some(plainNumberString(cell)) case _ => Some(dataFormatter.formatCellValue(cell)) } def parseNumber(string: Option[String]): Option[Double] = string.filter(_.trim.nonEmpty).map(stringToDouble) diff --git a/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala b/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala index facbb542..4a15fa0f 100644 --- a/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala +++ b/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala @@ -43,6 +43,7 @@ class DefaultSource extends RelationProvider with SchemaRelationProvider with Cr treatEmptyValuesAsNulls = parameters.get("treatEmptyValuesAsNulls").fold(false)(_.toBoolean), setErrorCellsToFallbackValues = parameters.get("setErrorCellsToFallbackValues").fold(false)(_.toBoolean), usePlainNumberFormat = parameters.get("usePlainNumberFormat").fold(false)(_.toBoolean), + usePlainNumberFormatForAllCells = parameters.get("usePlainNumberFormatForAllCells").fold(false)(_.toBoolean), userSchema = Option(schema), inferSheetSchema = parameters.get("inferSchema").fold(false)(_.toBoolean), addColorColumns = parameters.get("addColorColumns").fold(false)(_.toBoolean), diff --git a/src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala b/src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala index 6d1708e5..bdc80b48 100644 --- a/src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala +++ b/src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala @@ -33,6 +33,7 @@ case class ExcelRelation( header: Boolean, treatEmptyValuesAsNulls: Boolean, usePlainNumberFormat: Boolean, + usePlainNumberFormatForAllCells: Boolean, inferSheetSchema: Boolean, setErrorCellsToFallbackValues: Boolean, addColorColumns: Boolean = true, @@ -189,6 +190,7 @@ case class ExcelRelation( cell.getColumnIndex, treatEmptyValuesAsNulls, usePlainNumberFormat, + usePlainNumberFormatForAllCells, timestampParser, dateParser, setErrorCellsToFallbackValues diff --git a/src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala b/src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala index d6f7f144..57a4fbcf 100644 --- a/src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala +++ b/src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala @@ -38,8 +38,9 @@ object PlainNumberFormat extends Format { // It's an integer, format without decimal point toAppendTo.append(stripped.toBigInteger().toString()) } else { - // It's not an integer, format as plain string - toAppendTo.append(bd.toPlainString) + // It's not an integer, format the stripped value so no trailing zero from + // Double.toString's "d.0E-x" mantissa survives (0.0005 must not become "0.00050") + toAppendTo.append(stripped.toPlainString) } } diff --git a/src/main/scala/dev/mauch/spark/excel/package.scala b/src/main/scala/dev/mauch/spark/excel/package.scala index ce9709a2..6d2e3f7d 100644 --- a/src/main/scala/dev/mauch/spark/excel/package.scala +++ b/src/main/scala/dev/mauch/spark/excel/package.scala @@ -78,6 +78,7 @@ package object excel { setErrorCellsToFallbackValues: Boolean = false, inferSchema: Boolean = false, usePlainNumberFormat: Boolean = false, + usePlainNumberFormatForAllCells: Boolean = false, addColorColumns: Boolean = false, dataAddress: String = null, timestampFormat: String = null, @@ -92,6 +93,7 @@ package object excel { "treatEmptyValuesAsNulls" -> treatEmptyValuesAsNulls, "setErrorCellsToFallbackValues" -> setErrorCellsToFallbackValues, "usePlainNumberFormat" -> usePlainNumberFormat, + "usePlainNumberFormatForAllCells" -> usePlainNumberFormatForAllCells, "inferSchema" -> inferSchema, "addColorColumns" -> addColorColumns, "dataAddress" -> dataAddress, diff --git a/src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala b/src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala index e0e84e10..efda56b0 100644 --- a/src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala +++ b/src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala @@ -22,7 +22,7 @@ import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.poi.hssf.usermodel.HSSFWorkbookFactory import org.apache.poi.openxml4j.util.ZipInputStreamZipEntrySource import org.apache.poi.ss.SpreadsheetVersion -import org.apache.poi.ss.usermodel.{Cell, CellType, DataFormatter, FormulaError, Workbook, WorkbookFactory} +import org.apache.poi.ss.usermodel.{Cell, CellType, DataFormatter, DateUtil, FormulaError, Workbook, WorkbookFactory} import org.apache.poi.ss.util.{AreaReference, CellReference} import org.apache.poi.util.IOUtils import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory @@ -52,8 +52,9 @@ object PlainNumberFormat extends Format { // It's an integer, format without decimal point toAppendTo.append(stripped.toBigInteger().toString()) } else { - // It's not an integer, format as plain string - toAppendTo.append(bd.toPlainString) + // It's not an integer, format the stripped value so no trailing zero from + // Double.toString's "d.0E-x" mantissa survives (0.0005 must not become "0.00050") + toAppendTo.append(stripped.toPlainString) } } @@ -97,14 +98,39 @@ class ExcelHelper private (options: ExcelOptions) { */ case CellType.ERROR => FormulaError.forInt(cell.getErrorCellValue).getString case CellType.STRING => cell.getStringCellValue + case CellType.NUMERIC if renderPlainNumber(cell) => plainNumberString(cell) case CellType.NUMERIC => cell.getNumericCellValue.toString /* Get what displayed on the cell, for all other cases */ case _ => dataFormatter.formatCellValue(cell) } + case CellType.NUMERIC if renderPlainNumber(cell) => plainNumberString(cell) case _ => dataFormatter.formatCellValue(cell) } + /** Whether this (cached-)numeric cell should be rendered at full precision, ignoring its number format. Date cells + * keep their formatted rendering, non-finite values keep POI's display rendering. + */ + private def renderPlainNumber(cell: Cell): Boolean = + options.usePlainNumberFormatForAllCells && !DateUtil.isCellDateFormatted(cell) && + java.lang.Double.isFinite(cell.getNumericCellValue) + + /** Render a numeric cell at full precision, ignoring its number format. Invokes [[PlainNumberFormat]] with the same + * argument POI's DataFormatter passes to a registered format, so the rendering is identical to + * usePlainNumberFormat's for General-format cells. + */ + private def plainNumberString(cell: Cell): String = + PlainNumberFormat + .format(BigDecimal.valueOf(cell.getNumericCellValue), new StringBuffer(), new FieldPosition(0)) + .toString + + /** Header-cell rendering, honoring usePlainNumberFormatForAllCells so a numeric header is named consistently with its + * data cells. + */ + private def headerCellString(cell: Cell): String = + if (cell.getCellType == CellType.NUMERIC && renderPlainNumber(cell)) plainNumberString(cell) + else dataFormatter.formatCellValue(cell) + /** Get workbook * * @param conf @@ -230,14 +256,14 @@ class ExcelHelper private (options: ExcelOptions) { val dataColumns = if (options.header) { - val headerNames = firstRow.map(dataFormatter.formatCellValue) + val headerNames = firstRow.map(headerCellString) val duplicates = { val nonNullHeaderNames = headerNames.filter(_ != null) nonNullHeaderNames.groupBy(identity).filter(_._2.size > 1).keySet } firstRow.zipWithIndex.map { case (cell, index) => - val value = dataFormatter.formatCellValue(cell) + val value = headerCellString(cell) val cellType = cell.getCellType if ( cellType == CellType.ERROR || cellType == CellType.BLANK || diff --git a/src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala b/src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala index c3789565..1b246e44 100644 --- a/src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala +++ b/src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala @@ -106,6 +106,10 @@ trait ExcelOptionsTrait extends Serializable { /* If true, format the cells without rounding and scientific notations */ val usePlainNumberFormat = getBool("usePlainNumberFormat", default = false) + /* If true, render every non-date numeric cell without rounding and scientific notations, + ignoring the cell's number format; date-formatted cells keep their formatted rendering */ + val usePlainNumberFormatForAllCells = getBool("usePlainNumberFormatForAllCells", default = false) + /* If true, keep undefined (Excel) rows */ val keepUndefinedRows = getBool("keepUndefinedRows", default = false) diff --git a/src/test/resources/spreadsheets/plain_number_all_cells.xlsx b/src/test/resources/spreadsheets/plain_number_all_cells.xlsx new file mode 100644 index 00000000..d24209ba Binary files /dev/null and b/src/test/resources/spreadsheets/plain_number_all_cells.xlsx differ diff --git a/src/test/resources/spreadsheets/plain_number_numeric_header.xlsx b/src/test/resources/spreadsheets/plain_number_numeric_header.xlsx new file mode 100644 index 00000000..b6a28e65 Binary files /dev/null and b/src/test/resources/spreadsheets/plain_number_numeric_header.xlsx differ diff --git a/src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala b/src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala index 25141cb1..fac34407 100644 --- a/src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala +++ b/src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala @@ -81,6 +81,36 @@ object PlainNumberReadSuite { ) val issue747Data: util.List[Row] = List(Row("9024523", "902"), Row("1020001", "102"), Row("9764342", "L906")).asJava + + // Predefined data for usePlainNumberFormatForAllCells tests + val allCellsSchema = StructType( + List( + StructField("formatted_number", StringType, true), + StructField("text_and_number", StringType, true), + StructField("general_number", StringType, true), + StructField("date_col", StringType, true) + ) + ) + + val allCellsPlainData: util.List[Row] = List( + Row("84.789", "138,56", "123456789012", "01.07.2026"), + Row("3.2886", "5414874004074", "-0.12345678901", "15.01.2023"), + Row("0.0005", "0,0005", "123456789012", "31.12.2024") // 0.0005 must not gain a trailing zero; C4 is =123456789012*1 + ).asJava + + val allCellsDisplayData: util.List[Row] = List( + Row("84.79", "138,56", "1.23457E+11", "01.07.2026"), // explicit formats round, General goes scientific + Row("3.29", "5414874004074", "-0.123456789", "15.01.2023"), + Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") // cached formula results read as Double.toString + ).asJava + + // usePlainNumberFormat=true alone: General cells render plain, explicit formats still round, + // cached formula results stay Double.toString -- the gap usePlainNumberFormatForAllCells closes + val allCellsGeneralPlainData: util.List[Row] = List( + Row("84.79", "138,56", "123456789012", "01.07.2026"), + Row("3.29", "5414874004074", "-0.12345678901", "15.01.2023"), + Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") + ).asJava } class PlainNumberReadSuite extends AnyFunSpec with DataFrameSuiteBase with Matchers { @@ -149,5 +179,35 @@ class PlainNumberReadSuite extends AnyFunSpec with DataFrameSuiteBase with Match // Verify both dataframes should be equal assertDataFrameEquals(dfWithPlain, dfWithoutPlain) } + + // deliberately goes through the spark.read.excel(...) DSL so the option-key plumbing is exercised too + def readAllCellsFixture(usePlainNumberFormat: Boolean, usePlainNumberFormatForAllCells: Boolean): DataFrame = { + val url = getClass.getResource("/spreadsheets/plain_number_all_cells.xlsx") + spark.read + .excel( + header = true, + usePlainNumberFormat = usePlainNumberFormat, + usePlainNumberFormatForAllCells = usePlainNumberFormatForAllCells + ) + .load(url.getPath) + } + + it("should render explicitly formatted cells plain when usePlainNumberFormatForAllCells=true") { + val df = readAllCellsFixture(usePlainNumberFormat = false, usePlainNumberFormatForAllCells = true) + val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + it("should keep display rendering when usePlainNumberFormatForAllCells=false") { + val df = readAllCellsFixture(usePlainNumberFormat = false, usePlainNumberFormatForAllCells = false) + val expected = spark.createDataFrame(allCellsDisplayData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + it("should still round explicitly formatted cells when only usePlainNumberFormat=true") { + val df = readAllCellsFixture(usePlainNumberFormat = true, usePlainNumberFormatForAllCells = false) + val expected = spark.createDataFrame(allCellsGeneralPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } } } diff --git a/src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala b/src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala index 17846c89..fe28b539 100644 --- a/src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala +++ b/src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala @@ -77,6 +77,36 @@ object PlainNumberReadSuite { ) val issue747Data: util.List[Row] = List(Row("9024523", "902"), Row("1020001", "102"), Row("9764342", "L906")).asJava + + // Predefined data for usePlainNumberFormatForAllCells tests + val allCellsSchema = StructType( + List( + StructField("formatted_number", StringType, true), + StructField("text_and_number", StringType, true), + StructField("general_number", StringType, true), + StructField("date_col", StringType, true) + ) + ) + + val allCellsPlainData: util.List[Row] = List( + Row("84.789", "138,56", "123456789012", "01.07.2026"), + Row("3.2886", "5414874004074", "-0.12345678901", "15.01.2023"), + Row("0.0005", "0,0005", "123456789012", "31.12.2024") // 0.0005 must not gain a trailing zero; C4 is =123456789012*1 + ).asJava + + val allCellsDisplayData: util.List[Row] = List( + Row("84.79", "138,56", "1.23457E+11", "01.07.2026"), // explicit formats round, General goes scientific + Row("3.29", "5414874004074", "-0.123456789", "15.01.2023"), + Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") // cached formula results read as Double.toString + ).asJava + + // usePlainNumberFormat=true alone: General cells render plain, explicit formats still round, + // cached formula results stay Double.toString -- the gap usePlainNumberFormatForAllCells closes + val allCellsGeneralPlainData: util.List[Row] = List( + Row("84.79", "138,56", "123456789012", "01.07.2026"), + Row("3.29", "5414874004074", "-0.12345678901", "15.01.2023"), + Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") + ).asJava } class PlainNumberReadSuite extends AnyFunSuite with DataFrameSuiteBase with ExcelTestingUtilities { @@ -181,4 +211,60 @@ class PlainNumberReadSuite extends AnyFunSuite with DataFrameSuiteBase with Exce // Verify both dataframes should be equal assertDataFrameEquals(dfWithPlain, dfWithoutPlain) } + + test("explicitly formatted cells render plain when usePlainNumberFormatForAllCells=true") { + val df = readFromResources( + spark, + path = "plain_number_all_cells.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false) + ) + val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + test("explicitly formatted cells render plain when usePlainNumberFormatForAllCells=true and maxRowsInMemory") { + val df = readFromResources( + spark, + path = "plain_number_all_cells.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false, "maxRowsInMemory" -> 1) + ) + val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + test("explicitly formatted cells keep display rendering when usePlainNumberFormatForAllCells=false") { + val df = readFromResources( + spark, + path = "plain_number_all_cells.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> false, "inferSchema" -> false) + ) + val expected = spark.createDataFrame(allCellsDisplayData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + test("explicitly formatted cells still round when only usePlainNumberFormat=true") { + val df = readFromResources( + spark, + path = "plain_number_all_cells.xlsx", + options = Map("usePlainNumberFormat" -> true, "inferSchema" -> false) + ) + val expected = spark.createDataFrame(allCellsGeneralPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + test("numeric header cells are named consistently with their data cells") { + val plain = readFromResources( + spark, + path = "plain_number_numeric_header.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false) + ) + assert(plain.schema.fieldNames.toSeq == Seq("123456789012", "name")) + + val display = readFromResources( + spark, + path = "plain_number_numeric_header.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> false, "inferSchema" -> false) + ) + assert(display.schema.fieldNames.toSeq == Seq("1.23457E+11", "name")) + } }