From 5a5554a7dffafe437e48424cb988c3e5d4d7657e Mon Sep 17 00:00:00 2001 From: Vladyslav Ishchenko Date: Wed, 19 Aug 2026 10:39:55 +0200 Subject: [PATCH] feat: Add usePlainNumberFormatForAllCells option to read numeric cells at full precision regardless of cell format --- README.md | 2 + .../dev/mauch/spark/excel/DataColumn.scala | 18 ++++ .../dev/mauch/spark/excel/DefaultSource.scala | 1 + .../dev/mauch/spark/excel/ExcelRelation.scala | 2 + .../mauch/spark/excel/PlainNumberFormat.scala | 5 +- .../scala/dev/mauch/spark/excel/package.scala | 2 + .../mauch/spark/excel/v2/ExcelHelper.scala | 36 +++++++- .../spark/excel/v2/ExcelOptionsTrait.scala | 4 + .../spreadsheets/plain_number_all_cells.xlsx | Bin 0 -> 5095 bytes .../plain_number_numeric_header.xlsx | Bin 0 -> 4770 bytes .../spark/excel/PlainNumberReadSuite.scala | 60 ++++++++++++ .../spark/excel/v2/PlainNumberReadSuite.scala | 86 ++++++++++++++++++ 12 files changed, 209 insertions(+), 7 deletions(-) create mode 100644 src/test/resources/spreadsheets/plain_number_all_cells.xlsx create mode 100644 src/test/resources/spreadsheets/plain_number_numeric_header.xlsx diff --git a/README.md b/README.md index 6652a86d..7b27d380 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ val df = spark.read .option("treatEmptyValuesAsNulls", "false") // Optional, default: true .option("setErrorCellsToFallbackValues", "true") // Optional, default: false, where errors will be converted to null. If true, any ERROR cell values (e.g. #N/A) will be converted to the zero values of the column's data type. .option("usePlainNumberFormat", "false") // Optional, default: false, If true, format the cells without rounding and scientific notations + .option("usePlainNumberFormatForAllCells", "false") // Optional, default: false, If true, render every non-date numeric cell without rounding and scientific notations, ignoring the cell's number format; date-formatted cells keep their formatted rendering .option("inferSchema", "false") // Optional, default: false .option("addColorColumns", "true") // Optional, default: false .option("timestampFormat", "MM-dd-yyyy HH:mm:ss") // Optional, default: yyyy-mm-dd hh:mm:ss[.fffffffff] @@ -96,6 +97,7 @@ val df = spark.read.excel( treatEmptyValuesAsNulls = false, // Optional, default: true setErrorCellsToFallbackValues = false, // Optional, default: false, where errors will be converted to null. If true, any ERROR cell values (e.g. #N/A) will be converted to the zero values of the column's data type. usePlainNumberFormat = false, // Optional, default: false. If true, format the cells without rounding and scientific notations + usePlainNumberFormatForAllCells = false, // Optional, default: false. If true, render every non-date numeric cell without rounding and scientific notations, ignoring the cell's number format; date-formatted cells keep their formatted rendering inferSchema = false, // Optional, default: false addColorColumns = true, // Optional, default: false timestampFormat = "MM-dd-yyyy HH:mm:ss", // Optional, default: yyyy-mm-dd hh:mm:ss[.fffffffff] diff --git a/src/main/scala/dev/mauch/spark/excel/DataColumn.scala b/src/main/scala/dev/mauch/spark/excel/DataColumn.scala index e593fcad..fe8865db 100644 --- a/src/main/scala/dev/mauch/spark/excel/DataColumn.scala +++ b/src/main/scala/dev/mauch/spark/excel/DataColumn.scala @@ -20,6 +20,7 @@ import org.apache.spark.sql.types._ import java.math.BigDecimal import java.sql.{Date, Timestamp} +import java.text.FieldPosition import scala.util.{Failure, Success, Try} trait DataColumn extends PartialFunction[Seq[Cell], Any] { @@ -36,11 +37,26 @@ class HeaderDataColumn( val columnIndex: Int, treatEmptyValuesAsNulls: Boolean, usePlainNumberFormat: Boolean, + usePlainNumberFormatForAllCells: Boolean, parseTimestamp: String => Timestamp, parseDate: String => Date, setErrorCellsToFallbackValues: Boolean ) extends DataColumn { def name: String = field.name + + /** Whether this (cached-)numeric cell should be rendered at full precision, ignoring its number format. Date cells + * keep their formatted rendering, non-finite values keep POI's display rendering. + */ + private def renderPlainNumber(cell: Cell): Boolean = + usePlainNumberFormatForAllCells && !DateUtil.isCellDateFormatted(cell) && + java.lang.Double.isFinite(cell.getNumericCellValue) + + /** Same invocation POI's DataFormatter uses for a registered format, so digits match usePlainNumberFormat's. */ + private def plainNumberString(cell: Cell): String = + PlainNumberFormat + .format(BigDecimal.valueOf(cell.getNumericCellValue), new StringBuffer(), new FieldPosition(0)) + .toString + def extractValue(cell: Cell): Any = { val cellType = if (cell.getCellType == CellType.FORMULA) cell.getCachedFormulaResultType else cell.getCellType if (cellType == CellType.BLANK) { @@ -65,11 +81,13 @@ class HeaderDataColumn( case CellType.FORMULA => cell.getCachedFormulaResultType match { case CellType.STRING => Option(cell.getRichStringCellValue).map(_.getString) + case CellType.NUMERIC if renderPlainNumber(cell) => Some(plainNumberString(cell)) case CellType.NUMERIC => Option(cell.getNumericCellValue).map(_.toString) case CellType.BLANK => None case _ => Some(dataFormatter.formatCellValue(cell)) } case CellType.BLANK => None + case CellType.NUMERIC if renderPlainNumber(cell) => Some(plainNumberString(cell)) case _ => Some(dataFormatter.formatCellValue(cell)) } def parseNumber(string: Option[String]): Option[Double] = string.filter(_.trim.nonEmpty).map(stringToDouble) diff --git a/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala b/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala index facbb542..4a15fa0f 100644 --- a/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala +++ b/src/main/scala/dev/mauch/spark/excel/DefaultSource.scala @@ -43,6 +43,7 @@ class DefaultSource extends RelationProvider with SchemaRelationProvider with Cr treatEmptyValuesAsNulls = parameters.get("treatEmptyValuesAsNulls").fold(false)(_.toBoolean), setErrorCellsToFallbackValues = parameters.get("setErrorCellsToFallbackValues").fold(false)(_.toBoolean), usePlainNumberFormat = parameters.get("usePlainNumberFormat").fold(false)(_.toBoolean), + usePlainNumberFormatForAllCells = parameters.get("usePlainNumberFormatForAllCells").fold(false)(_.toBoolean), userSchema = Option(schema), inferSheetSchema = parameters.get("inferSchema").fold(false)(_.toBoolean), addColorColumns = parameters.get("addColorColumns").fold(false)(_.toBoolean), diff --git a/src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala b/src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala index 6d1708e5..bdc80b48 100644 --- a/src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala +++ b/src/main/scala/dev/mauch/spark/excel/ExcelRelation.scala @@ -33,6 +33,7 @@ case class ExcelRelation( header: Boolean, treatEmptyValuesAsNulls: Boolean, usePlainNumberFormat: Boolean, + usePlainNumberFormatForAllCells: Boolean, inferSheetSchema: Boolean, setErrorCellsToFallbackValues: Boolean, addColorColumns: Boolean = true, @@ -189,6 +190,7 @@ case class ExcelRelation( cell.getColumnIndex, treatEmptyValuesAsNulls, usePlainNumberFormat, + usePlainNumberFormatForAllCells, timestampParser, dateParser, setErrorCellsToFallbackValues diff --git a/src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala b/src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala index d6f7f144..57a4fbcf 100644 --- a/src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala +++ b/src/main/scala/dev/mauch/spark/excel/PlainNumberFormat.scala @@ -38,8 +38,9 @@ object PlainNumberFormat extends Format { // It's an integer, format without decimal point toAppendTo.append(stripped.toBigInteger().toString()) } else { - // It's not an integer, format as plain string - toAppendTo.append(bd.toPlainString) + // It's not an integer, format the stripped value so no trailing zero from + // Double.toString's "d.0E-x" mantissa survives (0.0005 must not become "0.00050") + toAppendTo.append(stripped.toPlainString) } } diff --git a/src/main/scala/dev/mauch/spark/excel/package.scala b/src/main/scala/dev/mauch/spark/excel/package.scala index ce9709a2..6d2e3f7d 100644 --- a/src/main/scala/dev/mauch/spark/excel/package.scala +++ b/src/main/scala/dev/mauch/spark/excel/package.scala @@ -78,6 +78,7 @@ package object excel { setErrorCellsToFallbackValues: Boolean = false, inferSchema: Boolean = false, usePlainNumberFormat: Boolean = false, + usePlainNumberFormatForAllCells: Boolean = false, addColorColumns: Boolean = false, dataAddress: String = null, timestampFormat: String = null, @@ -92,6 +93,7 @@ package object excel { "treatEmptyValuesAsNulls" -> treatEmptyValuesAsNulls, "setErrorCellsToFallbackValues" -> setErrorCellsToFallbackValues, "usePlainNumberFormat" -> usePlainNumberFormat, + "usePlainNumberFormatForAllCells" -> usePlainNumberFormatForAllCells, "inferSchema" -> inferSchema, "addColorColumns" -> addColorColumns, "dataAddress" -> dataAddress, diff --git a/src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala b/src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala index e0e84e10..efda56b0 100644 --- a/src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala +++ b/src/main/scala/dev/mauch/spark/excel/v2/ExcelHelper.scala @@ -22,7 +22,7 @@ import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.poi.hssf.usermodel.HSSFWorkbookFactory import org.apache.poi.openxml4j.util.ZipInputStreamZipEntrySource import org.apache.poi.ss.SpreadsheetVersion -import org.apache.poi.ss.usermodel.{Cell, CellType, DataFormatter, FormulaError, Workbook, WorkbookFactory} +import org.apache.poi.ss.usermodel.{Cell, CellType, DataFormatter, DateUtil, FormulaError, Workbook, WorkbookFactory} import org.apache.poi.ss.util.{AreaReference, CellReference} import org.apache.poi.util.IOUtils import org.apache.poi.xssf.usermodel.XSSFWorkbookFactory @@ -52,8 +52,9 @@ object PlainNumberFormat extends Format { // It's an integer, format without decimal point toAppendTo.append(stripped.toBigInteger().toString()) } else { - // It's not an integer, format as plain string - toAppendTo.append(bd.toPlainString) + // It's not an integer, format the stripped value so no trailing zero from + // Double.toString's "d.0E-x" mantissa survives (0.0005 must not become "0.00050") + toAppendTo.append(stripped.toPlainString) } } @@ -97,14 +98,39 @@ class ExcelHelper private (options: ExcelOptions) { */ case CellType.ERROR => FormulaError.forInt(cell.getErrorCellValue).getString case CellType.STRING => cell.getStringCellValue + case CellType.NUMERIC if renderPlainNumber(cell) => plainNumberString(cell) case CellType.NUMERIC => cell.getNumericCellValue.toString /* Get what displayed on the cell, for all other cases */ case _ => dataFormatter.formatCellValue(cell) } + case CellType.NUMERIC if renderPlainNumber(cell) => plainNumberString(cell) case _ => dataFormatter.formatCellValue(cell) } + /** Whether this (cached-)numeric cell should be rendered at full precision, ignoring its number format. Date cells + * keep their formatted rendering, non-finite values keep POI's display rendering. + */ + private def renderPlainNumber(cell: Cell): Boolean = + options.usePlainNumberFormatForAllCells && !DateUtil.isCellDateFormatted(cell) && + java.lang.Double.isFinite(cell.getNumericCellValue) + + /** Render a numeric cell at full precision, ignoring its number format. Invokes [[PlainNumberFormat]] with the same + * argument POI's DataFormatter passes to a registered format, so the rendering is identical to + * usePlainNumberFormat's for General-format cells. + */ + private def plainNumberString(cell: Cell): String = + PlainNumberFormat + .format(BigDecimal.valueOf(cell.getNumericCellValue), new StringBuffer(), new FieldPosition(0)) + .toString + + /** Header-cell rendering, honoring usePlainNumberFormatForAllCells so a numeric header is named consistently with its + * data cells. + */ + private def headerCellString(cell: Cell): String = + if (cell.getCellType == CellType.NUMERIC && renderPlainNumber(cell)) plainNumberString(cell) + else dataFormatter.formatCellValue(cell) + /** Get workbook * * @param conf @@ -230,14 +256,14 @@ class ExcelHelper private (options: ExcelOptions) { val dataColumns = if (options.header) { - val headerNames = firstRow.map(dataFormatter.formatCellValue) + val headerNames = firstRow.map(headerCellString) val duplicates = { val nonNullHeaderNames = headerNames.filter(_ != null) nonNullHeaderNames.groupBy(identity).filter(_._2.size > 1).keySet } firstRow.zipWithIndex.map { case (cell, index) => - val value = dataFormatter.formatCellValue(cell) + val value = headerCellString(cell) val cellType = cell.getCellType if ( cellType == CellType.ERROR || cellType == CellType.BLANK || diff --git a/src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala b/src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala index c3789565..1b246e44 100644 --- a/src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala +++ b/src/main/scala/dev/mauch/spark/excel/v2/ExcelOptionsTrait.scala @@ -106,6 +106,10 @@ trait ExcelOptionsTrait extends Serializable { /* If true, format the cells without rounding and scientific notations */ val usePlainNumberFormat = getBool("usePlainNumberFormat", default = false) + /* If true, render every non-date numeric cell without rounding and scientific notations, + ignoring the cell's number format; date-formatted cells keep their formatted rendering */ + val usePlainNumberFormatForAllCells = getBool("usePlainNumberFormatForAllCells", default = false) + /* If true, keep undefined (Excel) rows */ val keepUndefinedRows = getBool("keepUndefinedRows", default = false) diff --git a/src/test/resources/spreadsheets/plain_number_all_cells.xlsx b/src/test/resources/spreadsheets/plain_number_all_cells.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..d24209ba3410b8a96add34a5046389407193a848 GIT binary patch literal 5095 zcmZ`-1yodR*Bw$ih7JiyWoVRc0ZA!!=q_a#WI(z*r9nzbgBZF+x&{~;0civg>F#gT z^?$zdzu&#<-n;Hvcb}QF*FNVt&(>7Mz$6C%062gb=EsJLUFr{#P;dRHiv)F9LM=3% zppGzJGe<`rH;BDDQXRjQj{tlAllF_6R}l<(ME4~lvKo;*F2U8eAwj2G8<^4#?(W@m zIXvOF*sD_&c6gQ{f&yKHGUF_#Qi$H{i0oz068;(gs4`mOJVpkLwj9WWUoby`NZ(>j z>DnldqKykOS6jlm4O){DN+j zi&0=KggCC|j!8%?F$8qFe&l!;1P){ySF8xpN)4S%%}m6$u{sXhVCBOZjc4n^?ac|o zN038KE6h}^%OCcuxfnfhMugMdyR&exv*4~T1vNC^h!uE5mq@K#@$zXSC;eG<2(;r= z=&Vr2nqMu}i=eV+pa-#g=-~PKy^8L-q2cP-Qv*D`+3vBi>2Ie)zNGrlG^0J@CA~LK zj}M^V$^K^Mm0<=82{r(b!2tk}p_p;A=XJKVf>`~&^Z(>#L*D>4B|`4A1s`;?IfUSM z-t(`nB!k(#n67*kw4uQmObU*+cejiS^8KWAj|7oSxDl^jpDuw3T00w1P+I*a;=Z@Z zWBx59EOFP(bdq;^yrG77cK(R~E2%6W&0F|(GhV)$ah1LupiRWPhYZ;EB_I+<)r}l6 zZa8m1%gZwUy>fcyagd#THnozaskbr6R%TFFOCM^&RQZ)r@R$Oj8*P<~M)pLOy)LYB z_r$*Ad}>z83b`hBQ^4PWj}FY_I{;Rk*JJ>H2}^~#62 zBNR9SgyLeE;DHxjhwU4$qk5;Dg&?%^OFM2N)rjFpM|#Y8t%j`D%zgV$d9t{bxpsq= zpAV->;tD=Q>pUDe@}65yx6QzdHh4Jl%yniYCcLwF(2cNZ=V54DNqA@Bp!2#NJI6)I zv%ED*TRFw0bq^p#)mNDjCIHy+#dzN^MzZnkQP0?`D%gzqWal2uf zWFbYT%vdZUfN=Yenx+?AwmM?IHGbGIb{u)TN|P2Rt=s=VDS)}-3;g0BrJ*NJ&?D1Z zY~Wdf#@o{pg5fZta73IXzFmnj@K76R;z%uc^x#HGDq9Uup>u&D)Wm1tUWkLZm zX`IZE)Tf&C=y|yO$=M;tm7PVT^JLyNO*q|i(P2l0&@{;>*_)!g(BY%TFS#jT3bYjJIiU}CK^^06NzFUO84@?ifc=U1*s>uhtn9$|qCV;m$S%3^ zr!F%_WVRD|Sx;5sUfNZ#yme;p|0YP=q6nvR3AG)T@VLY}$+(M0A4hi+-G;70kvYh& zVVeQY8k1JsO3EcCWlnRC)~Slpzxw(6AX9x{=a8Gma3$%vUeVnIDbwyB{c$b><8ef! z@y5caZwDe{Nk1`!h^VA(U0E^*L%VAPB^5B;s#YyS}-3_w3N<#WG7i zpH7N=Ev;M=2BbdoiIpSLvSesd!bu7$p<{m7Kx zpc8{DHpfdXO5vO1YKo!B(op=;bb(cs1N^ot4dj-#XscZ+nN4ButW{adqnpS_2D_zA zPbDiUG?15i#oez7NHr+&fe$PasR~qb_GomPR=JehABlWY5Ud8KP&Y2EbUoM5sRh2M zCPo*iYY=_!Va0D#usg?1fD1AR(*rK58*PlOre~CR+HW^xfV$c0;^Z?5OdcV|D{Exc? zb+;)7ilsz-6+nR@8pgwHK=O!E0xV`Kr=_i}gr}}i6Q>}5NRY08~9znLX$XuvG8m-`1WRYkz9#nQr=BxDd zD532kp%8r>mNEaU>U+7%8h8?4F1H9vTiat6HX=Wv{k?$DepF|}L`SV0SO5U^?*+sa z>hug|Yh~pOFO8aGgbYS7RfQL9m}!rFn6_9BQkA)TPf%F- z=%|93jrI1lj@JfkKyno*F3EW@o(%ZbI3ZUJtGRE#iEAg!HrjZgi!q!*t};Qd{=Oy; zBDCGeG8TZD*qOJIuR#~(aC3#eI9C>YOuj69c52`rs zRfJ;Um?cFe%qeLdjgAU*trMTXjeIPi{RUvpox@KyTp2c_; zq;@e;Oto2svjtUkKHB49qBX3lNw^wE-zh)KGFM&U#2cM}&S*)Nbe5y%5s}+5Ts&8@ zMrv9$OtLWkenSm@SWM{@=koz(_U%|PYV8}=D>EFz+J{@^NZ?9;d_&@oF0{aKW1rM7 z_jhdS0^;%*gFq3{cMbw!3s!T!1lG@GCjG=JZW2|u#ZboBW>c{$dzm$amXU(YrBtn> zv=wcB(0`;x%xFWGPg|YcAZauFSFjO#zWx$xbx9RFXR|c*zO>o>`E|Y5=#(}Um*yaf zs}sNvF;#MMZ}PaGdI08qarq<`4A-$t%I=5VVy> zO}x-nwJY$jxYfLK=@lZFlTprE7;GWW9ErEbefOgf-7XJIK7sC z@z#JD4>|!j6n(=?{7zg3&R{49LZRG5T1z{jmQd0j%$q=(y12)}$kwY!nJI~LSZUG6 z?s;4HNxt~A=?2jHJ#ltr*J6VL`e9p^a2K75{`zq~IdLZCICZE)gc`79_^aH36B{JO z-WhJ6r17$Z^^4TVolNo?J7DWrTKOHx+UIOvDe`+ZHF_ILR1<#OdVY^}cFsgv!qC^X zjjTM?yUw;!EIc)@W#JaEC!Xrohf4MUIN?!c$O9gt7U_emXv!(p@iC+yQOTx_n#I2qiLHnOsl$= z_)4~}rOFUz9BfM6G~?d(ozmeMq&V4BOQTvp?#*%%B_Wk(Taa4-(aAUg@JD)ucb&hH zmeR=k2x}ZNj@Wq#LmVGU*F)Fm2A4S-Z)Yj;&d-(L+@3;33!c(*!W?LU+aoUt85<6k zfM0w{FMS!LBI$g_9|snQd60>`dvr<>Q!f%(wpmwd*fANS(>r=L>$JnI*N`&BqetqiF><%fe)0T#8i)PhH9DOKWYyCLFEy)+Xy1Xk-4v! zl}keZnG`1pO-_>&*%rFre+2Tp7_fT#2@hfNw}xw!)+3rx+NDJ47$5aCak8?9@$&q< zXU3~Kpz{;Rz=2jKYVX_8`HMc6%7@|ChOg$nYLByM-#?80+{|Q4s(O``#SJGGdh>!F zHl1(Z$=7Z;`S@-zp#0fw)2!z?;5&C!Ww^72Vh0aClYmzC*t)sf9L8(ZVCTyFcya6X zuu3aeJn{py)B8s#pDAka(kMned_?=Wp+B7xe8NFvaF5DL8`m=usZQVH4PZ@_q-V>?sT9w?MwB%3<=TN?_ zwiwLOH8I}v(KsWFSUgAm&@6fn#`B14xPR}<6K?PZ{2d0BHDTEpP=9uHR0!M>@nJR~ z!AqaKBBRAt)~TsupwR!2F?oyHdBXruW-X%^t-(!|Tk&LI7ZHQ`;aEpEw_HKwWG?u# zwbFjAeBiF$hy8nR1R{yJv$A;}Wedy=ufuXKypb2Z1l*gojodW>50M zW%{fM$F4Vk`H)P(QI)A6T%78rcJ^tJhr@>;m#LJ z&m_L@;{Q#q;QCU1F^beP6v4E=$^9*)e^UHQR8?`3M1gz+!EoSL0QI}kZ1e)2A_}Uo zfD=;x`(=f#8+0vz2RG~a^pMG)jx23>g(5t~wI;-Yd$#PoR$V_B0D0o{6bj=DT7s`fzI3S3myIWNcaci9#mw{j>xI{83U9R$`j4(+xxAX7_cFtB=md^Wzj&fPy6=BT#TnoC#9)5{YA@*EkA0Kqt zZw_NrRibe7I%m&CX~09ZX~E9xRVS@JJ`CFnw1u?biw;yX9iL4MmrO zEH@@Of@Yr+o<0_Nc+8UbVjHugw2yJm^)<(Z0j)c6_QjUxI6Ob0^C4l_f+SoTHFTk& zlVkjEQwF7)Uxy!x;s0;ZTt{E`Lw;ic06(<0f1>}-9k~v_ZW{c7&!B$(Uo3>{0j>{? z{|(^Jfb%QBzmAaCqg)>#{u>1ijQ1(*MZmj;2$7rm;N*L zf0pcZ@bxfh*GuV793_-N_b-K|sfva2cmM!g)G3AP;0ZrXFu;ER DQc>jZ literal 0 HcmV?d00001 diff --git a/src/test/resources/spreadsheets/plain_number_numeric_header.xlsx b/src/test/resources/spreadsheets/plain_number_numeric_header.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..b6a28e656ea54620388140a12c8831109d768c26 GIT binary patch literal 4770 zcmZ`-1ymGm+g=cm6j-{Ej-`=qq(cyp6_!v~0f|LIO46l68l;p~iKSa)=@3?Gk&s42 zq(j<&)$@P8^1a{8nP<+NGuJ(HUFXinK<5e$B>(`x2Ry!i1X1tN(@eoU4Pq`b%w-F+ zF>r&qx(nTNbrtk>cG5%X5w!~w=pXZ2uVGNVX(7m} z&w5>ipi{uDr7Z8*;QrX=+-Pdq6=bZaQ{|&rIg%P9uU^dBQ>bK4oiL?@v!P1&R z@T*laR96Z!_L%O}*a=(IupK_wrtHTV_!za_Tlkx1JM-dww!uVr}oIM2n zd2pfuO6Qa6d)g1GZVl>sT9|qaL^4S5EgbAD_?RicAon-oMNJrzX|<}KJ#67&Jgp0Z zb)ATxmMA?C(T#sXT+=tyH?Vu?;`idUw(&XyvO50Iobc{!@A&ximy_W@ax++l#UAO> z-IouK4q#s?{#NEWB#WI44*}?9({}L9Fyz6Z}DKw34t{0k}Hx=a|R~Du#Mr^kc7V28onmGVEr2P9RfE_a-(fB&v zlmlqUyg9uPJNj$Q^rt&;2d7+G4O?q}OOUJKG-GDc*7H86MFL3hDDt#%0N;`K~28H z2iuQ_Q{@RoZ(@yZjehr^ThFx5B8)Y^HR|Z~X*4dfyX=EEN$bw7@Q(7x?vf84>keGp z=jDzCYt;5C>PzdsK&sjmrBPM@*!3~`{m7M6OWqUDWco>_=16>m7Y^EC zgf3NF-7PyF_ZmR5b6D5l2|~FpYQ8;j1QL%%@zxs9<7e~+A83SJ@0vlJAEY(+6^Qv} z`^yYDCI>o{U=?pPXQGnM+`lZpC{CKtQyFq?0TJw+vYX!v=XQ~$aFb!QQ+a|qJWMF$ z6h&S0da{uf; z>~wmyP<{B&q-7>Q4NQfVMms0|<_4$>?VZxLQ&fk@LO4J};G& za=;N8cLms~<8^|ol{p&DqWib`ItjkmnsX8i7>`0FqkuUo3@&R%2ihqUD~Lv(208QP zHAJq50tj_1XSG^0NF=6O@22y$F1fXqOGfdBkC)jC zg^kiw8|YP9uQ)el&&WvSq$e~Ij^2)kbsVpCdDAaSPvSKCr<*#jejFY?T}2vJ+4NFILsD}IaS*6>hbQ_3MIy;qefYj@)8zy!@l=HUM0xz#VMHpi5-Zrro zG4H;@AD{O`H%9G?>uMUrYH2ueX}Snn>w@6*qH|`uE#2;rPGMKlKMSoynlPbE%y%nV zAIjHI>7y?0mi4|MCfBDX0&3bM(-djv?a>*vuJUVjnn=A<6RQKJ(Y7po?s=qd)Bt>3 zM~W@l*ew0p7b;>`v^yt2OaL;ExC>m=v)CA4&CDwIWqW**=7qH0J$LQXAYGANwc58( zEe|;Sv~=Q4UMG>itWPL6bFoEI(cT%iuo3+Z>+b>*=p!-v4pTVvFr}6DcLDK& zxjDMqL!ln-Lceams)%R2BVoHR85{w01Jp;NX&jVx@N!sdN=PKMw5T_B^h~tL&+9t^ z#5&4-;odp|Y?Vk)1(O!euipW$U5206;LD%1G{V0SL+KFlClN(H&mmW{jUi|&$7OQ`Vrp8+yO@^B-MV_CocoAFOl{< z?S76807w!803^RjyLC|zLr$coB=8>e%clLx{nMgLhj$Be=#Up8gE z1F(HOql!$n*l_0voU| zk%sL}4Oa+)cgVG4G%@g7X0?A~u!W(<=3euS-Z_t%3jX6!s>SisBbT81sa#L9F;Z@iXN`SmZ13Di5?GKP2 zHWuzJ<$yfG)wEpcp^T-{wr2eC3l;b4r6VK-%T@39@|4o*%Y_DB4c`C}9$m*W3~f_( zSFTgEI8MN$b;z9C$7>qMcYc~GH zAiwh=Xz@G30js}#xL#vDstx12)EGY|!rZOgpib^WfnL>(7w5n?3-5Nf6Sx)WQZ z^kao;1W`leYX0-i1e?zN!`P2)toGzOKXP&e5TxQSA2Yg77dmwdcS0ub+$aN7Ir3WP zJjw&}-Oy1I$Q6(OaPW}~^m&i7SD?cMU9anYrnOI)u-@ce1N9=LJit0C@5o@+RyCV^ z6eOneV{isVIF^x(DE)R;d}F)l{oVm87O(DND0j~Uy6>%JRs^YRp6cPf z*gbba6aJCGy%|#h@CM@56`BVmmE%CO+0`*|Ff!`RY)H}*Gs@~Lq`k6RYu8Xo@F7d; z7OlsIIiT`^(%o2n0h;`3)1lpgIGi^}M#lM7YEs8@p&uV;>^G=B-M#x}U*e@`^fiH; zTtSmu(YcYe1X|Sc%}l*6Tiw@O9o9r3@Ogsd@aN{)Gbp-28vhXo;@XdLr4?LnNTz#nKW(PD*0K^UT8DY<9x|c?&~hm-{Oj`FEy26L_Ng_ zO#fTl-;VT8iGMj&ZG!x@r^3Xc2;d5U_SIM}c9CEy6-`9QF?sOK%98dC1|&d}sY&=Q zD)rrWTRS1~C_h=l30dHtJy$=p=NmJiK$ek84ZT3g6?>4s$m&m|zouM^@4z^RWTC6- zjng$0xc1}vRMFZEg|dKWQahx=PfA%{soL`Lt(Y%E+MNz1kY-b{2d=vN|sQi@t`Dd7=8}Br=~veM^Sdf?OM01ivf*K$oRamZ{Z%$ zAN$q@n{5g}QB1IWg4TWt@cZgwkxDz0#+YGx&d8<3`2$kz9W|atw_arTO^B;XPGoh& z^?m*Y<0IdoX0!s_@Z`u_*}%kazd?jQIk%;*1Q`n_!6a=-H*1HsJrzYP3q_w!#?E;SJU zv4TZG@XN}dcH%Pha^C&}oxr5=e`N8?z{`2?53mi>#$c%bGh{D=FUR>GFw?bP;QtZz lm+f2*r9XDCF}e9)0?R-L7qdM901#kK1xyCl|G5PP_&>ODJrw`| literal 0 HcmV?d00001 diff --git a/src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala b/src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala index 25141cb1..fac34407 100644 --- a/src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala +++ b/src/test/scala/dev/mauch/spark/excel/PlainNumberReadSuite.scala @@ -81,6 +81,36 @@ object PlainNumberReadSuite { ) val issue747Data: util.List[Row] = List(Row("9024523", "902"), Row("1020001", "102"), Row("9764342", "L906")).asJava + + // Predefined data for usePlainNumberFormatForAllCells tests + val allCellsSchema = StructType( + List( + StructField("formatted_number", StringType, true), + StructField("text_and_number", StringType, true), + StructField("general_number", StringType, true), + StructField("date_col", StringType, true) + ) + ) + + val allCellsPlainData: util.List[Row] = List( + Row("84.789", "138,56", "123456789012", "01.07.2026"), + Row("3.2886", "5414874004074", "-0.12345678901", "15.01.2023"), + Row("0.0005", "0,0005", "123456789012", "31.12.2024") // 0.0005 must not gain a trailing zero; C4 is =123456789012*1 + ).asJava + + val allCellsDisplayData: util.List[Row] = List( + Row("84.79", "138,56", "1.23457E+11", "01.07.2026"), // explicit formats round, General goes scientific + Row("3.29", "5414874004074", "-0.123456789", "15.01.2023"), + Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") // cached formula results read as Double.toString + ).asJava + + // usePlainNumberFormat=true alone: General cells render plain, explicit formats still round, + // cached formula results stay Double.toString -- the gap usePlainNumberFormatForAllCells closes + val allCellsGeneralPlainData: util.List[Row] = List( + Row("84.79", "138,56", "123456789012", "01.07.2026"), + Row("3.29", "5414874004074", "-0.12345678901", "15.01.2023"), + Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") + ).asJava } class PlainNumberReadSuite extends AnyFunSpec with DataFrameSuiteBase with Matchers { @@ -149,5 +179,35 @@ class PlainNumberReadSuite extends AnyFunSpec with DataFrameSuiteBase with Match // Verify both dataframes should be equal assertDataFrameEquals(dfWithPlain, dfWithoutPlain) } + + // deliberately goes through the spark.read.excel(...) DSL so the option-key plumbing is exercised too + def readAllCellsFixture(usePlainNumberFormat: Boolean, usePlainNumberFormatForAllCells: Boolean): DataFrame = { + val url = getClass.getResource("/spreadsheets/plain_number_all_cells.xlsx") + spark.read + .excel( + header = true, + usePlainNumberFormat = usePlainNumberFormat, + usePlainNumberFormatForAllCells = usePlainNumberFormatForAllCells + ) + .load(url.getPath) + } + + it("should render explicitly formatted cells plain when usePlainNumberFormatForAllCells=true") { + val df = readAllCellsFixture(usePlainNumberFormat = false, usePlainNumberFormatForAllCells = true) + val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + it("should keep display rendering when usePlainNumberFormatForAllCells=false") { + val df = readAllCellsFixture(usePlainNumberFormat = false, usePlainNumberFormatForAllCells = false) + val expected = spark.createDataFrame(allCellsDisplayData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + it("should still round explicitly formatted cells when only usePlainNumberFormat=true") { + val df = readAllCellsFixture(usePlainNumberFormat = true, usePlainNumberFormatForAllCells = false) + val expected = spark.createDataFrame(allCellsGeneralPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } } } diff --git a/src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala b/src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala index 17846c89..fe28b539 100644 --- a/src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala +++ b/src/test/scala/dev/mauch/spark/excel/v2/PlainNumberReadSuite.scala @@ -77,6 +77,36 @@ object PlainNumberReadSuite { ) val issue747Data: util.List[Row] = List(Row("9024523", "902"), Row("1020001", "102"), Row("9764342", "L906")).asJava + + // Predefined data for usePlainNumberFormatForAllCells tests + val allCellsSchema = StructType( + List( + StructField("formatted_number", StringType, true), + StructField("text_and_number", StringType, true), + StructField("general_number", StringType, true), + StructField("date_col", StringType, true) + ) + ) + + val allCellsPlainData: util.List[Row] = List( + Row("84.789", "138,56", "123456789012", "01.07.2026"), + Row("3.2886", "5414874004074", "-0.12345678901", "15.01.2023"), + Row("0.0005", "0,0005", "123456789012", "31.12.2024") // 0.0005 must not gain a trailing zero; C4 is =123456789012*1 + ).asJava + + val allCellsDisplayData: util.List[Row] = List( + Row("84.79", "138,56", "1.23457E+11", "01.07.2026"), // explicit formats round, General goes scientific + Row("3.29", "5414874004074", "-0.123456789", "15.01.2023"), + Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") // cached formula results read as Double.toString + ).asJava + + // usePlainNumberFormat=true alone: General cells render plain, explicit formats still round, + // cached formula results stay Double.toString -- the gap usePlainNumberFormatForAllCells closes + val allCellsGeneralPlainData: util.List[Row] = List( + Row("84.79", "138,56", "123456789012", "01.07.2026"), + Row("3.29", "5414874004074", "-0.12345678901", "15.01.2023"), + Row("0.00", "0,0005", "1.23456789012E11", "31.12.2024") + ).asJava } class PlainNumberReadSuite extends AnyFunSuite with DataFrameSuiteBase with ExcelTestingUtilities { @@ -181,4 +211,60 @@ class PlainNumberReadSuite extends AnyFunSuite with DataFrameSuiteBase with Exce // Verify both dataframes should be equal assertDataFrameEquals(dfWithPlain, dfWithoutPlain) } + + test("explicitly formatted cells render plain when usePlainNumberFormatForAllCells=true") { + val df = readFromResources( + spark, + path = "plain_number_all_cells.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false) + ) + val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + test("explicitly formatted cells render plain when usePlainNumberFormatForAllCells=true and maxRowsInMemory") { + val df = readFromResources( + spark, + path = "plain_number_all_cells.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false, "maxRowsInMemory" -> 1) + ) + val expected = spark.createDataFrame(allCellsPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + test("explicitly formatted cells keep display rendering when usePlainNumberFormatForAllCells=false") { + val df = readFromResources( + spark, + path = "plain_number_all_cells.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> false, "inferSchema" -> false) + ) + val expected = spark.createDataFrame(allCellsDisplayData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + test("explicitly formatted cells still round when only usePlainNumberFormat=true") { + val df = readFromResources( + spark, + path = "plain_number_all_cells.xlsx", + options = Map("usePlainNumberFormat" -> true, "inferSchema" -> false) + ) + val expected = spark.createDataFrame(allCellsGeneralPlainData, allCellsSchema) + assertDataFrameEquals(expected, df) + } + + test("numeric header cells are named consistently with their data cells") { + val plain = readFromResources( + spark, + path = "plain_number_numeric_header.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> true, "inferSchema" -> false) + ) + assert(plain.schema.fieldNames.toSeq == Seq("123456789012", "name")) + + val display = readFromResources( + spark, + path = "plain_number_numeric_header.xlsx", + options = Map("usePlainNumberFormatForAllCells" -> false, "inferSchema" -> false) + ) + assert(display.schema.fieldNames.toSeq == Seq("1.23457E+11", "name")) + } }