From deb352b02855b67455840aed2f9b8461ecaad06a Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Tue, 31 Mar 2026 23:40:35 +0200 Subject: [PATCH 1/2] perf: Clip column range instead of filtering in V2 DataLocator When dataAddress specifies only a starting cell, colInd becomes a huge range (e.g. 1 to 16383). The previous code iterated the entire range per row using .filter(), causing O(rows * maxColumns) comparisons. Replace with direct range clipping: colInd.start to min(colInd.last, lastCellNum - 1). This is O(1) per row and also hoists the getLastCellNum call out of the per-element evaluation. Fixes #720 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../mauch/spark/excel/v2/DataLocator.scala | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala b/src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala index 3d3d119e..6b4538dc 100644 --- a/src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala +++ b/src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala @@ -36,28 +36,25 @@ trait DataLocator { */ def readFrom(workbook: Workbook): Iterator[Vector[Cell]] + private def readCells(r: org.apache.poi.ss.usermodel.Row, colInd: Range): Vector[Cell] = { + val lastCellNum = r.getLastCellNum.toInt + val effectiveCols = if (lastCellNum < colInd.last + 1) colInd.start to math.min(colInd.last, lastCellNum - 1) + else colInd + effectiveCols.map(r.getCell(_, MissingCellPolicy.CREATE_NULL_AS_BLANK)).toVector + } + def actualReadFromSheet(options: ExcelOptions, sheet: Sheet, rowInd: Range, colInd: Range): Iterator[Vector[Cell]] = { if (options.keepUndefinedRows) { rowInd.iterator.map(rid => { val r = sheet.getRow(rid) if (r == null) { Vector.empty } - else { - colInd - .filter(_ < r.getLastCellNum) - .map(r.getCell(_, MissingCellPolicy.CREATE_NULL_AS_BLANK)) - .toVector - } + else { readCells(r, colInd) } }) } else { sheet.iterator.asScala .filter(r => rowInd.contains(r.getRowNum)) - .map(r => - colInd - .filter(_ < r.getLastCellNum) - .map(r.getCell(_, MissingCellPolicy.CREATE_NULL_AS_BLANK)) - .toVector - ) + .map(r => readCells(r, colInd)) .filter(_.exists(_.getCellType != CellType.BLANK)) // #965 filter rows that are completely empty } } From b5a4fcbed90a539ac9b01e1e49c4d76b28d1d518 Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Wed, 1 Apr 2026 22:11:16 +0200 Subject: [PATCH 2/2] Update src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../dev/mauch/spark/excel/v2/DataLocator.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala b/src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala index 6b4538dc..af58e71d 100644 --- a/src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala +++ b/src/main/scala/dev/mauch/spark/excel/v2/DataLocator.scala @@ -37,10 +37,15 @@ trait DataLocator { def readFrom(workbook: Workbook): Iterator[Vector[Cell]] private def readCells(r: org.apache.poi.ss.usermodel.Row, colInd: Range): Vector[Cell] = { - val lastCellNum = r.getLastCellNum.toInt - val effectiveCols = if (lastCellNum < colInd.last + 1) colInd.start to math.min(colInd.last, lastCellNum - 1) - else colInd - effectiveCols.map(r.getCell(_, MissingCellPolicy.CREATE_NULL_AS_BLANK)).toVector + if (colInd.isEmpty) { + Vector.empty + } else { + val lastCellNum = r.getLastCellNum.toInt + val effectiveCols = + if (lastCellNum < colInd.last + 1) colInd.start to math.min(colInd.last, lastCellNum - 1) + else colInd + effectiveCols.map(r.getCell(_, MissingCellPolicy.CREATE_NULL_AS_BLANK)).toVector + } } def actualReadFromSheet(options: ExcelOptions, sheet: Sheet, rowInd: Range, colInd: Range): Iterator[Vector[Cell]] = {