refactor: remove old apply_date_shifts

README.md

@@ -19,64 +19,64 @@ Helper library for enabling data studies: utilities for study enablement such as
 ### Date shifting (basic example)
 
 ```python
-from nuh_helper import shift_excel_dates
-# or: from nuh_helper.date_shift import shift_excel_dates
+from nuh_helper import shift_excel_dates_inplace
+
+
+    source_file = Path(__file__).parent / (
+        "data/structural.with-blank.xlsx" if good else "data/structural.bad-blank.xlsx"
+    )
+    output_path = tmp_path / "target.xlsx"
+    linking_table_old = tmp_path / "linking_table_old.csv"
+    linking_table_out = tmp_path / "linking_table_out.csv"
 
 # Configure which sheets and columns to shift
 sheet_configs = {
-    "patients": {
-        "patient_id_col": "patient_id",
-        "date_columns": ["dob", "date_of_diagnosis"],
-        "header_row": 1,  # Optional: zero-based row index for column names
-    },
-    "labs": {
-        "patient_id_col": "patient_id",
-        "date_columns": ["test_date"],
-        "header_row": 1,
+
+    # shift the dates in the "paige" sheet
+    "paige": {
+        # patient IDs are in the "ptid" column
+        "patient_id_col": "ptid",
+        "date_columns": [
+            # shift the dates in the DOB column
+            "dob",
+        ],
+        "text_columns": [
+            # the food column has no dates
+            "food",
+        ],
+        "header_row": 0, # Optional: zero-based row index for column names. defaults to 0
+        "skip_rows_after_header": [],
     },
+
+    # ignore the "stuff" page
+    "stuff": "skip",
 }
 
+
 # Shift dates in the Excel file
-shift_excel_dates(
+shift_excel_dates_inplace(
+    # input / output files
     input_file="input.xlsx",
     output_file="output.xlsx",
-    patient_sheet="patients",
-    patient_id_col="patient_id",
+
+    # columns used to compute a list of patients
+    patient_sheet="paige",
+    patient_id_col="ptid",
+
+
     sheet_configs=sheet_configs,
     min_shift_days=-15,  # Lower range
     max_shift_days=15,   # Upper range
     seed=42,             # For reproducibility
-    date_format="YYYY-MM-DD",
-)
-```
-
-### Excluding fixed study dates with `shift_exceptions`
-
-Some columns contain a mix of patient-specific dates (which should be shifted) and fixed study-wide dates (e.g. an end-of-study date) that must remain unchanged. Use `shift_exceptions` in `sheet_configs` to list any date values that should never be shifted:
-
-```python
-sheet_configs = {
-    "patients": {
-        "patient_id_col": "patient_id",
-        "date_columns": ["last_alive"],
-        "shift_exceptions": {
-            "last_alive": ["2024-12-31"],  # end-of-study date — never shift
-        },
-    },
-}
 
-shift_excel_dates(
-    input_file="input.xlsx",
-    output_file="output.xlsx",
-    patient_sheet="patients",
-    patient_id_col="patient_id",
-    sheet_configs=sheet_configs,
-    seed=42,
+    # optional - csv files saving how many days each patient was shifted
+    # ... do not distribute these
+    # ... and probably best to keep them somewhere
+    linking_table_path="linking_table_old.csv",
+    linking_table_output="linking_table_new.csv",
 )
 ```
 
-The exception date strings are parsed with the same flexible parser used for all date values (supports multiple formats and placeholder strings). Exceptions are matched against the parsed date, so `"2024-12-31"` and `"31-12-2024"` both match the same calendar date.
-
 ### Reproducible Shifts with Linking Table
 
 To use the same shifts across multiple runs, save and reuse a linking table:
@@ -118,8 +118,30 @@ shift_excel_dates_inplace(
     output_file="output.xlsx",
     patient_sheet="patients",
     sheet_configs=sheet_configs,
-    seed=42,
-    linking_table_output="shift_mappings.csv",
+    min_shift_days=-15,  # Lower range
+    max_shift_days=15,   # Upper range
+    seed=42,             # For reproducibility
+
+    # optional - csv files saving how many days each patient was shifted
+    # ... do not distribute these
+    # ... and probably best to keep them somewhere specific
+    linking_table_output="linking_table_first.csv",
+)
+
+# Second run: reuse the same shifts
+shift_excel_dates_inplace(
+    input_file="input_final.xlsx",
+    output_file="output_second.xlsx",
+    patient_sheet="paige",
+    patient_id_col="ptid",
+    sheet_configs=sheet_configs,
+    min_shift_days=-15,  # Lower range
+    max_shift_days=15,   # Upper range
+    seed=42,             # For reproducibility
+
+    # write to a new file this time
+    linking_table_path="linking_table_first.csv",
+    linking_table_output="linking_table_second.csv",
 )
 ```
 

nuh_helper/__init__.py

@@ -11,19 +11,11 @@
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
 from nuh_helper.date_shift import (  # noqa: E402
-    apply_date_shifts,
-    generate_shift_mappings,
-    load_shift_mappings,
-    shift_excel_dates,
     shift_excel_dates_inplace,
 )
 from nuh_helper.profile import generate_scan_report  # noqa: E402
 
 __all__ = [
-    "shift_excel_dates",
-    "shift_excel_dates_inplace",
-    "apply_date_shifts",
-    "generate_shift_mappings",
-    "load_shift_mappings",
-    "generate_scan_report",
+    generate_scan_report,
+    shift_excel_dates_inplace,
 ]

nuh_helper/book_page/__init__.py

@@ -1,4 +1,4 @@
-from nuh_helper.book_page.book import BasePage, BaseWorkBook, normalize_row
+from nuh_helper.book_page.book import BasePage, BaseWorkBook, normalize_row, row_val_t
 from nuh_helper.book_page.csv_book import CSVWorkBook
 from nuh_helper.book_page.generic import book_open
 from nuh_helper.book_page.xlsx_book import XLSXWorkBook
@@ -10,4 +10,5 @@
     BasePage,
     book_open,
     normalize_row,
+    row_val_t,
 ]

nuh_helper/book_page/book.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from collections.abc import Generator as gen
-from collections.abc import Iterable as iterator
+from collections.abc import Iterable as iterable
 from datetime import date, datetime
 from functools import cached_property
 from pathlib import Path
@@ -17,7 +17,7 @@ def __init__(self, name: str, page: object) -> None:
     @abstractmethod
     def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]:
         """performs the workbook specific yield/write of the row data"""
-        raise NotImplementedError()
+        ...
 
     def stream_rows(self) -> gen[tuple[int, row_val_t]]:
         """public method for reading a RO stream of rows from a specific page"""
@@ -41,17 +41,15 @@ def __contains__(self, name: str) -> bool:
         return name in self.keys()
 
     @abstractmethod
-    def __getitem__(self, name: str) -> gen[BasePage]:
-        raise NotImplementedError()
+    def __getitem__(self, name: str) -> BasePage: ...
 
     @abstractmethod
-    def keys(self) -> iterator[str]:
-        raise NotImplementedError()
+    def keys(self) -> iterable[str]: ...
 
     @abstractmethod
     def _stream_copy_impl(self, into: Path) -> gen[tuple[str, int, row_val_t]]:
         """performs the workbook specific yield/write of the row data"""
-        raise NotImplementedError()
+        ...
 
     def stream_copy(
         self,
@@ -71,6 +69,9 @@ def stream_copy(
         date_format = date_format if date_format else "%Y-%m-%d"
 
         for page_name, row_idx, row_data in self._stream_copy_impl(into):
+            if BaseWorkBook.normalize_values:
+                for raw_idx, raw_val in enumerate(row_data):
+                    row_data[raw_idx] = normalize_val(raw_val)
             yield page_name, row_idx, row_data
             if BaseWorkBook.normalize_values:
                 for raw_idx, raw_val in enumerate(row_data):
@@ -81,11 +82,10 @@ def stream_copy(
 
     def __iter__(self) -> gen[BasePage]:
         for name in self.keys():
-            with self[name] as page:
-                yield page
+            yield self[name]
 
 
-def normalize_val(src: object) -> object:
+def normalize_val(src: object) -> cell_val_t:
     """parse to a datetime, .strip(), turn "" into None, retain bool"""
 
     if (
@@ -131,7 +131,7 @@ def normalize_val(src: object) -> object:
     return src
 
 
-def normalize_row(row: list[str]) -> list[str]:
+def normalize_row(row: iterable[str]) -> row_val_t:
     """modifies (and returns) the passed array to drop extra whitespace and replace
     fully blank entries with None. doesn't touch non-strings"""
 

nuh_helper/book_page/csv_book.py

@@ -1,4 +1,3 @@
-import contextlib
 import csv
 from _collections_abc import dict_keys
 from collections.abc import Generator as gen
@@ -16,8 +15,7 @@ def __init__(self, path: str | Path | list[Path]) -> None:
     def keys(self) -> dict_keys:
         return self._page_files.keys()
 
-    @contextlib.contextmanager
-    def __getitem__(self, name: str) -> gen[BasePage]:
+    def __getitem__(self, name: str) -> BasePage:
 
         class CSVPage(BasePage):
             def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]:
@@ -32,6 +30,7 @@ def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]:
                         # strip bom
                         if (
                             row == 0
+                            and line
                             and isinstance(line[0], str)
                             and line[0][:1] == "\ufeff"
                         ):
@@ -40,7 +39,7 @@ def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]:
                         yield (row, line)
 
         assert isinstance(self._page_files[name], Path)
-        yield CSVPage(name, self._page_files[name])
+        return CSVPage(name, self._page_files[name])
 
     def _stream_copy_impl(self, into: Path) -> gen[tuple[str, int, row_val_t]]:
 

nuh_helper/book_page/xlsx_book.py

@@ -1,11 +1,11 @@
-import contextlib
 import shutil
 from collections.abc import Generator as gen
 from pathlib import Path
 
 import openpyxl
+from openpyxl.cell.read_only import EmptyCell
 
-from .book import BasePage, BaseWorkBook
+from .book import BasePage, BaseWorkBook, row_val_t
 
 
 class XLSXWorkBook(BaseWorkBook):
@@ -15,12 +15,12 @@ def __init__(self, file: str | Path, read_only: bool = True) -> None:
         if not file.is_file():
             raise RuntimeError(f"xlsx file not found {file}")
         self._file = file
-        self._book = openpyxl.load_workbook(file, read_only=read_only)
+        self._book = openpyxl.load_workbook(file, read_only=read_only, data_only=True)
 
     def keys(self) -> list[str]:
         return [page.title for page in self._book.worksheets]
 
-    def _stream_copy_impl(self, into: str | Path) -> gen[tuple[str, int, list[str]]]:
+    def _stream_copy_impl(self, into: str | Path) -> gen[tuple[str, int, row_val_t]]:
         if not isinstance(into, Path):
             into = Path(into)
 
@@ -41,21 +41,20 @@ def _stream_copy_impl(self, into: str | Path) -> gen[tuple[str, int, list[str]]]
                         page.cell(r + 1, c + 1).value = val
         the_copy.save(into)
 
-    @contextlib.contextmanager
     def __getitem__(self, name: str) -> gen[BasePage]:
 
         if len(name) > 31:
             raise KeyError("excel limits you to 31 character page names")
 
         class XLSXPage(BasePage):
             def _stream_rows_impl(self) -> gen[tuple[int, list[str]]]:
-                for row in range(self._page.max_row):
+                for row_idx, row_data in enumerate(self._page.iter_rows()):
                     yield (
-                        row,
+                        row_idx,
                         [
-                            self._page.cell(row + 1, col + 1).value
-                            for col in range(self._page.max_column)
+                            None if isinstance(cell, EmptyCell) else cell.internal_value
+                            for cell in row_data
                         ],
                     )
 
-        yield XLSXPage(name, self._book[name])
+        return XLSXPage(name, self._book[name])