From 46cd740f5a745cd74fdba29225c75bcac4b18537 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Wed, 16 Sep 2026 17:07:49 +0100 Subject: [PATCH 01/82] that condition shouldn't work --- nuh_helper/date_shift/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 72fb3d6..dc2d221 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -683,9 +683,6 @@ def shift_excel_dates_inplace( # ... so parsed should always succeed parsed = _parse._parse_date_value(original_value) - if shift_days is None: - continue - exc_dates = parsed_exceptions.get(col_name, set()) if exc_dates and parsed.date() in exc_dates: continue From c96fa6990ee56188a6ce2e942663dcdee3de1bf2 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Wed, 16 Sep 2026 17:09:47 +0100 Subject: [PATCH 02/82] renamed the exception to be clearer --- nuh_helper/date_shift/__init__.py | 17 +++++++++++++---- tests/test_pass_as_is.py | 6 +++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index dc2d221..c58c7c1 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -27,11 +27,20 @@ def __init__(self, page: str, id: str) -> None: self._message = message -class ShiftFoundNonDate(Exception): - def __init__(self, page: str, row: int, col: int, col_name: str, val: str) -> None: - message = f"{page=}[{row}, {col} @ {col_name=}] {val=}" +class FoundNonDateInDateColumn(Exception): + """raised when a non-date value is found in a date column + + ... and - it's not a "passthrough" exceptional value""" + + def __init__(self, page: str, row: int, col: int, head: str, value: str) -> None: + message = f"[{page=} / {head=} @ {row} {col}] {value=}" super().__init__(message) self._message = message + self._page = page + self._row = row + self._col = col + self._value = value + self._head = head # >> pr 127 Exception goes here @@ -675,7 +684,7 @@ def shift_excel_dates_inplace( ): continue - raise ShiftFoundNonDate( + raise FoundNonDateInDateColumn( sheet_name, row_idx, date_col_idx, col_name, original_value ) diff --git a/tests/test_pass_as_is.py b/tests/test_pass_as_is.py index 7d3073f..0e79c33 100644 --- a/tests/test_pass_as_is.py +++ b/tests/test_pass_as_is.py @@ -4,7 +4,7 @@ from openpyxl import load_workbook from nuh_helper import shift_excel_dates_inplace -from nuh_helper.date_shift import ShiftFoundNonDate +from nuh_helper.date_shift import FoundNonDateInDateColumn @pytest.mark.parametrize("allow_passthrough", [True, False]) @@ -41,9 +41,9 @@ def body() -> None: ) if not allow_passthrough: - with pytest.raises(ShiftFoundNonDate) as info: + with pytest.raises(FoundNonDateInDateColumn) as info: body() - assert str(info.value) == "page='page-data'[6, 2 @ col_name='dob'] val='mssing'" + assert str(info.value) == "[page='page-data' / head='dob' @ 6 2] value='mssing'" return else: sheet_configs["page-data"]["pass_as_is"] = {"dob": ["mssing"]} From 43bcfcbe7e8253ed821efdbc1ea1f93f2cc79c8c Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Wed, 16 Sep 2026 17:10:27 +0100 Subject: [PATCH 03/82] [WIP] switching the old tests to use the new method --- tests/test_date_shift.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_date_shift.py b/tests/test_date_shift.py index b4d3241..b62f088 100644 --- a/tests/test_date_shift.py +++ b/tests/test_date_shift.py @@ -11,7 +11,7 @@ apply_date_shifts, generate_shift_mappings, load_shift_mappings, - shift_excel_dates, + shift_excel_dates_inplace, ) from nuh_helper.date_shift._excel import ( _description_merged_ranges, @@ -260,11 +260,15 @@ def test_header_row_and_skip_rows_after_header_produce_correct_columns( "patients": { "patient_id_col": "patient_id", "date_columns": ["date_result"], + "text_columns": [ + "type", + "measurement", + ], "header_row": 2, "skip_rows_after_header": [3], }, } - shift_excel_dates( + shift_excel_dates_inplace( input_file=str(xlsx), output_file=str(out), patient_sheet="patients", @@ -291,6 +295,10 @@ def test_patient_sheet_uses_config_header_when_sheet_in_sheet_configs( "patients": { "patient_id_col": "patient_id", "date_columns": ["date_result"], + "text_columns": [ + "measurement", + "type", + ], "header_row": 2, "skip_rows_after_header": [3], }, @@ -299,7 +307,7 @@ def test_patient_sheet_uses_config_header_when_sheet_in_sheet_configs( Do not pass patient_header_row; patient sheet "patients" is in config with header_row=2 """ - shift_excel_dates( + shift_excel_dates_inplace( input_file=str(xlsx), output_file=str(out), patient_sheet="patients", @@ -514,10 +522,11 @@ def test_exception_date_unchanged_other_dates_shifted(self, tmp_path: Path) -> N "patients": { "patient_id_col": "patient_id", "date_columns": ["last_alive"], + "text_columns": [], "shift_exceptions": {"last_alive": ["2024-12-31"]}, } } - shift_excel_dates( + shift_excel_dates_inplace( input_file=str(xlsx), output_file=str(out), patient_sheet="patients", From 4863491fb8f84895d9c510e06ee31b7f657b4c3a Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 17 Sep 2026 10:59:10 +0100 Subject: [PATCH 04/82] use parse instead of load to classify cell as date or not --- nuh_helper/date_shift/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index c58c7c1..175a5c1 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -673,8 +673,14 @@ def shift_excel_dates_inplace( if str(original_value).strip() == "": continue + # always try to parse it; not all cells auto-load as date + parsed = _parse._parse_date_value(original_value) + + if isinstance(original_value, datetime | date): + assert parsed is not None + # block non-dates in date columns - if not isinstance(original_value, datetime | date): + if parsed is None: # check if it's one of the non-dates allowed page_config = sheet_configs[sheet_name] pass_as_is = page_config.get("pass_as_is", {}) @@ -688,10 +694,6 @@ def shift_excel_dates_inplace( sheet_name, row_idx, date_col_idx, col_name, original_value ) - # original_value is datetime | date - # ... so parsed should always succeed - parsed = _parse._parse_date_value(original_value) - exc_dates = parsed_exceptions.get(col_name, set()) if exc_dates and parsed.date() in exc_dates: continue From 6b40159efba0368b14df0e346d6b62e219ba4044 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 17 Sep 2026 15:23:13 +0100 Subject: [PATCH 05/82] updated the older tests shift_inline is now passing all of the tests --- nuh_helper/__init__.py | 2 -- nuh_helper/date_shift/__init__.py | 6 +++--- tests/test_date_shift.py | 17 +++++++++++++---- tests/test_e2e_date_shift.py | 16 ++++++++++++---- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/nuh_helper/__init__.py b/nuh_helper/__init__.py index 6e863d0..6d0c951 100644 --- a/nuh_helper/__init__.py +++ b/nuh_helper/__init__.py @@ -14,13 +14,11 @@ apply_date_shifts, generate_shift_mappings, load_shift_mappings, - shift_excel_dates, shift_excel_dates_inplace, ) from nuh_helper.profile import generate_scan_report # noqa: E402 __all__ = [ - "shift_excel_dates", "shift_excel_dates_inplace", "apply_date_shifts", "generate_shift_mappings", diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 175a5c1..b59784c 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -456,9 +456,9 @@ def shift_excel_dates_inplace( """ Shift dates in an Excel file, preserving all cell formatting. - Unlike shift_excel_dates(), this function copies the input file and then - modifies date cells directly via openpyxl, so all formatting (cell styles, - merged cells, column widths, conditional formatting, etc.) is preserved. + This function copies the input file and then modifies date cells directly + via openpyxl, so all formatting (cell styles, merged cells, column widths, + conditional formatting, etc.) is preserved. Args: input_file: Path to input Excel file. diff --git a/tests/test_date_shift.py b/tests/test_date_shift.py index b62f088..754e6ff 100644 --- a/tests/test_date_shift.py +++ b/tests/test_date_shift.py @@ -279,9 +279,14 @@ def test_header_row_and_skip_rows_after_header_produce_correct_columns( ) df = pd.read_excel(str(out), sheet_name="patients", header=2) assert list(df.columns) == ["patient_id", "measurement", "date_result", "type"] - # Data should be 2 rows (P001, P002), not 3 (no "stupid" row) - assert len(df) == 2 - assert list(df["patient_id"]) == ["P001", "P002"] + # Data should now be 3 rows (stupid, P001, P002) + # ... Pandas can see "stupid" but date shifting doesn't + assert ( + # this should be 3 these days + # ... it used to be "2" because "skip_rows_after_header" removed the rows? + len(df) == 3 + ) + assert list(df["patient_id"]) == ["stupid", "P001", "P002"] def test_patient_sheet_uses_config_header_when_sheet_in_sheet_configs( self, tmp_path: Path @@ -320,7 +325,11 @@ def test_patient_sheet_uses_config_header_when_sheet_in_sheet_configs( df = pd.read_excel(str(out), sheet_name="patients", header=2) assert "patient_id" in df.columns assert "date_result" in df.columns - assert len(df) == 2 + assert ( + # this should be 3 these days + # ... it used to be "2" because "skip_rows_after_header" removed the rows? + len(df) == 3 + ) class TestApplyDateShifts: diff --git a/tests/test_e2e_date_shift.py b/tests/test_e2e_date_shift.py index ed8c999..ce0b5ad 100644 --- a/tests/test_e2e_date_shift.py +++ b/tests/test_e2e_date_shift.py @@ -13,7 +13,7 @@ import pandas as pd -from nuh_helper import shift_excel_dates +from nuh_helper import shift_excel_dates_inplace class _ShiftKwargs(TypedDict, total=False): @@ -32,10 +32,18 @@ class _ShiftKwargs(TypedDict, total=False): "patients": { "patient_id_col": "patient_id", "date_columns": ["dob", "last_alive"], + "text_columns": ["name"], }, "results": { "patient_id_col": "patient_id", "date_columns": ["date_result"], + "text_columns": [ + "measurement", + "type", + ], + "pass_as_is": { + "date_result": ["unknown"], + }, }, } @@ -48,7 +56,7 @@ def run_shift(base: Path, **kwargs: Unpack[_ShiftKwargs]) -> tuple[Path, Path]: """ output = base / "output.xlsx" linking = base / "linking.csv" - shift_excel_dates( + shift_excel_dates_inplace( str(INPUT_FILE), str(output), patient_sheet="patients", @@ -167,7 +175,7 @@ def test_placeholder_date_becomes_null_in_output(self, output_path: Path) -> Non test5_date = output_results.loc[ output_results["patient_id"] == "Test5", "date_result" ].iloc[0] - assert pd.isna(test5_date) + assert test5_date == "unknown" class TestReproducibility: @@ -192,7 +200,7 @@ def test_linking_table_reproduces_same_shifts_on_new_file( output1, linking = run_shift(output_path, seed=42) output2 = output_path / "output2.xlsx" - shift_excel_dates( + shift_excel_dates_inplace( str(INPUT_FILE), str(output2), patient_sheet="patients", From 3f042733f06809d969d8c0d1e7c7249796138079 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 17 Sep 2026 16:37:21 +0100 Subject: [PATCH 06/82] out with the old and updated the readme --- README.md | 167 +++++++++++----------- nuh_helper/__init__.py | 10 +- nuh_helper/date_shift/__init__.py | 223 ------------------------------ tests/test_date_shift.py | 176 ----------------------- 4 files changed, 83 insertions(+), 493 deletions(-) diff --git a/README.md b/README.md index a18b158..32e2633 100644 --- a/README.md +++ b/README.md @@ -13,122 +13,108 @@ Helper library for enabling data studies: utilities for study enablement such as - **`nuh_helper.date_shift`** — Date shifting for patient data in Excel/DataFrames (consistent shifts per patient ID, reproducible via linking tables). - **`nuh_helper.profile`** - Profile a dataset into a Scan Report -## Usage +## Usage; Date Shifting ### Date shifting (basic example) ```python -from nuh_helper import shift_excel_dates -# or: from nuh_helper.date_shift import shift_excel_dates +from nuh_helper import shift_excel_dates_inplace + + + source_file = Path(__file__).parent / ( + "data/structural.with-blank.xlsx" if good else "data/structural.bad-blank.xlsx" + ) + output_path = tmp_path / "target.xlsx" + linking_table_old = tmp_path / "linking_table_old.csv" + linking_table_out = tmp_path / "linking_table_out.csv" # Configure which sheets and columns to shift sheet_configs = { - "patients": { - "patient_id_col": "patient_id", - "date_columns": ["dob", "date_of_diagnosis"], - "header_row": 1, # Optional: zero-based row index for column names - }, - "labs": { - "patient_id_col": "patient_id", - "date_columns": ["test_date"], - "header_row": 1, + + # shift the dates in the "paige" sheet + "paige": { + # patient IDs are in the "ptid" column + "patient_id_col": "ptid", + "date_columns": [ + # shift the dates in the DOB column + "dob", + ], + "text_columns": [ + # the food column has no dates + "food", + ], + "header_row": 0, # Optional: zero-based row index for column names. defaults to 0 + "skip_rows_after_header": [], }, + + # ignore the "stuff" page + "stuff": "skip", } + # Shift dates in the Excel file -shift_excel_dates( +shift_excel_dates_inplace( + # input / output files input_file="input.xlsx", output_file="output.xlsx", - patient_sheet="patients", - patient_id_col="patient_id", + + # columns used to compute a list of patients + patient_sheet="paige", + patient_id_col="ptid", + + sheet_configs=sheet_configs, min_shift_days=-15, # Lower range max_shift_days=15, # Upper range seed=42, # For reproducibility - date_format="YYYY-MM-DD", -) -``` -### Excluding fixed study dates with `shift_exceptions` - -Some columns contain a mix of patient-specific dates (which should be shifted) and fixed study-wide dates (e.g. an end-of-study date) that must remain unchanged. Use `shift_exceptions` in `sheet_configs` to list any date values that should never be shifted: - -```python -sheet_configs = { - "patients": { - "patient_id_col": "patient_id", - "date_columns": ["last_alive"], - "shift_exceptions": { - "last_alive": ["2024-12-31"], # end-of-study date — never shift - }, - }, -} - -shift_excel_dates( - input_file="input.xlsx", - output_file="output.xlsx", - patient_sheet="patients", - patient_id_col="patient_id", - sheet_configs=sheet_configs, - seed=42, + # optional - csv files saving how many days each patient was shifted + # ... do not distribute these + # ... and probably best to keep them somewhere + linking_table_path="linking_table_old.csv", + linking_table_output="linking_table_new.csv", ) ``` -The exception date strings are parsed with the same flexible parser used for all date values (supports multiple formats and placeholder strings). Exceptions are matched against the parsed date, so `"2024-12-31"` and `"31-12-2024"` both match the same calendar date. - ### Reproducible Shifts with Linking Table To use the same shifts across multiple runs, save and reuse a linking table: ```python # First run: generate and save shifts -shift_excel_dates( +shift_excel_dates_inplace( input_file="input.xlsx", output_file="output.xlsx", - patient_sheet="patients", - patient_id_col="patient_id", + patient_sheet="paige", + patient_id_col="ptid", sheet_configs=sheet_configs, - linking_table_output="shift_mappings.csv", # Save shifts - seed=42, -) + min_shift_days=-15, # Lower range + max_shift_days=15, # Upper range + seed=42, # For reproducibility -# Subsequent runs: reuse the same shifts -shift_excel_dates( - input_file="new_input.xlsx", - output_file="new_output.xlsx", - patient_sheet="patients", - patient_id_col="patient_id", - sheet_configs=sheet_configs, - linking_table_path="shift_mappings.csv", # Reuse saved shifts + # optional - csv files saving how many days each patient was shifted + # ... do not distribute these + # ... and probably best to keep them somewhere specific + linking_table_output="linking_table_first.csv", ) -``` - -### Preserving formatting with `shift_excel_dates_inplace` - -If your workbook has rich formatting (cell styles, column widths, conditional formatting, etc.) use `shift_excel_dates_inplace` instead. It copies the input file and modifies date cells directly via openpyxl, so all formatting is preserved exactly. - -```python -from nuh_helper import shift_excel_dates_inplace +# Second run: reuse the same shifts shift_excel_dates_inplace( - input_file="input.xlsx", - output_file="output.xlsx", - patient_sheet="patients", - patient_id_col="patient_id", + input_file="input_final.xlsx", + output_file="output_second.xlsx", + patient_sheet="paige", + patient_id_col="ptid", sheet_configs=sheet_configs, - seed=42, - linking_table_output="shift_mappings.csv", + min_shift_days=-15, # Lower range + max_shift_days=15, # Upper range + seed=42, # For reproducibility + + # write to a new file this time + linking_table_path="linking_table_first.csv", + linking_table_output="linking_table_second.csv", ) ``` -The function accepts the same parameters as `shift_excel_dates` except; - -- `date_format` - - not needed — the original cell format is preserved -- `"text_columns": [???],` - - a per-sheet list of columns that have text only and shouldn't have dates - - these need to be explicitly specified to prevent extra columns in the CDM - ### Passing Non Dates (in `date_columns`) Studies frequently include data that's not parsable as a date in the date columns. @@ -174,20 +160,25 @@ While this does require repeated manual intervention ... - `patient_sheet`: Name of the sheet containing patient IDs - `patient_id_col`: Name of the column containing patient IDs - `sheet_configs`: Dictionary mapping sheet names to configuration dicts, or, the string 'skip' if that sheet should be skipped but is a valid part of the CDM. - - `patient_id_col`: Patient ID column name in that sheet - - `date_columns`: List of date column names to shift - - `text_columns`: List of non-date columns to ignore - - `header_row`: (Optional) Zero-based row index for the row that contains column names - - `skip_rows_after_header`: (Optional) List of zero-based row indices to exclude from data (e.g. a data-type row immediately below the header) - - `shift_exceptions`: (Optional) Dict mapping column names to lists of date strings that should never be shifted (e.g. a fixed end-of-study date). Dates are parsed using the same flexible parser as regular date values. - - `pass_as_is`: (Optional) Dict mapping column names to lists of "non dates" that are passed through without being changed -- `patient_header_row`: (Optional) Zero-based header row for the patient sheet (default: 0). If the patient sheet is in `sheet_configs`, that sheet’s `header_row` is used instead. + +- `patient_header_row`: (Optional) Zero-based header row for the patient sheet (default: 0). If the patient sheet is in `sheet_configs`, that sheet’s `header_row` is used instead. (... or is it?) - `patient_skip_rows`: (Optional) Zero-based row indices to exclude from patient data (e.g. a data-type row). If the patient sheet is in `sheet_configs`, that sheet’s `skip_rows_after_header` is used instead. + - `min_shift_days` / `max_shift_days`: Range of days to shift (default: -15 to 15) - `linking_table_path`: (Optional) Path to existing linking table CSV for reproducibility - `linking_table_output`: (Optional) Path to save the linking table CSV - `seed`: (Optional) Random seed for generating shifts -- `date_format`: (Optional, `shift_excel_dates` only) Excel date format string (e.g., ‘YYYY-MM-DD’) + +#### Sheet Configs Per-Page Options + +- `patient_id_col`: Patient ID column name in that sheet +- `date_columns`: List of date column names to shift +- `text_columns`: + - a per-sheet list of columns that have text only and shouldn't have dates + - these need to be explicitly specified to prevent extra columns in the CDM +- `header_row`: (Optional) Zero-based row index for the row that contains column names +- `skip_rows_after_header`: (Optional) List of zero-based row indices to exclude from data (e.g. a data-type row immediately below the header) +- `pass_as_is`: (Optional) Dict mapping column names to lists of "non dates" that are passed through without being changed ### Excel layout (header row and merged cells) @@ -207,7 +198,7 @@ Sheets can have a non-standard layout: e.g. a merged title row, then a descripti - Supports flexible date parsing (handles various formats and placeholders like "Unknown") - Reproducible shifts via linking tables -### Dataset Profile +## Usage; Dataset Profile Profile a dataset and generate a Scan Report. diff --git a/nuh_helper/__init__.py b/nuh_helper/__init__.py index 6d0c951..2edb216 100644 --- a/nuh_helper/__init__.py +++ b/nuh_helper/__init__.py @@ -11,7 +11,6 @@ logging.getLogger(__name__).addHandler(logging.NullHandler()) from nuh_helper.date_shift import ( # noqa: E402 - apply_date_shifts, generate_shift_mappings, load_shift_mappings, shift_excel_dates_inplace, @@ -19,9 +18,8 @@ from nuh_helper.profile import generate_scan_report # noqa: E402 __all__ = [ - "shift_excel_dates_inplace", - "apply_date_shifts", - "generate_shift_mappings", - "load_shift_mappings", - "generate_scan_report", + generate_scan_report, + generate_shift_mappings, + load_shift_mappings, + shift_excel_dates_inplace, ] diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index b59784c..15dbafd 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -216,229 +216,6 @@ def _get_patient_ids_and_shift_mappings( return (patient_ids, shift_mappings) -def apply_date_shifts( - df: pd.DataFrame, - patient_id_col: str, - date_columns: list[str], - shift_mappings: pd.DataFrame, - date_format: str | None = None, - shift_exceptions: dict[str, list[str]] | None = None, -) -> pd.DataFrame: - """ - Apply date shifts to specified columns in a DataFrame. - - Args: - df: pd.DataFrame containing patient data. - patient_id_col: Name of the column containing patient IDs. - date_columns: List of column names containing dates to shift. - shift_mappings: DataFrame with 'patient_id' and 'shift_days' columns. - date_format: Optional date format string (e.g., 'YYYY-MM-DD'). - Note: This parameter is kept for API compatibility but formatting - is applied at the Excel cell level, not in the DataFrame. - shift_exceptions: Optional dict mapping column names to lists of date strings - that should never be shifted (e.g. a fixed end-of-study date). - - Returns: - DataFrame with shifted dates. - """ - df = df.copy() - - # Normalize patient IDs in the working DataFrame to align with mapping keys - df[patient_id_col] = df[patient_id_col].apply(_parse._normalize_patient_id) - - shift_dict = dict( - zip( - shift_mappings["patient_id"], - shift_mappings["shift_days"], - strict=True, - ) - ) - - # Pre-parse exception dates once per column - parsed_exceptions: dict[str, set[date]] = {} - if shift_exceptions: - for col, exc_values in shift_exceptions.items(): - parsed_set: set[date] = set() - for v in exc_values: - ts = _parse._parse_date_value(v) - if ts is not None: - parsed_set.add(ts.date()) - if parsed_set: - parsed_exceptions[col] = parsed_set - - for date_col in date_columns: - if date_col not in df.columns: - logger.warning( - "Date column '%s' not found in DataFrame, skipping", date_col - ) - continue - - # Parse flexible date strings (handles YYYY-DD-MM and placeholders "Unknown") - non_null_before = df[date_col].notna().sum() - df[date_col] = df[date_col].apply(_parse._parse_date_value) - parse_failures = non_null_before - sum(x is not None for x in df[date_col]) - if parse_failures > 0: - logger.debug( - "Column '%s': %d value(s) could not be parsed as dates", - date_col, - parse_failures, - ) - - # Apply shifts - exc_dates = parsed_exceptions.get(date_col, set()) - df[date_col] = df.apply( - lambda row: ( - row[date_col] # noqa: B023 - + pd.Timedelta(days=shift_dict.get(row[patient_id_col], 0)) - if row[date_col] is not None # noqa: B023 - and row[patient_id_col] in shift_dict # noqa: B023 - and row[date_col].date() not in exc_dates # noqa: B023 - else row[date_col] # noqa: B023 - ), - axis=1, - ) - - # Convert back to date-only format (removes time component) - df[date_col] = df[date_col].apply( - lambda x: ( - x.date() if isinstance(x, pd.Timestamp | datetime | date) else None - ), - ) - - return df - - -def shift_excel_dates( - input_file: str, - output_file: str, - patient_sheet: str, - patient_id_col: str, - sheet_configs: dict[str, dict[str, Any]], - min_shift_days: int = -15, - max_shift_days: int = 15, - linking_table_path: str | None = None, - linking_table_output: str | None = None, - seed: int | None = None, - patient_header_row: int = 0, - patient_skip_rows: list[int] | None = None, - date_format: str | None = None, -) -> None: - """ - Shift dates in an Excel file for patient IDs consistently across sheets. - - Args: - input_file: Path to input Excel file. - output_file: Path to output Excel file with shifted dates. - patient_sheet: Name of the sheet containing patient IDs. - patient_id_col: Name of the column containing patient IDs in the patient sheet. - sheet_configs: Dictionary mapping sheet names to configuration dicts. - Each config dict should have: - - 'patient_id_col': Name of patient ID column in that sheet - - 'date_columns': List of date column names to shift - Optional per-sheet header handling: - - 'header_row': zero-based row index of the column names (default 0) - - 'skip_rows_after_header': list of zero-based row indices to exclude - from data (e.g. a data-type row immediately below the header) - min_shift_days: Minimum number of days to shift (default: -15). - max_shift_days: Maximum number of days to shift (default: 15). - linking_table_path: Optional path to existing linking table CSV for reproducibility. - linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). - seed: Optional random seed for generating shifts. - patient_header_row: Zero-based header row index for the patient sheet (default: 0). - patient_skip_rows: Optional zero-based row indices to exclude from patient data - (e.g. a data-type row immediately below the header). - date_format: Optional Excel date format string (e.g., 'YYYY-MM-DD', 'yyyy-mm-dd'). - If None, Excel's default date format is used. - Common formats: 'YYYY-MM-DD', 'MM/DD/YYYY', 'DD-MM-YYYY', etc. - """ # noqa: E501 - logger.info("Shifting dates: '%s' → '%s'", input_file, output_file) - logger.debug( - "Shift range: %d to %d days, seed=%s", - min_shift_days, - max_shift_days, - seed, - ) - - _patient_ids, shift_mappings = _get_patient_ids_and_shift_mappings( - input_file=input_file, - patient_sheet=patient_sheet, - patient_id_col=patient_id_col, - sheet_configs=sheet_configs, - min_shift_days=min_shift_days, - max_shift_days=max_shift_days, - linking_table_path=linking_table_path, - seed=seed, - patient_header_row=patient_header_row, - patient_skip_rows=patient_skip_rows, - ) - - with pd.ExcelWriter(output_file, engine="openpyxl") as writer: - excel_file = pd.ExcelFile(input_file, engine="openpyxl") - - for sheet_name in excel_file.sheet_names: - default_header_row = 0 - header_row = default_header_row - sheet_date_columns: list[str] | None = None - skip_rows_after_header: list[int] | None = None - sheet_shift_exceptions: dict[str, list[str]] | None = None - - if sheet_name in sheet_configs: - config = sheet_configs[cast(str, sheet_name)] - sheet_patient_id_col: str = cast(str, config["patient_id_col"]) - date_columns: list[str] = cast(list[str], config["date_columns"]) - header_row = cast(int, config.get("header_row", header_row)) - skip_rows_after_header = config.get("skip_rows_after_header") - sheet_shift_exceptions = config.get("shift_exceptions") - sheet_date_columns = date_columns - logger.info( - "Shifting %d date column(s) in sheet '%s'", - len(date_columns), - sheet_name, - ) - - df, _description_df, description_rows, description_merged_ranges = ( - _excel._read_sheet_with_structure( - excel_file, - sheet_name=cast(str, sheet_name), - header_row=header_row, - input_file=input_file, - skip_rows_after_header=skip_rows_after_header, - ) - ) - - if sheet_name in sheet_configs: - if sheet_patient_id_col not in df.columns: - raise ValueError( - f"Patient ID column '{sheet_patient_id_col}' not found in sheet '{sheet_name}'" # noqa: E501 - ) - - df = apply_date_shifts( - df, - sheet_patient_id_col, - date_columns, - shift_mappings, - date_format=None, - shift_exceptions=sheet_shift_exceptions, - ) - - _excel._write_sheet_with_structure( - writer, - sheet_name=cast(str, sheet_name), - data_df=df, - description_rows=description_rows, - header_row=header_row, - date_columns=sheet_date_columns, - date_format=date_format, - description_merged_ranges=description_merged_ranges, - ) - - logger.info("Output written to '%s'", output_file) - - linking_path = linking_table_output or "shift_mappings.csv" - shift_mappings.to_csv(linking_path, index=False) - logger.info("Linking table saved to '%s'", linking_path) - - def shift_excel_dates_inplace( input_file: str, output_file: str, diff --git a/tests/test_date_shift.py b/tests/test_date_shift.py index 754e6ff..7a20f3d 100644 --- a/tests/test_date_shift.py +++ b/tests/test_date_shift.py @@ -8,7 +8,6 @@ from openpyxl import Workbook from nuh_helper.date_shift import ( - apply_date_shifts, generate_shift_mappings, load_shift_mappings, shift_excel_dates_inplace, @@ -332,181 +331,6 @@ def test_patient_sheet_uses_config_header_when_sheet_in_sheet_configs( ) -class TestApplyDateShifts: - def _make_mappings(self, patient_id: str, shift_days: int) -> pd.DataFrame: - return pd.DataFrame({"patient_id": [patient_id], "shift_days": [shift_days]}) - - def test_shifts_date_forward(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["2023-01-15"]}) - result = apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", 10) - ) - assert result["visit_date"].iloc[0] == date(2023, 1, 25) - - def test_shifts_date_backward(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["2023-06-01"]}) - result = apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", -5) - ) - assert result["visit_date"].iloc[0] == date(2023, 5, 27) - - def test_zero_shift_leaves_date_unchanged(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["2023-01-15"]}) - result = apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", 0) - ) - assert result["visit_date"].iloc[0] == date(2023, 1, 15) - - def test_unknown_patient_leaves_date_unchanged(self) -> None: - df = pd.DataFrame({"patient_id": ["P999"], "visit_date": ["2023-01-15"]}) - result = apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", 10) - ) - assert result["visit_date"].iloc[0] == date(2023, 1, 15) - - def test_placeholder_date_becomes_none(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["Unknown"]}) - result = apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", 10) - ) - assert result["visit_date"].iloc[0] is None - - def test_none_date_stays_none(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": [None]}) - result = apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", 10) - ) - assert result["visit_date"].iloc[0] is None - - def test_missing_date_column_is_skipped(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["2023-01-15"]}) - mappings = self._make_mappings("P001", 10) - # "nonexistent" should be silently skipped; "visit_date" still shifted - result = apply_date_shifts( - df, "patient_id", ["nonexistent", "visit_date"], mappings - ) - assert result["visit_date"].iloc[0] == date(2023, 1, 25) - - def test_strips_whitespace_from_patient_ids(self) -> None: - df = pd.DataFrame({"patient_id": [" P001 "], "visit_date": ["2023-01-15"]}) - result = apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", 10) - ) - assert result["visit_date"].iloc[0] == date(2023, 1, 25) - - def test_multiple_patients_shifted_independently(self) -> None: - df = pd.DataFrame( - { - "patient_id": ["P001", "P002"], - "visit_date": ["2023-01-01", "2023-01-01"], - } - ) - mappings = pd.DataFrame({"patient_id": ["P001", "P002"], "shift_days": [5, -5]}) - result = apply_date_shifts(df, "patient_id", ["visit_date"], mappings) - assert result["visit_date"].iloc[0] == date(2023, 1, 6) - assert result["visit_date"].iloc[1] == date(2022, 12, 27) - - def test_result_dates_are_date_not_datetime(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["2023-01-15"]}) - result = apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", 0) - ) - val = result["visit_date"].iloc[0] - assert isinstance(val, date) - assert not isinstance(val, datetime) - - def test_does_not_mutate_input(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["2023-01-15"]}) - original_visit = df["visit_date"].iloc[0] - apply_date_shifts( - df, "patient_id", ["visit_date"], self._make_mappings("P001", 10) - ) - assert df["visit_date"].iloc[0] == original_visit - - -class TestApplyDateShiftsExceptions: - def _make_mappings(self, patient_id: str, shift_days: int) -> pd.DataFrame: - return pd.DataFrame({"patient_id": [patient_id], "shift_days": [shift_days]}) - - def test_exception_date_is_not_shifted(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["2024-12-31"]}) - result = apply_date_shifts( - df, - "patient_id", - ["visit_date"], - self._make_mappings("P001", 10), - shift_exceptions={"visit_date": ["2024-12-31"]}, - ) - assert result["visit_date"].iloc[0] == date(2024, 12, 31) - - def test_non_exception_date_in_same_column_is_shifted(self) -> None: - df = pd.DataFrame( - { - "patient_id": ["P001", "P002"], - "visit_date": ["2024-12-31", "2023-06-01"], - } - ) - mappings = pd.DataFrame( - {"patient_id": ["P001", "P002"], "shift_days": [10, 10]} - ) - result = apply_date_shifts( - df, - "patient_id", - ["visit_date"], - mappings, - shift_exceptions={"visit_date": ["2024-12-31"]}, - ) - assert result["visit_date"].iloc[0] == date(2024, 12, 31) - assert result["visit_date"].iloc[1] == date(2023, 6, 11) - - def test_exception_only_applies_to_specified_column(self) -> None: - df = pd.DataFrame( - { - "patient_id": ["P001"], - "col_a": ["2024-12-31"], - "col_b": ["2024-12-31"], - } - ) - result = apply_date_shifts( - df, - "patient_id", - ["col_a", "col_b"], - self._make_mappings("P001", 5), - shift_exceptions={"col_a": ["2024-12-31"]}, - ) - assert result["col_a"].iloc[0] == date(2024, 12, 31) - assert result["col_b"].iloc[0] == date(2025, 1, 5) - - def test_multiple_exception_dates(self) -> None: - df = pd.DataFrame( - { - "patient_id": ["P001", "P001", "P001"], - "visit_date": ["2024-12-31", "2024-06-30", "2023-01-15"], - } - ) - result = apply_date_shifts( - df, - "patient_id", - ["visit_date"], - self._make_mappings("P001", 7), - shift_exceptions={"visit_date": ["2024-12-31", "2024-06-30"]}, - ) - assert result["visit_date"].iloc[0] == date(2024, 12, 31) - assert result["visit_date"].iloc[1] == date(2024, 6, 30) - assert result["visit_date"].iloc[2] == date(2023, 1, 22) - - def test_no_exceptions_kwarg_shifts_normally(self) -> None: - df = pd.DataFrame({"patient_id": ["P001"], "visit_date": ["2024-12-31"]}) - result = apply_date_shifts( - df, - "patient_id", - ["visit_date"], - self._make_mappings("P001", 10), - shift_exceptions=None, - ) - assert result["visit_date"].iloc[0] == date(2025, 1, 10) - - class TestShiftExcelDatesExceptionsIntegration: """Integration tests for shift_exceptions passed via sheet_configs.""" From 69c6b656fb764f41cfc0401625a8b6ad34330112 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 28 Sep 2026 11:01:08 +0100 Subject: [PATCH 07/82] renamed the parameter for clarity --- README.md | 4 ++-- nuh_helper/date_shift/__init__.py | 11 +++++------ tests/data/{passed => shift_ignore}/offsets.csv | 0 tests/data/{passed => shift_ignore}/workbook.xlsx | Bin tests/{test_pass_as_is.py => test_shift_ignore.py} | 12 ++++++------ 5 files changed, 13 insertions(+), 14 deletions(-) rename tests/data/{passed => shift_ignore}/offsets.csv (100%) rename tests/data/{passed => shift_ignore}/workbook.xlsx (100%) rename tests/{test_pass_as_is.py => test_shift_ignore.py} (87%) diff --git a/README.md b/README.md index a18b158..c1c8fbc 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ sheet_configs = { ], "header_row": 1, "skip_rows_after_header": [], - "pass_as_is": { # the parameter is here + "shift_ignore": { # the parameter is here "dob": [ # any column can have "as is" values added "missing", # the values are each listed here ] @@ -180,7 +180,7 @@ While this does require repeated manual intervention ... - `header_row`: (Optional) Zero-based row index for the row that contains column names - `skip_rows_after_header`: (Optional) List of zero-based row indices to exclude from data (e.g. a data-type row immediately below the header) - `shift_exceptions`: (Optional) Dict mapping column names to lists of date strings that should never be shifted (e.g. a fixed end-of-study date). Dates are parsed using the same flexible parser as regular date values. - - `pass_as_is`: (Optional) Dict mapping column names to lists of "non dates" that are passed through without being changed + - `shift_ignore`: (Optional) Dict mapping `page:{column:[values]}` to lists of values that `shift_excel_dates_inplace()` passes as-is with no manipulation or checking. - `patient_header_row`: (Optional) Zero-based header row for the patient sheet (default: 0). If the patient sheet is in `sheet_configs`, that sheet’s `header_row` is used instead. - `patient_skip_rows`: (Optional) Zero-based row indices to exclude from patient data (e.g. a data-type row). If the patient sheet is in `sheet_configs`, that sheet’s `skip_rows_after_header` is used instead. - `min_shift_days` / `max_shift_days`: Range of days to shift (default: -15 to 15) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 72fb3d6..eb314da 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -469,10 +469,9 @@ def shift_excel_dates_inplace( - 'skip_rows_after_header': list of zero-based row indices to exclude from data (e.g. a data-type row immediately below the header) - - pass_as_is: a map of {column name: [value list]} of non-date - strings allowed in date cells. these are passed through - unchanged. blank entries, and, whitespace on the start/end of the - strings are always allowed. + - `shift_ignore`: (Optional) Dict mapping `page:{column:[values]}` to + lists of values that are passed as-is with no manipulation or + checking. min_shift_days: Minimum number of days to shift (default: -15). max_shift_days: Maximum number of days to shift (default: 15). linking_table_path: Optional path to existing linking table CSV for reproducibility. @@ -668,8 +667,8 @@ def shift_excel_dates_inplace( if not isinstance(original_value, datetime | date): # check if it's one of the non-dates allowed page_config = sheet_configs[sheet_name] - pass_as_is = page_config.get("pass_as_is", {}) - allowed_non_dates = pass_as_is.get(col_name, []) + shift_ignore = page_config.get("shift_ignore", {}) + allowed_non_dates = shift_ignore.get(col_name, []) if (original_value is None) or ( original_value.strip() in allowed_non_dates ): diff --git a/tests/data/passed/offsets.csv b/tests/data/shift_ignore/offsets.csv similarity index 100% rename from tests/data/passed/offsets.csv rename to tests/data/shift_ignore/offsets.csv diff --git a/tests/data/passed/workbook.xlsx b/tests/data/shift_ignore/workbook.xlsx similarity index 100% rename from tests/data/passed/workbook.xlsx rename to tests/data/shift_ignore/workbook.xlsx diff --git a/tests/test_pass_as_is.py b/tests/test_shift_ignore.py similarity index 87% rename from tests/test_pass_as_is.py rename to tests/test_shift_ignore.py index 7d3073f..dd9395d 100644 --- a/tests/test_pass_as_is.py +++ b/tests/test_shift_ignore.py @@ -7,12 +7,12 @@ from nuh_helper.date_shift import ShiftFoundNonDate -@pytest.mark.parametrize("allow_passthrough", [True, False]) -def test_just_shift(allow_passthrough: bool, tmp_path: Path) -> None: +@pytest.mark.parametrize("shift_ignore", [True, False]) +def test_ignore_in_dates(shift_ignore: bool, tmp_path: Path) -> None: - source_file = Path(__file__).parent / "data/passed/workbook.xlsx" + source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" output_path = tmp_path / "target.xlsx" - linking_table_old = Path(__file__).parent / "data/passed/offsets.csv" + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -40,13 +40,13 @@ def body() -> None: linking_table_output=str(linking_table_out), ) - if not allow_passthrough: + if not shift_ignore: with pytest.raises(ShiftFoundNonDate) as info: body() assert str(info.value) == "page='page-data'[6, 2 @ col_name='dob'] val='mssing'" return else: - sheet_configs["page-data"]["pass_as_is"] = {"dob": ["mssing"]} + sheet_configs["page-data"]["shift_ignore"] = {"dob": ["mssing"]} body() workbook = load_workbook(output_path) From ba6742d99676cd2b0dd35f9b0578d7b0a10e6a7e Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 28 Sep 2026 11:55:51 +0100 Subject: [PATCH 08/82] always check sheets we're checking (for hidden dates et al) --- nuh_helper/date_shift/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index eb314da..f6c1b65 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -609,9 +609,6 @@ def shift_excel_dates_inplace( else: raise DateColumnMissing(sheet_name, col) - if not date_col_indices: - continue - # Pre-parse exception dates once per column parsed_exceptions: dict[str, set[date]] = {} shift_exceptions_config: dict[str, list[str]] | None = config.get( From ff2f4a2e36a32fb494af705ff0b3e32f92121435 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 28 Sep 2026 12:02:30 +0100 Subject: [PATCH 09/82] detects the problem --- tests/data/shift_ignore/workbook.xlsx | Bin 9960 -> 10257 bytes tests/test_shift_ignore.py | 58 +++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/tests/data/shift_ignore/workbook.xlsx b/tests/data/shift_ignore/workbook.xlsx index e67e4fc1575f0d02911f184996bcfb32cb45a969..e6febc12b0c5b7bc43a2e346d89acac0ab161661 100644 GIT binary patch delta 3531 zcmZ9PcQhOBzsF-Gw%DbJmDCQktF>wEs8LmF?+~i0W>7S03pI*rLX8qs#c0e@dljW( z)Luo48Xa8U-#zDk@4cTtpZ7V>AJ6B!p7;B`VUmuh83m&CFocL%Dga=d4gg>Q0057X z@L<#fA7>QGN9M6FveMcfg;(K-aPT^(TAHh8iU&_Tq^9$ftj9}Wy@Fp#aOcPLPw|cy zSFE*-C`K#zTE9Vh;JZ@VQbt;iXBpNVh;B~T8NRFqDavp1$T68igYxdA${+le&@c3Q z_w0^19KB^YILJMg=O7b#z7uXSQ2Mkv@+zRn+r#~}<+95xELSWf%vu>2lNBUwL~s;? z1dwlaj4QEgXVDO80|NRE`e%hs_BoP5TZY+d%diBFteNOr=KeH?WGU1PYn8ESv}9BG z7H{T+8x@!(B>hdcz=T47S6g_ccOQgaHq3}0?IO9j-I>bfS@{EIbFH~zko|7MPul_t zh52;#w=da@H`Tb2sS*CXY#(l4U%wU~X8q}}sAO>JYHPIl{(vV(@uW{TXgz{$=u2ST zVB$g0eGIwaA=A8qLrA%ZRgQkgJgO%>sb^d#n$3Sy z%+8)X4#C}s>42)uyg!G`QGF}Xpg+_%KvSJL$Pff?Eu6^5GRCGe^F)-pI4|5uINbcZ z9n*;Hmx{rO72*V)<>>7PbPh&^qK7)Yz;zJEq=$ct*9!W9ik=&}eS(KKji$oY2#BTN zkYD=CS&We4-z-CoM>>})x95Fjly+YD9ga@VuUgrnzEW<*n8TMxo!J>azKHxOgEq4Q z#_V4o9hW_R+6$2{GUmTyrXAzyEq|c+ol?@#4|6Ht(moRC=UDh;sW;`yqpzXn8$UC@ zQ=W1l!5uq8UD896E`b{i+*Am-r+{xjwB2$uRV#^I5*ZJB;>Ckd=n@ zpL2ZUQQvavh~&`K<=a)FW`$qH8#uPp8j3pSsV=ZCi!Njrm;b=w7HvC5h#dXh_qdBQFyhy1$q8o-UGm5FHQyC^l-4@_4EcAJ{Te z)PIJSc>jus+!>RJqyzxYvD8egb!7T$l=N;pHMx5iN1s4g2onTtMfO{g1^>f+$G+MM zWo!&BCsP%XMY4(*7NRs9Yh$uh=_|A2eIC(cV4yt%vP}>q%d4*sKK^*$4sww6P0LFt zF)dUln@y}SNw1b^a|9CnMAGn`liuLfyPCINYL0KOuA3!3u}r;2BTLOn&;My6=fSQ@ zUTK>AjZ7$qrvbXSKewZN=3295`;xasp=#?k&~#W=%O|#}nOX;y@U`Z7c(#s{H<*S; zzRaN%H*w`%z5`{p{Le;7klh+o8>tH$w4gCG7+yanYfBLOWOwn(?)iM2XW@VJujG0E z<+PA#%&>*IV|q=iPJ>S5J=SFB1zS?ig)gDLiA^(vMWLlL&*co}i#zu3)r^5UVt|9M zjfR4SnKu{6eUn=-TN6-6mjjD?$25QC$K_2&{z2GS@5_+jvmK3QyDmzzX?;BoQ6c+~ z1F@pv2j-I_a0<{I5BmotmNW|GC_Vuw@w)PDb=#gWy9cc1cbK`-14NF%TF>JY@|@$& z%T@d_)M(QWqx>(YcPnDfUiZd(VO)>yb;0xuk@=U{hy%tRtZ^i%$n2+ZL1MYq6rj#& zz)L`*N-MC^WkUH-S~;7CZBe5yZs(?y&ZA1IijY4%(D2gaL9?!`l~4z0_z>egnat~8 zDKY9%Xx+}{(2fvEFERXcP+Y5+t_$wSD0uOk96I#O`Tge1$8}5 zoo{(qo8PMp-$hA_d*+@QTDKL&=Q%5Oor5thB1Yt93r0T?CZ$!c=Sw=EY0F)(kx=@l zeAx6d*D=K)K@=*fbpzuO?od-z!uFWc5`OoVQF)ScCn96%SDmg7Q6{dX_gkPZHEQjs z-~0aZDjI;LVB(FY1u`Z zmj0f)7mTOVH^d5G*xEFuhFlyf8=^6os;yp;DVGd*g<#0{wjY)<8p~YOP1ft9uR)7y zujzh3>`GY~>Mwbl7v3fKf~-v+NBhL)l7Kp*bfrj<4Rwz=f|lq226!g1TH-zWAe70@ zbY;1f&pbUFP`k=AWyUF{uUas-M=#W;^d#aXUn=5Z#`l25u<_XPHMsCqc*>l+mDRt)@Nv%8B%Ia`YUDUCOh=xOoLRvP^Tv{`lzQ z@0IGC=`Lz`|1CYd0S7hSYQ_XQM`Q`VPPkaDmrRLmjv&*l=Xt^-S_Utd7F;aHG`&BsvM8&R@ zZk^lo{MZQql+)WmM4Io|t}sPE3vFqgX^;1On{O1LTO=rmGj!e>%gwE9;}BTlouUno zPKz<-f36(34Y*r$?)#=yS_R!u^_y5zf49A;gkq~H4|qaUAWdydooeF254#b8l~rVb$8|Lbcycm< zVj*QvlR=WowfB<2W%H^Ll7l&DqI+_zqaksZc_qO8AuvSNydq}fGrCO}`<*Ggz2U=U z>tqV=RM__&p_008oq~y-GQwKMw`p$SU=l*i#b8p|v9|JZ;teHDoWh)FY%D4+{%kbq zcB3l~U)&9#6-;}>9pK^W56PL&E)#`fS^n_s62nFE$|c>*Z_fXE9}H~?4;7#A9eHSwF3 zWsEzf2?c4tAh4ZN7f{FTNKqbA?@*|4_1x}Z<4^b7&TqZyZede5ar5>{7_ND$%_)8% zvn%kM$$`z~B9*s41Ga`nAPbQDj2P#4DuKoGNYS;RlRxF`=Dqdiq05%pC@Ux)AGP1h z(9PH6PhWPL^V;yau4gx^R}T6Okzi(}31Lbi>F*4U-=Xq3+>>mX)BNomZ~A9+)8A=V zg1^~YV}6upoOLl1Bjq1U?8E-G_3+$YJIx`wmhc4tQHh8Ish5eL)ywM|vnC-B+Wvuz z0yUt#I_?`M9see8b^3Xi2(xgoU-BC@PD**?UdOM8Bn1z3vSP;g(TEIKJ_}P zu&kUgvB9yftBx2ehpby2OZ&`JcpaJe9gPB+r5&lO@np>@@H3`9!?PDP{pTJhpnrZV z8rm`y7Q?y`0R;*)9v@Gn%DUkzTZ22{%6MNjT`ZKaeY9snR(-=BZam!NFx({Sz~{!! z*FN2BMI&sUM2R4A*3WQmG}*@U&{w(`*1Wbt6tUa96GNdM5X4bTNaHshjl9UO2K~1Q z4DW(XEs~@cp13OQGp~00`iKUec|Nj{KO24&g;gnZp-Vy2tzFuUe$@&d;}JihDc3NT z2T=EnMp3Uy!B{&M?t}DaG(bSNrObLYuZ2y%;C`-f4)`v<3jUF>jq_9L`hH3mXx_jAsp$Tv0%2c@noqj4U2$r^8H_g{=FIi&=R-+3dZ-J3Ajp delta 3202 zcmZ8jc{tRK7M`*1>xgDC+4rTQtYs^^ETKVmS;ttyAbyNOi>`^tTAEbXA6z&E)Vm{)5jt{FJo)K-6a`U3az#!Z{{Yj%4_I);7z zCNCCnbHL4s+6~UUjl72|fVNxI8kM+N$_?`SJgb=V&2Zvz^h=oZo9@z%<`#<4Fhu8p zjQQWVR)gH)h8j}PBi#lM`@mp+gHVpE9XGCPmPJroK0aN*vALdS-uFTM3Ns4)X}1cM zF3r;&@2O!;9j$Mgod5+uEk#`~wU@YOsCV{?H33BaonNcQs$73#1r!m8x$9jN?+$BU z5E^ZYq+V;jGSMhvhW`=~KYL)!!lb}rrD*zTI& zctNeKr!F~kvuqyy)%doc-jZ<3w)#pn_N=VB=!wB zil9VWjH8FI`zv12Re$Z9x=Op2?sA1J=5AJaz^lH`=M*$SZA`VL3?h4Vj7aTNB| zf6O9kMgtMXDyC9Xh0n;C(_~X^%e)79L#|>ymWkE!QCd?|c6fv7jNGRATP|W6E)Fje zM--OhGW+61M%s}J9!tC1@_o6iFDS-d;co3$Iz)A1PX9o!C3-*l)qLAcsGbQy1&%$o zd0lmL2J!~rA}ucV3YUB{R0)V?^@%zvl$wRGQMn4~flzt4t&QS@ed_R*#bbLjWF#4d zp>W5Y^5bzNAOZ-!*Wf}f8&fk8Fj3E7sQBz?>V)T$yX)NLWfx)PyWcO8lh{TGVGduc zFCU&ebHucmNLqzONX>;Xe{}#>1u8TOR};jGwUjI4-6hJjE|^hXSuL4rnR_{YaqM>< zIw1`62tm?SlqOre4JMf<%0AjhO$eXGpBWQlIJFaJej$=BqOF&8OVThqL42J|7%TBf z*A{D)jA&&wSxKO!CUW zQDK>1@luIj{+-@Zw=%!joY%7c~qhi;G&G)eZ$7xjK4Y- zD)4|JS#;*|uyPpXuC#KR9Zn|Y?)%Lp3ym50&i)ZKy^OQFQ1om?2+=5 zH2lhKHn*R0xInl`V*=ql1qEB@IkXr!glfC4iZOsdUzqR+ZX}Rxno*<*8U2xSh)`TX z363#=B?>KOHLus#EBlV>3~;yyXcyJT|Bg-=GMa=>XEpEqq{TiwiKJlz0>VJm_ksk! z%_~byg%vy`Sx_>5z%~;F&^L-&ok*Ee=lu@z<7s@z>V>Zy1ALNgS`|5GZ+_A!bevK} zPw+Uo$W;a84+0hPh$T69YI9v3VkN?`)}yjoLd^EJs<#arPvFbn+-dPj;-qqLG8J|$ZPhFQ zB?5c-T4WD>Xzu$}^-@;wUO(shB!ZdsEpk<19B?yn1yqSNNAgxla|rI&XWO@~pV;Yp z5FPI_hxs86MgXle+!g4vM;&rXnQ&L%%Ud1+4$WsYgt0?$4o3P}Q?DMjBxw2TP`=bA z5cL4cjkz+ay}t?taq~RR6*){I%nK@wu1(7=Nb>G zO*e7iqR=wrnXD!x$gQZs6jMl(&M4E*&e6BKtuZD!{w8n=9`SH@{y6(p;pyBFGf{Rk z5P!E9{oYUUt^TgFY0;R%k6M*#H8CbjWKDM(9QIfm?rqHFaD(Ih$SSAYq|a2@ z=Dtz23f^5%h~2mInbqze!XWYbd}jdr#o$F%t~g{fz_(v|U_0uM)3~?Zcmv78wW<8( zfyJl+dIF!_w$uzDKdX_}NqSCaz=~NqI+Z+iFXjwUnYyw{-acvN(f}@9?(0$GY%l7Fw>q?1o?&b;Lxd8x#p8m;%Sr>)sXjtwWjAD z=)Q+p`s(X+KN=2cd(ey8I-Joca5*7NT#?^N{$7`gQ>-LjyjC-dQijrLkdVJ(SDrUd zDnSOg-FY$u6yBs|N4?u+>2NWcO-s7s{UKFX(i*~VX>k4Ehcs)vC%{lUIdyXBR!=Y! z-9C$ZCUa~p&%`oaju>h?Ms8B=wUqqXMzvf9ScsF{-JU~NFYN7yau*WPvsTC$-i}+z zO2TH#g4s+4l)`zIy$#YMhm_%g&p7AzB60j?>o&h99^Pv;KBm1SeOnx|bDr+7bS_Hb za)QdHwECB0%q2vo0Lr2?BQ$RD^MqP6gYS&qN^bJSvGx`{Qm18IO}whC<1}w+?<*-R zqwxOsPOd=DL?`7$WM@)$F;mnC{;1$tBG+N0H;dB^)ERRYSOvd**c$zTRuE^2g6;nI zmLtaKVv$~bL{M&xSoeXW_qjMmp}G6#X#DMj`qSpPg;| zZS~&W)?sNFHVGL$W%4eP$AY*D^M9n6(ExX|95{{~eKKxg7Vff&Kz(pl?J1mXWQkBX zUiDHZ7|ADwS4W?ca5DFS2qr&<|2{EMgKRpH>-<*YC9plDy}#lX@=n059)jB(92v*Y zl@!1rP&or0B?Vzk8v9Wd!jZ9i4 z)4qAZ%i40Aj4_`|jjO8Qnpi=*(^gI+2_t41BJ}cxN-_|Dfy+^_mECV54DU&yL$z2kL0*BUbD>&D z7kUO!!IG_6QL-xA0TKm!#;v1;yHO<8;^(^%^ANy7bu@&N>zAI9aMdcKEs$T&$d+Eg zz&?($5XbfR@;qt-cy5o6$(cvJ>zk3<&R4>!@NDCiSiNS{Z19n1@5JNW-Z#`X1>C5Q zbc9=Hv0HV4Hv;T}2KwQR>AdtwPNdHfD&-_KN)HGe~$V`4uOAMf&YI1 X&qVR{GP(?x#PDBb#96o`|2_N{ME&CM diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index dd9395d..a9e0da3 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -4,11 +4,11 @@ from openpyxl import load_workbook from nuh_helper import shift_excel_dates_inplace -from nuh_helper.date_shift import ShiftFoundNonDate +from nuh_helper.date_shift import HiddenDate, ShiftFoundNonDate @pytest.mark.parametrize("shift_ignore", [True, False]) -def test_ignore_in_dates(shift_ignore: bool, tmp_path: Path) -> None: +def test_ignore_in_date_columns(shift_ignore: bool, tmp_path: Path) -> None: source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" output_path = tmp_path / "target.xlsx" @@ -79,3 +79,57 @@ def body() -> None: assert worksheet.cell(5, 2).value is None assert str(worksheet.cell(6, 2).value) == "mssing" # change that's under test assert str(worksheet.cell(7, 2).value) == "1999-11-30 00:00:00" + + +@pytest.mark.parametrize("shift_ignore", [True, False]) +def test_ignore_in_text_columns(shift_ignore: bool, tmp_path: Path) -> None: + + source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" + output_path = tmp_path / "target.xlsx" + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_out = tmp_path / "linking_table_out.csv" + + sheet_configs = { + "page-desc": { + "patient_id_col": "pid", + "date_columns": [], + "text_columns": ["foo", "glitter"], + "header_row": 0, + "skip_rows_after_header": [], + }, + "page-data": "skip", + } + + def body() -> None: + shift_excel_dates_inplace( + input_file=str(source_file), + output_file=str(output_path), + patient_sheet="page-desc", + patient_id_col="pid", + sheet_configs=sheet_configs, + min_shift_days=-20, + max_shift_days=-1, + seed=14333, + linking_table_path=str(linking_table_old), + linking_table_output=str(linking_table_out), + ) + + if not shift_ignore: + print(output_path) + with pytest.raises(HiddenDate) as info: + body() + assert str(info.value._message) == ( + "hidden date in [sheet_name='page-desc', 3, 3] " + + 'value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"' + + " // found=datetime.datetime(2001, 12, 11, 0, 0)" + ) + return + + else: + raise Exception("add the exception") + + # sheet_configs["page-data"]["shift_ignore"] = {"dob": ["mssing"]} + body() + + # workbook = load_workbook(output_path) + raise Exception("do checks?") From 1789a147b4f317fd88ca3b97874e44fdcd128b9d Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 28 Sep 2026 13:15:39 +0100 Subject: [PATCH 10/82] more detail for the error message --- nuh_helper/date_shift/__init__.py | 16 ++++++++++--- tests/test_hidden_dates.py | 6 ++--- tests/test_shift_ignore.py | 37 ++++++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index f6c1b65..6a6a79f 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -43,14 +43,24 @@ def __init__(self, page: str, row: int, col: int, col_name: str, val: str) -> No class HiddenDate(Exception): def __init__( - self, sheet_name: str, row: int, col: int, value: str, found: datetime + self, + sheet_name: str, + row: int, + col: int, + col_name: str, + value: str, + found: datetime, ) -> None: - message = f"hidden date in [{sheet_name=}, {row}, {col}] {value=} // {found=}" + message = ( + f"hidden date in [{sheet_name=}, {row}, {col} @ {col_name}] {value=} " + + f"// {found=}" + ) super().__init__(message) self._message = message self._sheet_name = sheet_name self._row = row self._col = col + self._col_name = col_name self._value = value self._found = found @@ -704,7 +714,7 @@ def shift_excel_dates_inplace( for found in datefinder.find_dates(value): raise HiddenDate( - sheet_name, row_idx, non_date_col_idx, value, found + sheet_name, row_idx, non_date_col_idx, col_name, value, found ) wb.save(output_file) diff --git a/tests/test_hidden_dates.py b/tests/test_hidden_dates.py index e79d819..b93ddf8 100644 --- a/tests/test_hidden_dates.py +++ b/tests/test_hidden_dates.py @@ -44,7 +44,7 @@ def test_iso8601(tmp_path: Path) -> None: ) assert info.value._message == ( - "hidden date in [sheet_name='args', 4, 3]" + "hidden date in [sheet_name='args', 4, 3 @ dob]" + " value='hives on 2023-10-12'" + " // found=datetime.datetime(2023, 10, 12, 0, 0)" ) @@ -85,7 +85,7 @@ def test_us_date(tmp_path: Path) -> None: ) assert info.value._message == ( - "hidden date in [sheet_name='data', 6, 3]" + "hidden date in [sheet_name='data', 6, 3 @ dob]" + ' value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"' + " // found=datetime.datetime(2001, 12, 11, 0, 0)" ) @@ -126,7 +126,7 @@ def test_written(tmp_path: Path) -> None: ) assert info.value._message == ( - "hidden date in [sheet_name='yeah', 5, 3]" + "hidden date in [sheet_name='yeah', 5, 3 @ dob]" + " value='flu on mar 21st, 2009'" + " // found=datetime.datetime(2009, 3, 21, 0, 0)" ) diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index a9e0da3..16ee913 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -119,17 +119,42 @@ def body() -> None: with pytest.raises(HiddenDate) as info: body() assert str(info.value._message) == ( - "hidden date in [sheet_name='page-desc', 3, 3] " + "hidden date in [sheet_name='page-desc', 3, 3 @ glitter] " + 'value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"' + " // found=datetime.datetime(2001, 12, 11, 0, 0)" ) return else: - raise Exception("add the exception") - - # sheet_configs["page-data"]["shift_ignore"] = {"dob": ["mssing"]} + # add the exception + sheet_configs["page-desc"]["shift_ignore"] = { + "glitter": [ + "can't recall the date but on 12/11/2001 they had an itchy tummy" + ] + } + + # run the shift body() - # workbook = load_workbook(output_path) - raise Exception("do checks?") + workbook = load_workbook(output_path) + page = workbook["page-desc"] + + obtained = [ + [str(page.cell(row + 1, col + 1).value) for col in range(page.max_column)] + for row in range(page.max_row) + ] + + expected = [ + ["foo", "pid", "glitter"], + ["bar", "nuh71", "road"], + [ + "shoe", + "nuh06", + "can't recall the date but on 12/11/2001 they had an itchy tummy", + ], + ["agreed on 21st jul 2017", "nuh23", "grip"], + ["farm", "nuh67", "grim"], + ["cake", "nuh27", "cheese"], + ] + + assert expected == obtained From 920b301f71d59c7969df820cf72b0f2660b2b102 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 28 Sep 2026 13:31:27 +0100 Subject: [PATCH 11/82] validates what it should --- nuh_helper/date_shift/__init__.py | 57 ++++++++++++++++++++----------- tests/test_shift_ignore.py | 7 ++-- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 6a6a79f..def84b6 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import Any, cast +import datefinder import pandas as pd from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet @@ -529,25 +530,31 @@ def shift_excel_dates_inplace( if sheet_name not in sheet_configs: raise ExtraPage(sheet_name) - for sheet_name, config in sheet_configs.items(): + for sheet_name, sheet_config in sheet_configs.items(): if sheet_name not in wb.sheetnames: raise PageMissing(sheet_name) - if isinstance(config, str) and config == "skip": + if isinstance(sheet_config, str) and sheet_config == "skip": # it's a skipped sheet continue - assert isinstance(config, dict) + assert isinstance(sheet_config, dict) - assert "text_columns" in config, f"no text_columns setting for {sheet_name=}" - assert "date_columns" in config, f"no date_columns setting for {sheet_name=}" + assert "text_columns" in sheet_config, ( + f"no text_columns setting for {sheet_name=}" + ) + assert "date_columns" in sheet_config, ( + f"no date_columns setting for {sheet_name=}" + ) ws = cast(Worksheet, wb[sheet_name]) - sheet_patient_id_col: str = cast(str, config["patient_id_col"]) - date_columns: list[str] = cast(list[str], config["date_columns"]) - text_columns: list[str] = cast(list[str], config["text_columns"]) - header_row: int = cast(int, config.get("header_row", 0)) - skip_rows_after_header: list[int] | None = config.get("skip_rows_after_header") + sheet_patient_id_col: str = cast(str, sheet_config["patient_id_col"]) + date_columns: list[str] = cast(list[str], sheet_config["date_columns"]) + text_columns: list[str] = cast(list[str], sheet_config["text_columns"]) + header_row: int = cast(int, sheet_config.get("header_row", 0)) + skip_rows_after_header: list[int] | None = sheet_config.get( + "skip_rows_after_header" + ) assert sheet_patient_id_col not in date_columns, ( f"{sheet_patient_id_col=} shouldn't be in date_columns of {sheet_name=}" @@ -593,16 +600,18 @@ def shift_excel_dates_inplace( elif ( # check the config to see if we know what to do with this column - (col_name not in config["date_columns"]) - and (col_name not in config["text_columns"]) - and (col_name != config["patient_id_col"]) + (col_name not in sheet_config["date_columns"]) + and (col_name not in sheet_config["text_columns"]) + and (col_name != sheet_config["patient_id_col"]) ): raise ExtraColumn(sheet_name, col_name) else: # happy normal column col_indexes[col_name] = col_index - for text_column in [config["patient_id_col"]] + config["text_columns"]: + for text_column in [sheet_config["patient_id_col"]] + sheet_config[ + "text_columns" + ]: if text_column not in header_values: raise TextColumnMissing(sheet_name, text_column) @@ -621,7 +630,7 @@ def shift_excel_dates_inplace( # Pre-parse exception dates once per column parsed_exceptions: dict[str, set[date]] = {} - shift_exceptions_config: dict[str, list[str]] | None = config.get( + shift_exceptions_config: dict[str, list[str]] | None = sheet_config.get( "shift_exceptions" ) if shift_exceptions_config: @@ -710,12 +719,22 @@ def shift_excel_dates_inplace( continue value = str(ws.cell(row=row_idx, column=non_date_col_idx).value) - import datefinder for found in datefinder.find_dates(value): - raise HiddenDate( - sheet_name, row_idx, non_date_col_idx, col_name, value, found - ) + # skip ignored values + sheet_ignored = sheet_config.get("shift_ignore", {}) + col_ignored = sheet_ignored.get(col_name, []) + if value is None or value == "" or value in col_ignored: + pass + else: + raise HiddenDate( + sheet_name, + row_idx, + non_date_col_idx, + col_name, + value, + found, + ) wb.save(output_file) logger.info("Output written to '%s'", output_file) diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 16ee913..e4bda61 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -129,8 +129,11 @@ def body() -> None: # add the exception sheet_configs["page-desc"]["shift_ignore"] = { "glitter": [ - "can't recall the date but on 12/11/2001 they had an itchy tummy" - ] + "can't recall the date but on 12/11/2001 they had an itchy tummy", + ], + "foo": [ + "agreed on 21st jul 2017", + ], } # run the shift From 054756988346fdd4409bdf5d99e98485b2bcefae Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 28 Sep 2026 14:51:17 +0100 Subject: [PATCH 12/82] fixes ignores to skip temp xlsx files --- .gitignore | 3 --- tests/data/.gitignore | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index bbd53dc..c7f194b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,3 @@ tests/output/ *.jsonl.tar.gz .DS_Store - -# ignore temp excel files -~* diff --git a/tests/data/.gitignore b/tests/data/.gitignore index 58eac15..4d4b9b6 100644 --- a/tests/data/.gitignore +++ b/tests/data/.gitignore @@ -1,2 +1,5 @@ !*.csv !*.xlsx + +# ignore temp excel files +~$*.xlsx From 34469d515f09745b86e638c59fd5851c0f02317b Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 28 Sep 2026 15:22:14 +0100 Subject: [PATCH 13/82] passes all tests again --- nuh_helper/date_shift/__init__.py | 254 +++++++++++++----------------- tests/test_hidden_dates.py | 10 +- tests/test_shift_ignore.py | 4 +- tests/test_structural.py | 4 +- 4 files changed, 118 insertions(+), 154 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index def84b6..4794aee 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -452,8 +452,8 @@ def shift_excel_dates_inplace( linking_table_path: str | None = None, linking_table_output: str | None = None, seed: int | None = None, - patient_header_row: int = 0, - patient_skip_rows: list[int] | None = None, + header_row: int = 0, + skip_rows: list[int] | None = None, ) -> None: """ Shift dates in an Excel file, preserving all cell formatting. @@ -510,8 +510,8 @@ def shift_excel_dates_inplace( max_shift_days=max_shift_days, linking_table_path=linking_table_path, seed=seed, - patient_header_row=patient_header_row, - patient_skip_rows=patient_skip_rows, + patient_header_row=header_row, + patient_skip_rows=skip_rows, ) shift_dict: dict[str, int] = dict( @@ -548,194 +548,158 @@ def shift_excel_dates_inplace( ) ws = cast(Worksheet, wb[sheet_name]) - sheet_patient_id_col: str = cast(str, sheet_config["patient_id_col"]) + patient_id_col: str = cast(str, sheet_config["patient_id_col"]) date_columns: list[str] = cast(list[str], sheet_config["date_columns"]) text_columns: list[str] = cast(list[str], sheet_config["text_columns"]) header_row: int = cast(int, sheet_config.get("header_row", 0)) - skip_rows_after_header: list[int] | None = sheet_config.get( - "skip_rows_after_header" - ) + skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) + + if "shift_exceptions" in sheet_config: + raise ValueError("??? are we abandoning shift_exceptions?") - assert sheet_patient_id_col not in date_columns, ( - f"{sheet_patient_id_col=} shouldn't be in date_columns of {sheet_name=}" + assert patient_id_col not in date_columns, ( + f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" ) - assert sheet_patient_id_col not in text_columns, ( - f"{sheet_patient_id_col=} shouldn't be in text_columns of {sheet_name=}" + assert patient_id_col not in text_columns, ( + f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" ) - max_col = ws.max_column or 0 - if not max_col: - continue - - header_row_1based = header_row + 1 - header_values = _excel._get_row_values_resolving_merged( - ws, header_row_1based, max_col + col_names = _excel._get_row_values_resolving_merged( + ws, header_row + 1, ws.max_column + 1 ) - col_indexes: dict[str, int] = {} - for col_index, col_name in enumerate(header_values, start=1): - # simplify the column name - col_name = str(col_name).strip() if col_name is not None else "" + # find patient_id_idx and normalise the column name list + patient_id_idx = None + for col_idx in range(len(col_names)): + col_name = col_names[col_idx] + if col_name is None: + continue + col_name = col_name.strip() if col_name == "": - # there's no column name - the column should be blank - # loop through the values in that column to be sure they're all empty - # check each row - for row in range(ws.max_row): - row += 1 - - value = ws.cell(row, col_index).value - - # blank is good - if value is None: - continue - - # empty strings are also fine - if str(value).strip() == "": - continue - - # raise an error - raise BlankColumnHasData(sheet_name, row, col_index, value) - - # we don't do more work on blank columns - - elif ( + col_names[col_idx] = None + continue + col_names[col_idx] = col_name + if ( # check the config to see if we know what to do with this column (col_name not in sheet_config["date_columns"]) and (col_name not in sheet_config["text_columns"]) and (col_name != sheet_config["patient_id_col"]) ): raise ExtraColumn(sheet_name, col_name) - else: - # happy normal column - col_indexes[col_name] = col_index - - for text_column in [sheet_config["patient_id_col"]] + sheet_config[ - "text_columns" - ]: - if text_column not in header_values: - raise TextColumnMissing(sheet_name, text_column) + if col_name == patient_id_col: + patient_id_idx = col_idx - if sheet_patient_id_col not in col_indexes: - raise ValueError( - f"Patient ID column '{sheet_patient_id_col}' not found in sheet '{sheet_name}'" # noqa: E501 - ) + if patient_id_idx is None: + raise NameError(f"{patient_id_col=} not found in {sheet_name=}") - pid_col_idx = col_indexes[sheet_patient_id_col] - date_col_indices: dict[str, int] = {} - for col in date_columns: - if col in col_indexes: - date_col_indices[col] = col_indexes[col] - else: - raise DateColumnMissing(sheet_name, col) - - # Pre-parse exception dates once per column - parsed_exceptions: dict[str, set[date]] = {} - shift_exceptions_config: dict[str, list[str]] | None = sheet_config.get( - "shift_exceptions" - ) - if shift_exceptions_config: - for exc_col, exc_values in shift_exceptions_config.items(): - parsed_set: set[date] = set() - for v in exc_values: - ts = _parse._parse_date_value(v) - if ts is not None: - parsed_set.add(ts.date()) - if parsed_set: - parsed_exceptions[exc_col] = parsed_set - - skip_row_set: set[int] = ( - {idx + 1 for idx in skip_rows_after_header} - if skip_rows_after_header - else set() - ) + for text_column in sheet_config["text_columns"]: + if text_column not in col_names: + raise TextColumnMissing(sheet_name, text_column) + for date_columns in sheet_config["date_columns"]: + if date_columns not in col_names: + raise DateColumnMissing(sheet_name, date_columns) - logger.info( - "Shifting %d date column(s) in sheet '%s'", - len(date_col_indices), - sheet_name, - ) + # for each row + for row_idx in range(ws.max_row): + if row_idx <= header_row: + continue - data_start_1based = header_row + 2 - for row_idx in range(data_start_1based, (ws.max_row or 0) + 1): - if row_idx in skip_row_set: + if row_idx in skip_rows: continue - pid_cell = ws.cell(row=row_idx, column=pid_col_idx) - pid = _parse._normalize_patient_id(pid_cell.value) + # get the patient id + pid_cell = ws.cell(row=row_idx + 1, column=patient_id_idx + 1) + for found in datefinder.find_dates(str(pid_cell.value)): + raise HiddenDate( + sheet_name, + row_idx, + patient_id_idx, + patient_id_col, + pid_cell.value, + found, + ) + + pid = _parse._normalize_patient_id(pid_cell.value) if pid is None: - # skip rows with no person id + for col_idx in range(ws.max_column): + cell = ws.cell(row=row_idx + 1, column=col_idx + 1) + original_value = cell.value + if original_value is None: + original_value = "" + else: + original_value = original_value.strip() + if original_value != "": + raise ValueError("row has no patient - row should be blank") continue + # determine how many days to shift shift_days = shift_dict.get(pid) if shift_days is None: raise UnknownPatient(sheet_name, pid) - for col_name, date_col_idx in date_col_indices.items(): - cell = ws.cell(row=row_idx, column=date_col_idx) - original_value = cell.value - - # skip blank values - if str(original_value).strip() == "": + # now scan each column + for col_idx in range(ws.max_column): + # skip the patient id column + if col_idx == patient_id_idx: continue - # block non-dates in date columns - if not isinstance(original_value, datetime | date): - # check if it's one of the non-dates allowed - page_config = sheet_configs[sheet_name] - shift_ignore = page_config.get("shift_ignore", {}) - allowed_non_dates = shift_ignore.get(col_name, []) - if (original_value is None) or ( - original_value.strip() in allowed_non_dates - ): - continue - - raise ShiftFoundNonDate( - sheet_name, row_idx, date_col_idx, col_name, original_value - ) + col_name = col_names[col_idx] - # original_value is datetime | date - # ... so parsed should always succeed - parsed = _parse._parse_date_value(original_value) + cell = ws.cell(row=row_idx + 1, column=col_idx + 1) + original_value = cell.value + if isinstance(original_value, str) and original_value.strip() == "": + original_value = None + cell.value = original_value - if shift_days is None: + if original_value is None: continue - exc_dates = parsed_exceptions.get(col_name, set()) - if exc_dates and parsed.date() in exc_dates: - continue + if col_name is None: + raise BlankColumnHasData( + sheet_name, row_idx, col_idx, original_value + ) - shifted = parsed + pd.Timedelta(days=shift_days) - if isinstance(original_value, date) and not isinstance( - original_value, datetime + # skip shift_ignore cell values + if original_value in sheet_config.get("shift_ignore", {}).get( + col_name, [] ): - cell.value = cast(Any, shifted.to_pydatetime().date()) - else: - cell.value = cast(Any, shifted.to_pydatetime()) - - # check for dates in non-date columns - for non_date_col_idx in range(1, 1 + (ws.max_column or 0)): - if non_date_col_idx in date_col_indices.values(): continue - value = str(ws.cell(row=row_idx, column=non_date_col_idx).value) - - for found in datefinder.find_dates(value): - # skip ignored values - sheet_ignored = sheet_config.get("shift_ignore", {}) - col_ignored = sheet_ignored.get(col_name, []) - if value is None or value == "" or value in col_ignored: - pass - else: + if col_name in sheet_config["text_columns"]: + # check for dates in non-date columns + for found in datefinder.find_dates(str(original_value)): raise HiddenDate( sheet_name, row_idx, - non_date_col_idx, + col_idx, col_name, - value, + original_value, found, ) + if col_name in sheet_config["date_columns"]: + # original_value can be str | datetime | date + # ... so parsed might not always succeed + parsed = _parse._parse_date_value(original_value) + + if parsed is None: + raise ShiftFoundNonDate( + sheet_name, row_idx, col_idx, col_name, original_value + ) + + shifted = parsed + pd.Timedelta(days=shift_days) + + # store it as either "DATE" or "DATETIME" + if isinstance(original_value, date) and not isinstance( + original_value, datetime + ): + cell.value = cast(Any, shifted.to_pydatetime().date()) + else: + cell.value = cast(Any, shifted.to_pydatetime()) + + continue + wb.save(output_file) logger.info("Output written to '%s'", output_file) diff --git a/tests/test_hidden_dates.py b/tests/test_hidden_dates.py index b93ddf8..373f516 100644 --- a/tests/test_hidden_dates.py +++ b/tests/test_hidden_dates.py @@ -44,7 +44,7 @@ def test_iso8601(tmp_path: Path) -> None: ) assert info.value._message == ( - "hidden date in [sheet_name='args', 4, 3 @ dob]" + "hidden date in [sheet_name='args', 3, 2 @ diagnosis]" + " value='hives on 2023-10-12'" + " // found=datetime.datetime(2023, 10, 12, 0, 0)" ) @@ -85,7 +85,7 @@ def test_us_date(tmp_path: Path) -> None: ) assert info.value._message == ( - "hidden date in [sheet_name='data', 6, 3 @ dob]" + "hidden date in [sheet_name='data', 5, 2 @ diagnosis]" + ' value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"' + " // found=datetime.datetime(2001, 12, 11, 0, 0)" ) @@ -126,7 +126,7 @@ def test_written(tmp_path: Path) -> None: ) assert info.value._message == ( - "hidden date in [sheet_name='yeah', 5, 3 @ dob]" + "hidden date in [sheet_name='yeah', 4, 2 @ diagnosis]" + " value='flu on mar 21st, 2009'" + " // found=datetime.datetime(2009, 3, 21, 0, 0)" ) @@ -167,7 +167,7 @@ def test_hidden_in_patient_id(tmp_path: Path) -> None: ) assert info.value._sheet_name == "data" - assert info.value._row == 5 - assert info.value._col == 1 + assert info.value._row == 4 + assert info.value._col == 0 assert info.value._value == "nuh012 12/11/2001" assert info.value._found == datetime(2001, 12, 11) diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index e4bda61..e70e464 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -43,7 +43,7 @@ def body() -> None: if not shift_ignore: with pytest.raises(ShiftFoundNonDate) as info: body() - assert str(info.value) == "page='page-data'[6, 2 @ col_name='dob'] val='mssing'" + assert str(info.value) == "page='page-data'[5, 1 @ col_name='dob'] val='mssing'" return else: sheet_configs["page-data"]["shift_ignore"] = {"dob": ["mssing"]} @@ -119,7 +119,7 @@ def body() -> None: with pytest.raises(HiddenDate) as info: body() assert str(info.value._message) == ( - "hidden date in [sheet_name='page-desc', 3, 3 @ glitter] " + "hidden date in [sheet_name='page-desc', 2, 2 @ glitter] " + 'value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"' + " // found=datetime.datetime(2001, 12, 11, 0, 0)" ) diff --git a/tests/test_structural.py b/tests/test_structural.py index ac31a61..3899e88 100644 --- a/tests/test_structural.py +++ b/tests/test_structural.py @@ -58,8 +58,8 @@ def body() -> None: with pytest.raises(BlankColumnHasData) as info: body() assert info.value._page == "paige" - assert info.value._row == 3 - assert info.value._col == 2 + assert info.value._row == 2 + assert info.value._col == 1 assert info.value._value == "forbidden" From 45ab14dca110d3f16e8244fba61027795c8f010e Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 29 Sep 2026 15:18:36 +0100 Subject: [PATCH 14/82] checks for padded cell values --- tests/data/shift_ignore/workbook.xlsx | Bin 10257 -> 10365 bytes tests/test_shift_ignore.py | 1 + 2 files changed, 1 insertion(+) diff --git a/tests/data/shift_ignore/workbook.xlsx b/tests/data/shift_ignore/workbook.xlsx index e6febc12b0c5b7bc43a2e346d89acac0ab161661..6937d153dd834e07f57908d0623990b91930e766 100644 GIT binary patch delta 3608 zcmZYCcQ_l|8UXN!8Ld5%+9P(U*&1o>t+r~^h`p;#s8xchQZr&xM2)JQpww2gDky5z zR@7*1N^$!=_ulV&?sLyy=Q;2DoPW;qJMa5qjL(g$hbhoc$zeO+NCAKa5CA|20008K zZut8kJso^}JS76Wy~^)B@OdrI9BSu&NxC#!M}h)*C`ORgrcRrT_t!QVrCu?t`RE<* zZfdbVzm}mA)bEkH?UgdQyS?vE{6!2Oy)pAvHn(9*!))m8W@H9M6+sYu_~GmCRy`90 z+bah@;zxAxDr!H|e>I&jk5-zM-P>(-=b{0;X=;QEk4HiL;T>XMUdf7A**nkny%d3b zr2JZ!OlTI9x3q}yR)bkfnPe+P8go4xDWBk59uV9Vvn}I{UJ)=;X%o|!x$9=}Wbbp+;VppL(K z?Qi*t>xB}ql^xSZVr9OCdZWVdl4NV)+@ZVNQG-!!*-0Vn#|4oVu>tkYON5r3szS`Z z9C}l~4#;b?gBlgASC5_pbw=xTI}bA3$<_fm6P}#EU8UI490}`;_q_1G0e#U03<<9o z#$R_jo@Ft6J~DuOxb%|~lK6nuX(xq|v+@kc((Gk#8a~BI%yfVxJ68ONyeHf1Q#e4o zzoHzS?mW#8vBtk{IlCOW-EJF&RjEpmMK4_b+5<)$#NSr!Q-S=-7+F&kB-*^{AATQP zv z7^wn~Y-<+_OE^GJ+U9rNG?Cpd%PlUKT2TF}!Y#W39z6d>cQj*qK|4-{d8A#_eW|dO zDNvP>l2rM3)Pk{3`1&yWVG_T1@PhD~4Mm+z`f|r+F|T?^JZ$JioIDJm7=!{)FIUYfYSFrY4Zk?sPQEWMZPoVe51w7Qw$$J5_I3OQ*NU~f`2;K?l#9||ZVQ9-Dcp6-c(@o$*m8_DCd-a%Wd>(aJby+0Ext6sO=>(3iyIpO9! zm0Hk`B}QX4`W~|~n&#!9ZF_rz%ao(N0N-ujN(try8=NBuEW_0sBvEVTprjO^pMLB8O_!GPG ziCyBIKz0k(Ya}W2QU5~wGW}&^-n`e3fj?3y&A$zFJT66u% z?kuD9sbx1PQN>LT9XGpn~ep%$)k!m%P z%aA}E3x-6TI1eyCZzc~hn>Z=vBh}tXL_XZ0`<#$dc2;_{d9*(2^WnK(H4?aZChpieQlIxGJ>8wT(NRH)h}`uJ$gB51KvNqKi2&*n9p=sNbhA8LVtwu^bqOVH>X^+NiPhw-DObMZW@BMT-Ej#-P5D{OZ8?ns*}G04 z`#BKJ_M)`jBFjL2^R@`onJ=>SYW@slPN?1IxsPTd?{d2(tir%&p#q~(f><<7yP`O z>igR>jL^O0U#hd*0O8Kx{w9~bbcDtfxb8u}AZTLXUQ~4Br z(LF|3C|FO4Aj@4Q|EiXLy{$3v4J zoqV;bLAjSakX`~*35g0ri>I93(IeJ&ARkosD|P&)_>`hAK?kGVv70Bnn=5gbrRJQu zHppr#pY@_uc{-BFS33)b-1WI!JQQ7h)F~(ig)=S;!?jRe6_%-?p~ycHJPZ4Oa+l2#g;60oOZn8D*R(?RPh%gQ%RfxFV+pETC(@4kJmiXri62&i;D-ayth%Gti6 zi}rYA-~LWubhNVva7|k4vzGZt>ao~HSSy5s^XUwF{h<+;rB;b0B~y_{^pi_5>CBdJ zx|8~V^GYd|!I2ZQCZq2j`(lbxW*%;o&wSK-ys>454x@60rv{aA-f&OM*oW#|F@Y(< z4UHdxIOCpVj|f#ZJlEr+$wH`}%!JFM10|fjsm%}5fGY?d>fo#A7%JS2TluWdf`<-p z8Ui8`naykHg~g_J9UWQ}KIju=1GA}qr<|5Z`6UCQuU_Dm*LJ07 zX@{B`_M@mFp9BD)C&R+I*wJPK55G!YZ3eF^)4xARIjPw$rt(@d&bJ#51xzeieY8pu z`{@3vs><4q+TCw7Da9qq>3FYqa7RnEe9cxau@1O2;yGYe6?$=Uv11Xg;S^7B-5A#@ zu#HOOR>V-1rWIKV_Izu6>m<8k%yq7o!)M9O(c?>FCZNwV<`7+wU52i&97u~g*IQ(VMEVmw7 zl>3;XEY=w0juvUH=Vgn306)J-pg+e$*Y*$*DB(a&HHG(k@#j87dm-eSh3#M?eMb8c z*ZsVGZL?J~6s4C@H~P3NW1$7BmRNJGJQsF;7JfgfE3!*4A);?zQ~BdOd~rpVOWukLoj1am zRC-cAi)LxVT8dX!x8sq|VtUTerZ*ASCFyT(*Xjy?WT2nPQLiuDjA2`Vu*efVdLQh7vGz>NWJI2>_k>}9>(vbnqiU$vgsT%oI^J5TMiCJs(rqoe zjm6D3tL5q_mi4N`g+%lliNhTVcctoo1Cpoulv2R{b(l}TUQam=o+!#I;hq2@!Nk5GX&PSjX?i2XSM0Iv`+~3`o+PoRM z*u>a%{_EYZ>4kn93>O{W&WM}civG>OOc;z+EJX;K`P_+1aFj3MyRTmV?~0Ji-lyz! zZAt0OmyBS0yzr9J1`c|Sp$!oDvZ_`4f`H(&BL)3ESYi7F3`xeYU_o(?zq>0c008o5 z*!pMD(qMUM*swu@^c;T|oc~L@L;oj<1=}QegQOU{DX2n{h`lbv!~Tyi0D$?QEC1}m ou&zQv9Dlv1-XDLSfBHIx|B0LOVBZSeCK<)93-MFn1pbQs3t^(PMF0Q* delta 3508 zcmZ8kXE+;<*Nv6fLX6rwVusqKHmw~ss!Hu0VpP?rp!iW+s8LkKE4>@sAX+yb_hf+#0GOfz02lxOz+)sl7!}~{ zghF{sJ@!FXTiKzoO3Y8}J+H`C7aQr~z|#-OsXW9Rv0@m{;Fn@-`O!nO98)D#o1NqG zQF1<3uaWN9-jvRi@%Hltnk{?0tD`E-=glB_*&TKnIy2#*yt}EgCqKpX3O(OGyDJJu z?;4DZuubOKOGRAmhno+V|I-?A9Z=-u?p9{8=DYySg@uG!DHca(1xXs>9ALcugxlRy z3J|R<3OuF1|KQ2c0{_J^b5dyg7^J=egJaH`kGgH_XH zlF4XCg$a&-e~_%Y;)9-rs2ei zCBH280tq)LaVh#+GouDDRdjN-Y8U~1286QYEUspXr|>I(+Etr<#(JUA`b)`_`1El| z@y+ONVU_uJSG+k5wn8 zq|g5u;FT>h;=XID746|Adm?{CD(>JLf6f2eF%szOQ22CpAVuWSmr%3q@0mv=Kif9vC(Iu_KzT^3f z?akZec6yDonK>_-=Q1irCy_(0yT+=DoD0qMjwPYZp8jblh+u1%!)CG9%VJx@W?5;9f6Y_EloY zdN0bQ@p8}fxdOv^ImKP>lJ>ESh_8DJJZ|Thy~SQIA((0p&NhFDe!qhzT+mg+dlO!) z5g>|VoeyeprVC*#H3x^Q=8B-%6#=Cu=sH^ql$sc6+%Cv`{6J7X7#o__8P={rpy$z@1dn$ zKb}PFPfA6Q0svPSEQAr`0i|y^qLv_~cipecJ^T~?2m%1E0Wc#_4s^aH!FN>}{PzHK zAFHgB#za%H(AD4>#B1*>DJTjjPnLR6*qY6s5^4~p2qE$8dG~4Tc(LmEmCh#NRu;CbAO!9 z2{=&7D^HWXnJLWdfz~e>%I&V4ztL*Zwd!SFsNAs!G#S&;^p0t1CD(=~e5rdLo~`ZZ z1*Tw^t*|dIo)&qVZ%>*n`@LBlWV

h17wLm{S<&k8Pb3w8Zd((g#>Yw|p)ZV)!pT zkvy;8Eap~zV?(WforI*usNDAbfjeFm5lyUN%!&nx=B!X zG;pNMa5PweerK65II|12F$Q(_+B3Ly&v93OSle;n9)V5{ybK8@?yI-j_L7>;>FF{H z@!N%*z>3BK%x1>nB%noh$a@8bG!n%~E*@dLs^T3roBlA{07kRB^sMRrf@fgO=dp5m zPO(>&N`CQZa+CKH+%M-2s-lTy196`5E@$_9p}Gdh{A*13VIz0O*fxoX?0?{V_)5)L zK!fA3Cy#oKW?;4RwBo6xVm3R|iuz#e{w)dZN7ZCiA-~v#!^@LLOnbA|L+ypbM``a% zWvYTDVB{0R4f{Kz`}}RXiQ%7uVmn|u&c$bj!7E?UGNDALcRTYRw(u3+CnCv{npDa3 z)n$o6MJVk1y&a{7Mq5jO+3a?c*m#TslN4)oM>4pxw#57Nho~}vy{#bQn@VO8>UBwW z25!{5Wj8b52g&eVLKqWFskv+wzij?dKJ=#ZLww+mM+S8=QfgeMK3Pej2CaIv?!EU* z7M1sqRFz?!zOOQTR>aNkn|Wkt-jNqwVyQZC3XXRcG$gc|)A|b1DQtK?U)2W9SsZ{3 z`O`n)DeLlbh02gq-*z~_}tGL`39iNEO@6)aC8k_qN&tlQrHYa(M zdKiqQ(lfva{L!^JQgxYFR5q{rNUD}vRi;cb;1z-<-^*@H!f-NkLnm2xkh)F?t+lE1 z9&sRHX`r|2Wmb3(=L51bc^u^(liLQ=7NROg3T~^p$Ko`FhU0-mv5iu%iGWZ#Ta)#* z4lc9wY(V`6`>ZJoOi#IB@sOH-P~qv5mt3ichZ#ryD`8VHm78#ZjeLVwI>r5w$eDb2 zlX_zr>nq8R8^qdZe^tXbWQrj4oMy`*cKt%~IU#DX4I<&uZPdDs9ABd#LLQ%e__$j_m)n_KhTCN)j66mmv=yNvsmdrm8g?CyR!ZcjU4MFiW8*LxQO~+EI&Xbb3 znzU3o-rtO=C(*1=G83dNiBP--o_=xR=#W;;T6MM1wPZQ;TWs<<49|-G4CmQnHXs)h za-;g}-m19s7PH*@O?K^_#ZzwpL7_Fb&y=$aM#&nVYzKMqjFz+;KBUadBLovE-@UT# z|F-WBsHC>#6>Pn0vrZTFEVR92zAMh-O}?SOP7xnpv4PX>WNvPCCo|6~$1G)dR9dtV z_jAR-J;1%9E1%aLl1feJnxFW(#(P~wr6jv8dB6+2TpLMp$0+Z_HUSEsilntQCJVpM zF~;|42#~vORY`4!O2Y-Azn!{DEpawSqWWEPAY{`4-)zTu);Ey;?y4%h;F*~OlI4^U z4H|JWmw{{goMtuSZT4oM>AsoC?xw^8`gMP|hrkeNv#RLrPn~E1%n@C9SJQh=t7Hl~Xva2o>VtUsgZP+!fNcvC^6SZ+}$CI%H7N1RBy)9k{| z6?+qC3Dw$m1Gu~R@#ZXLR|pAX7=E!I;6?Sc0=7+NBwM>m(`=YB$JQ<2~xS&|*uaqDcS zW889PZ{Szs6Kl>Qr8nRGcSpy0mw6x1#yh=L3M^Sd3T*~m{6bek%my0GLf0&^QI^73 zY~=9(O&?c_A9clLPMN`T9gjXn&z#qIoT;S-FI`fb-u~#+T{7>}L-F=Sjh{|&Cch?j z{2UL&xLdu{mnPV!7*{gmCH!LWgP7ko?jC!ampOQsQZ9cWDiM(&@iOs~T4h6X)(kH% z<ZE_`w%=rSxe4o{YK(?`0l_<}X#UCmqxf5P6`p)o=EH6+|2V3tIRXk%%dXW4GCiDRd(b~dOhvqTg@tlz>=r~sTOiVkV?y1SVJ z`9*)|4vyw+u!(t+AH?Ix?x~^t6s(3E%a0L&shBLztur=@xxV>Uuh2En} zstCK}G8^9cI;}(*ufI278C%Lq>gpa6t$4@oCdz$3qe@>xJ8oM$7 z2nh1%e8EoE4`eGIUk`g+{k+0x)rIH7y_^yjemP^${PypMS2>5+6t?61Rr^)D92b@Z z&l}}bx!_KKYf5d$txKxEUBoEE=o=0NRLQ9R{f97eP!p12%u}c&=l``uf0qAWP+*3k z;++3V2IXJd^uHm7VHD9NamU;h(I7!!azywz|38oZCtDLUB_i@) None: ["agreed on 21st jul 2017", "nuh23", "grip"], ["farm", "nuh67", "grim"], ["cake", "nuh27", "cheese"], + ["glip", "nuh65", "jkl"], ] assert expected == obtained From 5250972e89677751f67bb6750cb0dfb30f3a0ead Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 29 Sep 2026 16:54:40 +0100 Subject: [PATCH 15/82] updates it to handle ignored values with spaces around them --- nuh_helper/date_shift/__init__.py | 273 +++++++++++++++++++----------- tests/test_shift_ignore.py | 6 +- tests/test_structural.py | 12 +- 3 files changed, 187 insertions(+), 104 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 4794aee..490a718 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -29,10 +29,18 @@ def __init__(self, page: str, id: str) -> None: class ShiftFoundNonDate(Exception): - def __init__(self, page: str, row: int, col: int, col_name: str, val: str) -> None: - message = f"{page=}[{row}, {col} @ {col_name=}] {val=}" + def __init__( + self, page: str, row: int, col: int, col_name: str, value: str + ) -> None: + # the spaces and newline in the message make the value easier to copy/paste back + message = f"{page=}[{row}, {col} @ {col_name=} ]\n\t'{value}'" super().__init__(message) - self._message = message + self._message: str = message + self._page: str = page + self._row: int = row + self._col: int = col + self._col_name: str = col_name + self._value: str = value # >> pr 127 Exception goes here @@ -70,24 +78,24 @@ def __init__( # << end of pr 131 -class DateColumnMissing(Exception): - def __init__(self, page_name: str, column_name: str) -> None: - message = f"date {column_name=} is missing from the cdm {page_name=}" +class DateColumnsMissing(Exception): + def __init__(self, page_name: str, column_names: list[str]) -> None: + message = f"date {column_names=} is missing from the cdm {page_name=}" super().__init__(message) self._message = message self._page_name = page_name - self._column_name = column_name + self._column_names = column_names -class TextColumnMissing(Exception): - def __init__(self, page_name: str, column_name: str) -> None: - message = f"un-shifted {column_name=} is missing from the cdm {page_name=}" +class TextColumnsMissing(Exception): + def __init__(self, page_name: str, column_names: list[str]) -> None: + message = f"text {column_names=} is missing from the cdm {page_name=}" super().__init__(message) self._message = message self._page_name = page_name - self._column_name = column_name + self._column_names = column_names class BlankColumnHasData(Exception): @@ -530,175 +538,246 @@ def shift_excel_dates_inplace( if sheet_name not in sheet_configs: raise ExtraPage(sheet_name) + # start looping through each sheet in the configuration for sheet_name, sheet_config in sheet_configs.items(): + ### + # load the workbook sheet and configuration + ## + if sheet_name not in wb.sheetnames: raise PageMissing(sheet_name) if isinstance(sheet_config, str) and sheet_config == "skip": # it's a skipped sheet + logger.info(f"Skipping {sheet_name=}") continue + logger.info(f"Shifting {sheet_name=}") - assert isinstance(sheet_config, dict) - - assert "text_columns" in sheet_config, ( - f"no text_columns setting for {sheet_name=}" - ) - assert "date_columns" in sheet_config, ( - f"no date_columns setting for {sheet_name=}" - ) - + # these will raise errors if the keys are missing - that's fine ws = cast(Worksheet, wb[sheet_name]) - patient_id_col: str = cast(str, sheet_config["patient_id_col"]) - date_columns: list[str] = cast(list[str], sheet_config["date_columns"]) - text_columns: list[str] = cast(list[str], sheet_config["text_columns"]) + patient_id_col: str = cast(str, sheet_config["patient_id_col"]).strip() + date_columns: list[str] = [col.strip() for col in sheet_config["date_columns"]] + text_columns: list[str] = [col.strip() for col in sheet_config["text_columns"]] + header_row: int = cast(int, sheet_config.get("header_row", 0)) skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) + ### + # do some checks of the configuration + ## + + # this is an old field i want to be careful about skipping if "shift_exceptions" in sheet_config: - raise ValueError("??? are we abandoning shift_exceptions?") + raise RuntimeError(f"shift_exceptions was removed, update {sheet_name=}") - assert patient_id_col not in date_columns, ( - f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" - ) - assert patient_id_col not in text_columns, ( - f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" - ) + # check patient_id_col isn't re-used as other column types + if patient_id_col in date_columns: + raise ValueError( + f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" + ) + if patient_id_col in text_columns: + raise ValueError( + f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" + ) + # check that column names don't appear in both + col_names = [col for col in date_columns if col in text_columns] + if col_names: + raise ValueError( + f"{sheet_name=} has the some columns in both date and text {col_names=}" + ) col_names = _excel._get_row_values_resolving_merged( ws, header_row + 1, ws.max_column + 1 ) - # find patient_id_idx and normalise the column name list + # find patient_id_idx and normalize the column name list patient_id_idx = None for col_idx in range(len(col_names)): col_name = col_names[col_idx] + + # if the name is None ... leave it as that (it's fine) if col_name is None: continue + + # strip the name col_name = col_name.strip() + + # if the name is now blank; store None as the column name if col_name == "": col_names[col_idx] = None continue + + # update it to just be the stripped version col_names[col_idx] = col_name - if ( - # check the config to see if we know what to do with this column - (col_name not in sheet_config["date_columns"]) - and (col_name not in sheet_config["text_columns"]) - and (col_name != sheet_config["patient_id_col"]) - ): + + # check the config to see if we know what to do with this column + if col_name not in ([patient_id_col] + date_columns + text_columns): raise ExtraColumn(sheet_name, col_name) + + # if col_name == patient_id_col: patient_id_idx = col_idx + # check to be sure we found the patient_id_col/patient_id_idx if patient_id_idx is None: - raise NameError(f"{patient_id_col=} not found in {sheet_name=}") - - for text_column in sheet_config["text_columns"]: - if text_column not in col_names: - raise TextColumnMissing(sheet_name, text_column) - for date_columns in sheet_config["date_columns"]: - if date_columns not in col_names: - raise DateColumnMissing(sheet_name, date_columns) - - # for each row + raise NameError(f"{patient_id_col=} column not in {sheet_name=}") + + missing = [col for col in text_columns if col not in col_names] + if missing: + raise TextColumnsMissing(sheet_name, missing) + missing = [col for col in date_columns if col not in col_names] + if missing: + raise DateColumnsMissing(sheet_name, missing) + + # map and strip the shift_ignore values + shift_ignore = { + col_name: [ + ignore.strip() + for ignore in sheet_config.get("shift_ignore", {}).get(col_name, []) + ] + for col_name in col_names + } + + ### + # process each row of the workbook + ## for row_idx in range(ws.max_row): + # skip all rows that happen before the header row if row_idx <= header_row: continue + # skip any skip rows if row_idx in skip_rows: continue - # get the patient id - pid_cell = ws.cell(row=row_idx + 1, column=patient_id_idx + 1) + # update the user every 40 rows + if (row_idx % 40) == 0: + logger.info(f"Shifting {sheet_name=} up to row {row_idx}") - for found in datefinder.find_dates(str(pid_cell.value)): + # get the patient id for this row + cell_value = ws.cell(row=row_idx + 1, column=patient_id_idx + 1).value + + if cell_value: + if not isinstance(cell_value, str): + raise ValueError( + f"bad val {sheet_name=} {patient_id_col=} ;; {cell_value=}" + ) + + cell_value = cell_value.strip() + + # check for date in the cell_value + for found in datefinder.find_dates(str(cell_value)): raise HiddenDate( sheet_name, row_idx, patient_id_idx, patient_id_col, - pid_cell.value, + cell_value, found, ) - pid = _parse._normalize_patient_id(pid_cell.value) + pid = _parse._normalize_patient_id(cell_value) if pid is None: - for col_idx in range(ws.max_column): - cell = ws.cell(row=row_idx + 1, column=col_idx + 1) - original_value = cell.value - if original_value is None: - original_value = "" - else: - original_value = original_value.strip() - if original_value != "": - raise ValueError("row has no patient - row should be blank") + # if the pid is None; the rest of the row should be None as well + non_blank = [ + cell + for cell in [ + # get all cells + ( + col_idx, + col_names[col_idx], + ws.cell(row=row_idx + 1, column=col_idx + 1).value, + ) + for col_idx in range(ws.max_column) + ] + # keep the cells that aren't blank + if cell[2] is not None and cell[2].strip() != "" + ] + if non_blank: + raise ValueError( + f"{row_idx=} has no pid, should be blank but has {non_blank}" + ) + + # we've completed the checks for a None pid + # ... so we need to skip the rest of this loop continue # determine how many days to shift shift_days = shift_dict.get(pid) if shift_days is None: + # all patients need to have a shift value raise UnknownPatient(sheet_name, pid) + shift_delta = pd.Timedelta(days=shift_days) # now scan each column + # ... even when page has no date_columns; still check for hidden dates for col_idx in range(ws.max_column): - # skip the patient id column + # skip the patient id column (it was already checked anyway) if col_idx == patient_id_idx: continue - col_name = col_names[col_idx] - + # get the cell value. replace it with None if it's just whitespace cell = ws.cell(row=row_idx + 1, column=col_idx + 1) - original_value = cell.value - if isinstance(original_value, str) and original_value.strip() == "": - original_value = None - cell.value = original_value - - if original_value is None: + cell_value = cell.value + if isinstance(cell_value, str): + cell_value = cell_value.strip() + if not cell_value: + cell_value = None + cell.value = None + + # so if the cell is empty; skip the rest of the checks for this cell + # ... we go tot he next column in the row + if cell_value is None: continue + # if there's no column name; this column of the row should be None + # ... and we win't reach this line if the cell_value was None + col_name = col_names[col_idx] if col_name is None: - raise BlankColumnHasData( - sheet_name, row_idx, col_idx, original_value - ) + raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) - # skip shift_ignore cell values - if original_value in sheet_config.get("shift_ignore", {}).get( - col_name, [] - ): + # skip values in shift_ignore + if cell_value in shift_ignore[col_name]: continue - if col_name in sheet_config["text_columns"]: + # for text columns, we now *just* need to check for a hidden date + if col_name in text_columns: # check for dates in non-date columns - for found in datefinder.find_dates(str(original_value)): + for found in datefinder.find_dates(str(cell_value)): raise HiddenDate( sheet_name, row_idx, col_idx, col_name, - original_value, + cell_value, found, ) + cell.value = cell_value + continue - if col_name in sheet_config["date_columns"]: - # original_value can be str | datetime | date - # ... so parsed might not always succeed - parsed = _parse._parse_date_value(original_value) + # assert col_name in date_columns + # ... which has to be true ... - if parsed is None: - raise ShiftFoundNonDate( - sheet_name, row_idx, col_idx, col_name, original_value - ) + # cell_value can be str | datetime | date + # ... so parsed might not always succeed + parsed = _parse._parse_date_value(cell_value) + if parsed is None: + # raise an error; we already handled + raise ShiftFoundNonDate( + sheet_name, row_idx, col_idx, col_name, cell_value + ) - shifted = parsed + pd.Timedelta(days=shift_days) + # perform the actual shifting + shifted = parsed + shift_delta - # store it as either "DATE" or "DATETIME" - if isinstance(original_value, date) and not isinstance( - original_value, datetime - ): - cell.value = cast(Any, shifted.to_pydatetime().date()) - else: - cell.value = cast(Any, shifted.to_pydatetime()) + # controls wether the data is displayed with the 00:00:00 in Excel + # ... it might be nice to just convert it all to iso8601 text + if isinstance(cell_value, datetime): + cell.value = cast(Any, shifted.to_pydatetime()) + else: + cell.value = cast(Any, shifted.to_pydatetime().date()) - continue + logger.info(f"Shifting {sheet_name=} processed {row_idx} rows") wb.save(output_file) logger.info("Output written to '%s'", output_file) diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 92248ea..8007ec2 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -43,7 +43,11 @@ def body() -> None: if not shift_ignore: with pytest.raises(ShiftFoundNonDate) as info: body() - assert str(info.value) == "page='page-data'[5, 1 @ col_name='dob'] val='mssing'" + assert info.value._page == "page-data" + assert info.value._row == 5 + assert info.value._col == 1 + assert info.value._col_name == "dob" + assert info.value._value == "mssing" return else: sheet_configs["page-data"]["shift_ignore"] = {"dob": ["mssing"]} diff --git a/tests/test_structural.py b/tests/test_structural.py index 3899e88..2de3ea9 100644 --- a/tests/test_structural.py +++ b/tests/test_structural.py @@ -5,11 +5,11 @@ from nuh_helper import shift_excel_dates_inplace from nuh_helper.date_shift import ( BlankColumnHasData, - DateColumnMissing, + DateColumnsMissing, ExtraColumn, ExtraPage, PageMissing, - TextColumnMissing, + TextColumnsMissing, ) @@ -85,7 +85,7 @@ def test_date_column_missing(tmp_path: Path) -> None: }, "stuff": "skip", } - with pytest.raises(DateColumnMissing) as info: + with pytest.raises(DateColumnsMissing) as info: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -99,7 +99,7 @@ def test_date_column_missing(tmp_path: Path) -> None: linking_table_output=str(linking_table_out), ) - assert info.value._column_name == "a-missing-date-column" + assert info.value._column_names == ["a-missing-date-column"] def test_extra_column(tmp_path: Path) -> None: @@ -162,7 +162,7 @@ def test_text_column_missing(tmp_path: Path) -> None: }, "stuff": "skip", } - with pytest.raises(TextColumnMissing) as info: + with pytest.raises(TextColumnsMissing) as info: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -176,7 +176,7 @@ def test_text_column_missing(tmp_path: Path) -> None: linking_table_output=str(linking_table_out), ) - assert info.value._column_name == "a-missing-text-column" + assert info.value._column_names == ["a-missing-text-column"] def test_page_missing(tmp_path: Path) -> None: From d727465ef0fabd8e4231b26c0e3979870c381716 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 29 Sep 2026 16:59:37 +0100 Subject: [PATCH 16/82] Update log message and comments for clarity --- nuh_helper/date_shift/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 490a718..8551b78 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -650,7 +650,7 @@ def shift_excel_dates_inplace( if row_idx in skip_rows: continue - # update the user every 40 rows + # write a log message for the user every 40 rows if (row_idx % 40) == 0: logger.info(f"Shifting {sheet_name=} up to row {row_idx}") @@ -755,14 +755,14 @@ def shift_excel_dates_inplace( cell.value = cell_value continue + # we've already checked this (sort of) # assert col_name in date_columns - # ... which has to be true ... # cell_value can be str | datetime | date # ... so parsed might not always succeed parsed = _parse._parse_date_value(cell_value) if parsed is None: - # raise an error; we already handled + # raise an error; we already handled values we should pass as-is raise ShiftFoundNonDate( sheet_name, row_idx, col_idx, col_name, cell_value ) From 592c1301823468f6bef6ff50f4202312e2653f8a Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 29 Sep 2026 17:02:05 +0100 Subject: [PATCH 17/82] might do the dates --- nuh_helper/date_shift/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 490a718..707a08d 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -776,6 +776,7 @@ def shift_excel_dates_inplace( cell.value = cast(Any, shifted.to_pydatetime()) else: cell.value = cast(Any, shifted.to_pydatetime().date()) + cell.number_format = "yyyy-mm-dd" logger.info(f"Shifting {sheet_name=} processed {row_idx} rows") From 4b4479a025251782149887ac506b626d7f5701b0 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 29 Sep 2026 17:31:50 +0100 Subject: [PATCH 18/82] more corrections --- nuh_helper/date_shift/__init__.py | 2 +- tests/data/missing-id.xlsx | Bin 11117 -> 11163 bytes tests/data/shift_ignore/workbook.xlsx | Bin 10365 -> 10388 bytes tests/test_shift_ignore.py | 205 +++++++++++++++++++++++++- 4 files changed, 199 insertions(+), 8 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 252762a..ac89e3e 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -771,7 +771,7 @@ def shift_excel_dates_inplace( shifted = parsed + shift_delta # controls wether the data is displayed with the 00:00:00 in Excel - # ... it might be nice to just convert it all to iso8601 text + # ... it might be nice to just drop the time if it's 00:00:00 if isinstance(cell_value, datetime): cell.value = cast(Any, shifted.to_pydatetime()) else: diff --git a/tests/data/missing-id.xlsx b/tests/data/missing-id.xlsx index 5737b01e283d1604edd5c47bb471a4f13b55424f..1be4f8e106d234d9a7d82554d75507df0278a2ed 100644 GIT binary patch delta 2789 zcmYk8XE+-Q1BDYSH6jENdlO^0u7un>$R%D^i*b)z;sA@ zmQwmWQi*;mibf9>I+^+AHc)7cnR(v8tTEi->S=VH%MGmA+i6Y(m*j$OH2R@V%XwXy zfoHE)e`ZkmDt)1Q<;RSy$xjv+WI_4hKemLcaJCBanVZ#>U)t?1E@!{DFw5LFK6nkh zId6QVIiw&vB8|LD&~ANR>>|G{$z^6nYq9TvuX4D)N7v_Ul>yTv*_hdH-HW)H6+R}J zhH=6YkJu`{!?bFW4e}P*-tFkTk+=aOyoUMLio-W~ERhVt$D{Z4T zq54X}{n$(6&~xelikI`1s?5Cer}84L)I-CWG+&@5?{Y~?q$HPs3vJh{)Z~~91DJdR z70>SZnbioGFl9Ct($%0$Ze0)4a13qhxC4I-8!g#=F{HVrtvoTZ$D`!k$T0BAN#D

5Ko?ZbGH^WjRcb!&2Ao>;6Xg2B=&n=gRrzR24lB${D7({*4^npbjTUhoJCLT#rS6$4YImR3i8!wf)N5ine&(t9K=C*^G%v8nXQQ^m@2%UBDZBC(s`K6iBEO(Q6hmb{ zsg=hFo&0SI>TfE|xwaXV)6x`8YHDh-v|HdylAEbLA6QVUYXU9D&7o{vO+7YIO#5E_oVrwb?$JJ_6M`1v3>Fo30QuP{ONFlD)BekZOl91PAT7j95v{+ zp3xFJX;|ww>4vCioz=t=B>&j~A;k9(e!@$`giKkc{=L*Q9?3;Y_%A_&x}juJOSFMi zQ$oFyQ4LJCA-mseVsAmtgTLm_Jmhz3^g(WJ|Ej=nCN3vm=#K5cgUFaw#~yl7C$sH4 zr~N0bW86ZXkC?w0!Ga@m-}WhPCGfozE93PaYSzdz-C#3`7iT9`*bL#yd2D`BND^W! zmF0cPj!DvNC0MqUhorqg3&oPx9SvQY;ITw~LHMEpejw1%jky=?7IMGj^*DP&7C33R zEC`b#Bzgc9#tX8nl|g$E51*5t3b4())a;@;ETTTr*{i(^H_X_)tT1YJFmOwUgHq-* zejok1rRXQ_o?>53NMzt>jpd7+Cv;d35%{3#*KPko!robU%siX+E$h|jL7t1vMdhAX zttYq!IHNOkR|hxn&=H;4A9J~03dLA}M^3 z;rcSmcZwIh<2OpyH_4WUXZdJIunhktl@NYZXD!WHX5U|!wJA1Y&rle(+z$$$>p#N5Cg{%hpp%mTo|V@AQe>7XFF*?_okW; z4q>KmNaF25W02rmafMLXk@EFjx-@eR?Z{35ARVY-nOzdfVQu2ZwdDLlW{-hIXg z_%}kop}LH8$$5GFX+Qvgg^Fk(z)djro|R>I57{8_`*~+W`0vrr_R8kH2U9j?gL;cS z8&E%Dee2H2avF@`MS!-A{V`!azTx%uf@n$_tWm0sdL;wD{G7@r@_;-!e8El$m1_8k zc62d8tTnfhEn~Gt?UIKW zs&gH2DRD#6^=kgBfdC#|rmPM**`ubO$78NNW`^aAiny$um|hWZ1LwfEUX9k#PeBmx zdi!Xlz1z(^#an9^o%@oPR6W9aE1X;2*Y?F4V30g^I$f&30G27Jy*F~bw8vF_Kjcb| zUUl=oQBkj@9t0TH(+CLz2@%O=?p?C1k&7TQL}br%2BCfHo+Fo<__eL6se5`P+pmQJ zgO!)$qN)}z#^YC6xb`fZexuTmic(0eRe^7cmK@#9eH}};w1PBWj*q4eJQiUS@nPKA z9j~>qgfy&ipp%0WC%Dhbf%3NbOW*8*_qdD)9=~~o>yrWyGG1Ilg!BX5`tFQo8^Pt9Prjtk=JPRpGb2;MhocLi+mc|A)72M+RI}V+0U2Cqe!8W$~k?q$@*KT z#-;El{(PSP@{OKc4Mer5Pq|A?N1M&noYv*DY#>YLFbl7c<~~-gpE4jM-oYQt!7Dxw zb9bGekqN#R`b6nMS7DJ8(=3Bp(x6h~N0%=;7wRPcdntjvE1Ac$B~{{2WIOn}QoDREs2Auv$| z%t=fXXX5(b##{pc;DEoYg4imqO#x;hQb=e4gNV8kH-S#XXA=C}|7QvSVEwxTU@1T} j5r7eAB$)maDREDNjeuo< z>)4VkA?sMnQfl6N?|IL8?|uGy&htIbU*GdN-*1K$&9Zu&nN5jNuVIxD1OgLgnfU?l z`vn(azn_f$VtE@BbBY;07J!%yNSU@6AE*z?K_fGrQyNId__Aqwy9*po*2wbz919oP zT-a4q9O6)=*Otn-^h7LqD=O{^D4d#D^6ttYrX4>{w&9^B#VA86r=tcaNXY>#vV$Jj z?$fjH@<~$&G~AbFgC38pm)*yY12?nlPTzBM{}_{wZbV$K-()I?#0cDYct?%(IM6u6 z#fST{A5^!+ZZA{9dv>zI%of&#E_>RJ2|1@nzJ!BR6@O{ku4t`Pdm6liu);Eyc#MSv zACHm4H7(~4n8V)bc22$!)|I=sXi)uqp(K4JKZRzV#XxS8_|*l(dBx2|0FL|e%tTRc z+iJw5>I@0~`<%^j7={jt8o?GgB~9d}z`_p=l+eMHiSY2*Q zOA3}*&nQC!8uvG>E~q#Hy?5B5Dn1|QG>{xn_HU5(ZvFiv&G$lOh6TuR3c16`cXo?E zsiqg76i7|A>@VZwLo~|L!qJ|>r72G^@$_7)^bL_ISfnhK_Wfrx$cZ~4z16Jy>Bfg8 z=-ys@m55Ppq_frA)JvzfbO^C9T&u~sv{wQ*tK2Cj4V|^f|BI3%$=wD;;8YhgI z+)rLLkgb+YK!Txe@V_t`X$^Byt+KmYa!dt+gO~iP#j~YIFKW~S=+M`n$ zxwv6W!jzh_Ywx=UOV8HcVe(nOvZerP?{yWw4;MvB;W@^Pc6e>fmUuoGDNM5a!il#7 zMan-rJgMa|*K9U_gX-eEI#>`hgxNguY=RH;&|ZZgdMK7+PUKjfI|}o=E4%IN3q(oR$|3juQ2zLrvAA6YDJf~06(yOm~u0~N^ADI>otK;^BuMYp{zGb z+YVRIkQ?S7D8ad#4HPW(`pH`&MY@ebM++z%AaDS*1Q17sVt zS`m-DDUuzs7Xs=`_abt)9J>OHy^(~b&p`QPl4$-z`{&rRc}^OM<^A2bZm}Am{*#fA z=;Os=ca&NI7YFCtt)4oO`3(HLsLb= zi~@9)KW{KNhkixUXOEWqBWDem3o~Yy2B)4sD{Bt5=$!1LFk>uyUKVM{Oy4eB%Lej2vFLbc_Ju6X9{oB! z?fEWJavhB>z0^Qp7W|?9e!+QZ*!~&Ig=7PdLB_iQu-CuAfAb>5qsIxI1^CMo89<;l zMuL|p3h46rB*)RlHHCvUHzZK=Y}r5cm^!wd((qc*uJN#~yb_w9sRZC8S>-zYK@Q)J zvJEux<5az!>RzdYQsrgZJCON@tb#n{4&Ybgv6~jYXfRZ=Vt~6x{X!1TLX7REk$HxE z(OQjE;c0p6re!t*agAO_g65)WAMZ$@f>`l95KD7=;J~!er}aUxXnN+cyd2@D+TuP$ z%J{|~_sSmN_0qh``@dtu{Su3tZgN;8r#q*UXqu-!pqu z#VFzK;e}1xsc{AbRm1sa4Bnpt9&mc$*IyJeGxPfCa$tN&!-Tvqml~4#3KG%~F&cdy z=j(3U8q5JQT$lu!|yW1R{O(R7tpg z9Te_TqV?>6lV}jr(RQZ7AK(ml-&b0Q%NSmR2T}N1kt~P({asT^#-rDo`9*;khsknc zl_%z5iwtK81wxzw@7Kwyz-}j4ZuXTy{ngETMeQr1 zg0$F2(pbWaUK%__px;eCt@ops(bnBdMCzLD&)pdFkJJ*VqOWE=h=gLFQ6+ z#}B1&gidAzsZzp{@gE(LL)wIC1_}ZZ=ukmIurwNc=G5Pr(IkyP{X0rP|CnN~{{;&c zyaE4ny~1Ekc!<_w&?eNQMTGv36bQuok1r5Nfv}4{hx*&pKasl!27$Q$rSFRolw>p* P{0ROs;><=;e`)^(G#?_S diff --git a/tests/data/shift_ignore/workbook.xlsx b/tests/data/shift_ignore/workbook.xlsx index 6937d153dd834e07f57908d0623990b91930e766..a3c84279dc41231ad19a5aa2c7dfc63c6b5b131c 100644 GIT binary patch delta 2177 zcmV-{2!8kdP?S-y&;|t;qFuk^lhp<(e+`BZ)N}*^w3XUX)ycHGPn8Q!LFKl_G^wij z?{iEby=1k@b~XwI-#&lmobMbs{j#k}<4MS_XzGPYPl#!VI&HF|epr~_R|)b=qf@-j zc-hoqVLpq_{BrjDZ>O6^KHfLYqX7VQw=i?1S|8&s%|*q#lcp6l%&Z$(aRpuZe}KDI z3Z8Yj5UMJ%MTm8CbC9;J%gKo1C?*?SwhJ8`OEG4c*O?N4Y3l7*t?(hcm7?hU8eQyFx2k;5X9chk5cJ z#LH9d5ezQb(cV3ObPuqX{v^;VB@K8!k1ilc-}n=R41|%58GWu}V&B!N^y%G85G1Y@ zP>MXyijccx1bKl&5lI|-e;&a_tmp?ABV9jjc(3wN4H|l3G6?;_)Ft0eGnDv!k-gG* zA>$9}`-f(l`=Jfib?>GSo9KDhEAz9>{4N z`$H5DIWVSe|Zi{Vmq1Hf$PPg zH}J!62oCiYhz@t79&)7R4+3SPOQE*way)q5G^DSOk>}Zv5r-Qp> z>vxR*e^lBbll5*sM8geHj5IYD9Q zGUP;w?aXMjjGg3_tTuoB$vWrZVYlF_hZ>;Mf9U(VO~%pv;CeSO#`azI`qz>6`?&kx z7JdWkOL64lC@-pZ81D4a{2aCWGrVo{u|5WL%3EX%fFIk2S$&7nZibUD*BZ4;41m86%X=)ZR;MQaj34d6xM4BvY*e5ACz zxuyv^GpV@9LhOufj95m+j%SAu|M-48nq#Ci$#x_W84K|vQ}|}}@x$^W&F5Vqm%}BhfGMaO`z_p;}MFk z5pxR0<{bdJM=`srhC33u|E!S|h!ze-Mo0S(2T(S-^h=%6*?CXIf*q z4cO$JV15l|+9iV%--Y;(@7b~UX&Q~zj_r(Q3vWG|uYG^CSv!lh?atQG^!6`aEq5GN z*i?li+lP4V1o0eOt7YMzpPXG(>l$gYy=MudngNG{k+J7ihzFB+2SYX>sv^N45~}%` zMJ!1|96`4FU2sHB5IKLgH=oXY!(`P|?dyv6ws7`WiMB+sNF=}bP9K3AcF>OP$tlsT zLEv~E{3b4+89y8~r25bn(c|8g42QAc9|%a@d$2_>o+T|4Yd;wZ}fR z(s!pA(Ua05tppU(%lSnCG0|kTl;VQGVW!xenEmPm0QxFD@S1oUNXgJnQ&EyL+?Z48iU zZqcfl>-E|dRdKVjrnhk$ZSlG|Sf0V+Ah!|MZ;}D3#d)r0cN-N|@mgF5t;z1JAlPoO zI>>F*_1v;CKt_LE#UAws;?DFp3z z1N%bcfoBROY+sy$P=+jzNqcNc61h1WXW)gc&~VyScSlEfrC_k5>@jSqy%4&(frc2| zd!|o0%1O@TJ%55H`2r&$Ih?h{J41+bA+;o2V|0Y^Cjo&Z&bfJHm=`0>X?<^=3H5u= z>L0V86Tu7x7@ZEmOS5GlGXVjnlbInd14N=2lgc4UA>w$3B{(LSW4Hx|dLW_;Qt$z( zKTOpD1jkT;1m!Fy5y6u*A|ro(f?qf5r><&Ps&-c#P@W1-xdG;ClW)Pl9IV-qRTZRo zq_kVCm~|V!@LdVw{`}i~ZSdr-NRI~*_-M2*@};~?9v+Jo%9A*mh4DO0rUfB%I;HXa zt$qK;d>ggDke3wy5*J~zD9DT^F(r2+;uj!K@zwXpKLG#$|NjF3P)n1j6AY6%C=Rpl zB0K{H7ouIiXTF`BL=u700000 Dy?hW` delta 2175 zcmV-_2!QvLQTddX^)?QE3T@$K_>&iT%P(=XeqG@iKZL{raA>x7sFud^l>^~2o!zDiMG8l7Tw z&dR3dbMu*Z=9jbIe>>eY^6|cD9t{AfySZ5?)rJ^%S-~sToir`4VP)OOiYXY%e+S&P zlC!)kI9FAPZ9+U;F;Sa4gis!X(X7`ZC3Brg<#bwPDUo^D>9gcSWs^$jlMbUu5jcfjQFC>S=e*%4ht11$UqqRn9=7tCeB@*$PE=($ND^ao3qY&)%3Bmh0(BmMQVXSL_~Uv!u+?eT_WNhm1Hp06Kbe01Dv#fIY}y zx+-I326?2r0XeXUJ<~h*R$X*z=t05)OtL76d@`emB#w({FtbrW9UCp^(oS4ISthZo zBdnh>;r~%-hfE5me}|W5Sum+ql4XzZ$h+q2m~|k@!2=lhz9k#GK};N=NK-3C)S3|# z#~wxQGIdET0c0ak^O^mU9m%ZWjQu}>8 z{cjt;f%PRna&vlf8krssp|If>=+ZQ1x#({vaR~>9m56)yDPCP_uJ7NBc(|ZlS~vWz>iGfoAt*JtCNrqDrHO~Kqypz zQ?1LTWht7noT#xVSpk+rf5@C@xRrZLm6DOT+LULOYumn+6JFpNVJU}Th$P{ZZ3R8# ztk5-xWEs)GU!}ZM7qpxXfy;?}IFutQ@)96+JmdPY0>vn&%e#FcB-v$<_u26XMP~^g z7rv8Am}>o5q?}VJlt}a#kgb~Y^YjUR~R4q!DAf7Yw^`V#H767*Mzc0{pAWWRWrCrYPtb_>EnY92FS^f?t$#=trnqPR`Z;sg5(X$39fhcc&Q9gVG|u z5HQ=VdViI`mT0nGNpV6TEFhvvV%pSMn!|W0Jj6AwA@~9ze+30Qt|u$&Xkedgo372j zwnbgrtg%_ZprHYx8cYLx8ya4ZM%Om&(dgRdFVlbu45k6T4GpIkYh!>+ajR0zT(4KI zsJnUbG8gX+T)?;G(yxgDYPfi=XZPxIwYfE3oezj=aACl=CDYTw#sC>IM$$%9(Utcr z>VUo&936byf9P&c9#=-ER|gZNYKV6KD>9kAfe-k$c>S(1K+VzkQ@c;;YV#{PF;i-2 z2ZL`z#%+_UPGZwFPWnu)jBQzKk<`Pmo+~BUvoA#Md7)6ol4^oM8nPa7_SmE(N;B7H zaP&K&VNzZ0Qg{nsAPRd7{gDWvFE^kTgL}{Pp+qGqFqypPf8gP}z(`0AbFzBO2e=ec zOTslqDXjkyV92&5H&foc%Vo$JjyAs6x`M3Gn;K^N) z9uFe$(P&@fOL>_*JQgdICnTALWF98dB97^FO3D1KegDUN8@0cXmlXdJ7h$p};ydc0 zBpDIE0C|e9zDNEE009600{~D<1CyB^3$yqlN&^La1w8QOlSd{}1k?%u0F&?v8k4ss zDgh3Y;wCE_bOt None: + """check that misconfigurations cause an error""" + + source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" + output_path = tmp_path / "target.xlsx" + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_out = tmp_path / "linking_table_out.csv" + + sheet_configs = { + "page-desc": "skip", + "page-data": { + "patient_id_col": "pid", + "date_columns": ["dob"], + "text_columns": ["top"], + "header_row": 1, + "skip_rows_after_header": [], + "shift_exceptions": {"dob": ["mssing"]}, + }, + } + with pytest.raises(RuntimeError) as info: + shift_excel_dates_inplace( + input_file=str(source_file), + output_file=str(output_path), + patient_sheet="page-data", + patient_id_col="pid", + sheet_configs=sheet_configs, + min_shift_days=-20, + max_shift_days=-1, + seed=14333, + linking_table_path=str(linking_table_old), + linking_table_output=str(linking_table_out), + ) + assert str(info.value) == ( + "shift_exceptions was removed, update sheet_name='page-data'" + ) + + +def test_ptid_in_date_columns(tmp_path: Path) -> None: + """check that misconfigurations cause an error""" + + source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" + output_path = tmp_path / "target.xlsx" + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_out = tmp_path / "linking_table_out.csv" + + sheet_configs = { + "page-desc": "skip", + "page-data": { + "patient_id_col": "pid", + "date_columns": ["dob", "pid"], + "text_columns": ["top"], + "header_row": 1, + "skip_rows_after_header": [], + "shift_ignore": {"dob": ["mssing"]}, + }, + } + with pytest.raises(ValueError) as info: + shift_excel_dates_inplace( + input_file=str(source_file), + output_file=str(output_path), + patient_sheet="page-data", + patient_id_col="pid", + sheet_configs=sheet_configs, + min_shift_days=-20, + max_shift_days=-1, + seed=14333, + linking_table_path=str(linking_table_old), + linking_table_output=str(linking_table_out), + ) + assert str(info.value) == ( + "patient_id_col='pid' shouldn't be in date_columns of sheet_name='page-data'" + ) + + +def test_ptid_in_text_columns(tmp_path: Path) -> None: + """check that misconfigurations cause an error""" + + source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" + output_path = tmp_path / "target.xlsx" + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_out = tmp_path / "linking_table_out.csv" + + sheet_configs = { + "page-desc": "skip", + "page-data": { + "patient_id_col": "pid", + "date_columns": ["dob"], + "text_columns": ["top", "pid"], + "header_row": 1, + "skip_rows_after_header": [], + "shift_ignore": {"dob": ["mssing"]}, + }, + } + with pytest.raises(ValueError) as info: + shift_excel_dates_inplace( + input_file=str(source_file), + output_file=str(output_path), + patient_sheet="page-data", + patient_id_col="pid", + sheet_configs=sheet_configs, + min_shift_days=-20, + max_shift_days=-1, + seed=14333, + linking_table_path=str(linking_table_old), + linking_table_output=str(linking_table_out), + ) + assert str(info.value) == ( + "patient_id_col='pid' shouldn't be in text_columns of sheet_name='page-data'" + ) + + +def test_columns_overlap(tmp_path: Path) -> None: + """check that misconfigurations cause an error""" + + source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" + output_path = tmp_path / "target.xlsx" + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_out = tmp_path / "linking_table_out.csv" + + sheet_configs = { + "page-desc": "skip", + "page-data": { + "patient_id_col": "pid", + "date_columns": ["dob"], + "text_columns": ["top", "dob"], + "header_row": 1, + "skip_rows_after_header": [], + "shift_ignore": {"dob": ["mssing"]}, + }, + } + with pytest.raises(ValueError) as info: + shift_excel_dates_inplace( + input_file=str(source_file), + output_file=str(output_path), + patient_sheet="page-data", + patient_id_col="pid", + sheet_configs=sheet_configs, + min_shift_days=-20, + max_shift_days=-1, + seed=14333, + linking_table_path=str(linking_table_old), + linking_table_output=str(linking_table_out), + ) + assert str(info.value) == ( + "sheet_name='page-data' has the some columns in both date and text " + + "col_names=['dob']" + ) + + +def test_ptid_column_missing(tmp_path: Path) -> None: + """check that misconfigurations cause an error""" + + source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" + output_path = tmp_path / "target.xlsx" + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_out = tmp_path / "linking_table_out.csv" + + sheet_configs = { + "page-desc": "skip", + "page-data": { + "patient_id_col": "ptid", + "date_columns": ["dob"], + "text_columns": ["top", "pid"], + "header_row": 1, + "skip_rows_after_header": [], + "shift_ignore": {"dob": ["mssing"]}, + }, + } + with pytest.raises(NameError) as info: + shift_excel_dates_inplace( + input_file=str(source_file), + output_file=str(output_path), + patient_sheet="page-data", + patient_id_col="pid", + sheet_configs=sheet_configs, + min_shift_days=-20, + max_shift_days=-1, + seed=14333, + linking_table_path=str(linking_table_old), + linking_table_output=str(linking_table_out), + ) + assert str(info.value) == ( + "patient_id_col='ptid' column not in sheet_name='page-data'" + ) + + @pytest.mark.parametrize("shift_ignore", [True, False]) def test_ignore_in_date_columns(shift_ignore: bool, tmp_path: Path) -> None: @@ -66,14 +252,19 @@ def body() -> None: assert worksheet.cell(6, 1).value == "nuh67" assert worksheet.cell(7, 1).value == "nuh27" + # check column 3 is all none + assert not [ + val for val in [worksheet.cell(r + 1, 3).value for r in range(7)] if val + ] + # last column - assert worksheet.cell(1, 3).value == "pizza topping" - assert worksheet.cell(2, 3).value == "top" - assert worksheet.cell(3, 3).value == "cheese" - assert worksheet.cell(4, 3).value == "unknown" - assert worksheet.cell(5, 3).value == "mushrooms" - assert worksheet.cell(6, 3).value == "this can't be a date - sorry 2016" - assert worksheet.cell(7, 3).value == "idk - this can't be a date anymore" + assert worksheet.cell(1, 4).value == "pizza topping" + assert worksheet.cell(2, 4).value == "top" + assert worksheet.cell(3, 4).value == "cheese" + assert worksheet.cell(4, 4).value == "unknown" + assert worksheet.cell(5, 4).value == "mushrooms" + assert worksheet.cell(6, 4).value == "this can't be a date - sorry 2016" + assert worksheet.cell(7, 4).value == "idk - this can't be a date anymore" # the important column to check - the dates assert worksheet.cell(1, 2).value == "birthday" From 19ff976d1875a9971190dd0d9fb59e500376e7e8 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 1 Oct 2026 16:58:23 +0100 Subject: [PATCH 19/82] raises a specific error for missing patient columns --- nuh_helper/date_shift/__init__.py | 14 +++++++++++++- tests/test_shift_ignore.py | 9 ++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index ac89e3e..e184217 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -113,6 +113,18 @@ def __init__(self, page: str, row: int, col: int, value: any) -> None: self._value: any = value +class PatientColumnMissing(Exception): + """raised when the patient column is missing from any sheet""" + + def __init__(self, page_name: str, column_name: str) -> None: + message = f"The patient {column_name=} is not present on {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_name = column_name + + class ExtraColumn(Exception): """raised when an unknown column appears in a page we're shifting. @@ -620,7 +632,7 @@ def shift_excel_dates_inplace( # check to be sure we found the patient_id_col/patient_id_idx if patient_id_idx is None: - raise NameError(f"{patient_id_col=} column not in {sheet_name=}") + raise PatientColumnMissing(sheet_name, patient_id_col) missing = [col for col in text_columns if col not in col_names] if missing: diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 7daf0c4..849c066 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -4,7 +4,7 @@ from openpyxl import load_workbook from nuh_helper import shift_excel_dates_inplace -from nuh_helper.date_shift import HiddenDate, ShiftFoundNonDate +from nuh_helper.date_shift import HiddenDate, PatientColumnMissing, ShiftFoundNonDate def test_shift_exceptions(tmp_path: Path) -> None: @@ -175,7 +175,7 @@ def test_ptid_column_missing(tmp_path: Path) -> None: "shift_ignore": {"dob": ["mssing"]}, }, } - with pytest.raises(NameError) as info: + with pytest.raises(PatientColumnMissing) as info: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -188,9 +188,8 @@ def test_ptid_column_missing(tmp_path: Path) -> None: linking_table_path=str(linking_table_old), linking_table_output=str(linking_table_out), ) - assert str(info.value) == ( - "patient_id_col='ptid' column not in sheet_name='page-data'" - ) + assert info.value._page_name == "page-data" + assert info.value._column_name == "ptid" @pytest.mark.parametrize("shift_ignore", [True, False]) From af53890d3460b27f5d183743bcd8303b2e4546a9 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Wed, 30 Sep 2026 17:29:23 +0100 Subject: [PATCH 20/82] (WIP) need to digest/adapt the patient_id loading --- nuh_helper/date_shift/__init__.py | 69 +++++++++++++++++-------------- pyproject.toml | 4 ++ uv.lock | 10 +++++ 3 files changed, 53 insertions(+), 30 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index e184217..802c42d 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -6,15 +6,14 @@ """ import logging -import shutil from datetime import date, datetime from pathlib import Path from typing import Any, cast import datefinder import pandas as pd -from openpyxl import load_workbook -from openpyxl.worksheet.worksheet import Worksheet +from book_page import CSVBook, XLSXBook +from book_page.book import Page from nuh_helper.date_shift import _excel, _parse, mappings @@ -159,7 +158,7 @@ def __init__(self, page_name: str) -> None: def _get_patient_ids_and_shift_mappings( - input_file: str, + input_file: XLSXBook | CSVBook, patient_sheet: str, patient_id_col: str, sheet_configs: dict[str, dict[str, Any]], @@ -462,8 +461,8 @@ def shift_excel_dates( def shift_excel_dates_inplace( - input_file: str, - output_file: str, + input_file: str | Path | list[Path], + output_file: str | Path, patient_sheet: str, patient_id_col: str, sheet_configs: dict[str, dict[str, Any]], @@ -519,10 +518,13 @@ def shift_excel_dates_inplace( seed, ) - shutil.copy2(input_file, output_file) + if str(input_file).endswith(".xlsx"): + book: XLSXBook = XLSXBook.copy(input_file, output_file) + else: + book: CSVBook = CSVBook.copy(input_file, output_file) _patient_ids, shift_mappings = _get_patient_ids_and_shift_mappings( - input_file=input_file, + input_file=book, patient_sheet=patient_sheet, patient_id_col=patient_id_col, sheet_configs=sheet_configs, @@ -542,11 +544,8 @@ def shift_excel_dates_inplace( ) ) - wb = load_workbook(output_file, keep_links=False) - wb.defined_names.clear() - # check for sheets we didn't have an explanation for - for sheet_name in wb.sheetnames: + for sheet_name in book: if sheet_name not in sheet_configs: raise ExtraPage(sheet_name) @@ -556,7 +555,7 @@ def shift_excel_dates_inplace( # load the workbook sheet and configuration ## - if sheet_name not in wb.sheetnames: + if sheet_name not in book: raise PageMissing(sheet_name) if isinstance(sheet_config, str) and sheet_config == "skip": @@ -566,7 +565,25 @@ def shift_excel_dates_inplace( logger.info(f"Shifting {sheet_name=}") # these will raise errors if the keys are missing - that's fine - ws = cast(Worksheet, wb[sheet_name]) + with book[sheet_name] as page: + function_that_preserves_indentation( + shift_dict, sheet_name, sheet_config, page + ) + + logger.info("Output written to '%s'", output_file) + + linking_path = linking_table_output or "shift_mappings.csv" + shift_mappings.to_csv(linking_path, index=False) + logger.info("Linking table saved to '%s'", linking_path) + + +def function_that_preserves_indentation( + shift_dict: dict[str, pd.Timedelta], + sheet_name: str, + sheet_config: dict[str, any], + page: Page, +) -> None: + if True: patient_id_col: str = cast(str, sheet_config["patient_id_col"]).strip() date_columns: list[str] = [col.strip() for col in sheet_config["date_columns"]] text_columns: list[str] = [col.strip() for col in sheet_config["text_columns"]] @@ -598,9 +615,8 @@ def shift_excel_dates_inplace( raise ValueError( f"{sheet_name=} has the some columns in both date and text {col_names=}" ) - col_names = _excel._get_row_values_resolving_merged( - ws, header_row + 1, ws.max_column + 1 - ) + # get teh real column names now + col_names = [page[header_row, c].value for c in range(page.columns)] # find patient_id_idx and normalize the column name list patient_id_idx = None @@ -653,7 +669,7 @@ def shift_excel_dates_inplace( ### # process each row of the workbook ## - for row_idx in range(ws.max_row): + for row_idx in range(page.rows): # skip all rows that happen before the header row if row_idx <= header_row: continue @@ -667,7 +683,7 @@ def shift_excel_dates_inplace( logger.info(f"Shifting {sheet_name=} up to row {row_idx}") # get the patient id for this row - cell_value = ws.cell(row=row_idx + 1, column=patient_id_idx + 1).value + cell_value = page[row_idx, patient_id_idx].value if cell_value: if not isinstance(cell_value, str): @@ -698,9 +714,9 @@ def shift_excel_dates_inplace( ( col_idx, col_names[col_idx], - ws.cell(row=row_idx + 1, column=col_idx + 1).value, + page[row_idx, col_idx].value, ) - for col_idx in range(ws.max_column) + for col_idx in range(page.columns) ] # keep the cells that aren't blank if cell[2] is not None and cell[2].strip() != "" @@ -723,13 +739,13 @@ def shift_excel_dates_inplace( # now scan each column # ... even when page has no date_columns; still check for hidden dates - for col_idx in range(ws.max_column): + for col_idx in range(page.columns): # skip the patient id column (it was already checked anyway) if col_idx == patient_id_idx: continue # get the cell value. replace it with None if it's just whitespace - cell = ws.cell(row=row_idx + 1, column=col_idx + 1) + cell = page[row_idx, col_idx] cell_value = cell.value if isinstance(cell_value, str): cell_value = cell_value.strip() @@ -792,13 +808,6 @@ def shift_excel_dates_inplace( logger.info(f"Shifting {sheet_name=} processed {row_idx} rows") - wb.save(output_file) - logger.info("Output written to '%s'", output_file) - - linking_path = linking_table_output or "shift_mappings.csv" - shift_mappings.to_csv(linking_path, index=False) - logger.info("Linking table saved to '%s'", linking_path) - # Re-export public API from nuh_helper.date_shift.mappings import ( # noqa: E402 diff --git a/pyproject.toml b/pyproject.toml index 47ade72..a5eb216 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "pandas>=3.0.5", "openpyxl>=3.1.0", "datefinder>=1.0.0", + "book-page", ] [build-system] @@ -37,6 +38,9 @@ dev = [ [tool.uv] exclude-newer = "7 days" +[tool.uv.sources] +book-page = { git = "https://github.com/peterlavalle/book_page.git" } + [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/uv.lock b/uv.lock index e70b55f..6862f4e 100644 --- a/uv.lock +++ b/uv.lock @@ -14,6 +14,14 @@ resolution-markers = [ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P7D" +[[package]] +name = "book-page" +version = "0.1.0" +source = { git = "https://github.com/peterlavalle/book_page.git#15e9f9cb09cc69a2ba74cebc8a921d77ddd56b41" } +dependencies = [ + { name = "openpyxl" }, +] + [[package]] name = "cfgv" version = "3.5.0" @@ -194,6 +202,7 @@ wheels = [ name = "nuh-helper" source = { editable = "." } dependencies = [ + { name = "book-page" }, { name = "datefinder" }, { name = "openpyxl" }, { name = "pandas" }, @@ -213,6 +222,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "book-page", git = "https://github.com/peterlavalle/book_page.git" }, { name = "datefinder", specifier = ">=1.0.0" }, { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pandas", specifier = ">=3.0.5" }, From 3fa2186421b79edc6c336965e8a7f7bc97c9acf3 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 1 Oct 2026 16:12:04 +0100 Subject: [PATCH 21/82] updates the rest of the date shift - creates IDs anew - adjusts coverage rules --- .gitignore | 1 + nuh_helper/date_shift/__init__.py | 149 ++++++++++++++++++++++-------- pyproject.toml | 3 + tests/test_hidden_dates.py | 4 - tests/test_missing_id.py | 1 - tests/test_shift_ignore.py | 8 -- tests/test_structural.py | 6 -- uv.lock | 2 +- 8 files changed, 115 insertions(+), 59 deletions(-) diff --git a/.gitignore b/.gitignore index c7f194b..f068ccf 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ tests/output/ *.jsonl.tar.gz .DS_Store +.coverage diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 802c42d..b791429 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -5,15 +5,16 @@ in an Excel file, with support for reproducible shifts using a linking table. """ +import csv import logging +import random from datetime import date, datetime from pathlib import Path from typing import Any, cast import datefinder import pandas as pd -from book_page import CSVBook, XLSXBook -from book_page.book import Page +from book_page import CSVBook, Page, XLSXBook from nuh_helper.date_shift import _excel, _parse, mappings @@ -464,15 +465,12 @@ def shift_excel_dates_inplace( input_file: str | Path | list[Path], output_file: str | Path, patient_sheet: str, - patient_id_col: str, sheet_configs: dict[str, dict[str, Any]], min_shift_days: int = -15, max_shift_days: int = 15, linking_table_path: str | None = None, linking_table_output: str | None = None, seed: int | None = None, - header_row: int = 0, - skip_rows: list[int] | None = None, ) -> None: """ Shift dates in an Excel file, preserving all cell formatting. @@ -485,7 +483,6 @@ def shift_excel_dates_inplace( input_file: Path to input Excel file. output_file: Path for the output file (copy of input with shifted dates). patient_sheet: Name of the sheet containing patient IDs. - patient_id_col: Name of the column containing patient IDs in the patient sheet. sheet_configs: Dictionary mapping sheet names to configuration dicts, or, the string 'skip' if that sheet should be skipped but is a valid part of the @@ -507,8 +504,6 @@ def shift_excel_dates_inplace( linking_table_path: Optional path to existing linking table CSV for reproducibility. linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). seed: Optional random seed for generating shifts. - patient_header_row: Zero-based header row index for the patient sheet (default: 0). - patient_skip_rows: Optional zero-based row indices to exclude from patient data. """ # noqa: E501 logger.info("Shifting dates in-place: '%s' → '%s'", input_file, output_file) logger.debug( @@ -523,31 +518,28 @@ def shift_excel_dates_inplace( else: book: CSVBook = CSVBook.copy(input_file, output_file) - _patient_ids, shift_mappings = _get_patient_ids_and_shift_mappings( - input_file=book, - patient_sheet=patient_sheet, - patient_id_col=patient_id_col, - sheet_configs=sheet_configs, - min_shift_days=min_shift_days, - max_shift_days=max_shift_days, - linking_table_path=linking_table_path, - seed=seed, - patient_header_row=header_row, - patient_skip_rows=skip_rows, - ) - - shift_dict: dict[str, int] = dict( - zip( - shift_mappings["patient_id"], - shift_mappings["shift_days"], - strict=True, + ## ### + ## double check that the parameters match what's in the sheet config + + assert patient_sheet in sheet_configs + + with book[patient_sheet] as page: + shift_deltas = patient_shift_deltas( + page, + linking_table_path, + linking_table_output or "shift_mappings.csv", + sheet_configs[patient_sheet]["patient_id_col"], + sheet_configs[patient_sheet]["header_row"], + sheet_configs[patient_sheet].get("skip_rows", []), + seed, + min_shift_days, + max_shift_days, ) - ) # check for sheets we didn't have an explanation for - for sheet_name in book: - if sheet_name not in sheet_configs: - raise ExtraPage(sheet_name) + for page in book: + if page.name not in sheet_configs: + raise ExtraPage(page.name) # start looping through each sheet in the configuration for sheet_name, sheet_config in sheet_configs.items(): @@ -567,18 +559,14 @@ def shift_excel_dates_inplace( # these will raise errors if the keys are missing - that's fine with book[sheet_name] as page: function_that_preserves_indentation( - shift_dict, sheet_name, sheet_config, page + shift_deltas, sheet_name, sheet_config, page ) logger.info("Output written to '%s'", output_file) - linking_path = linking_table_output or "shift_mappings.csv" - shift_mappings.to_csv(linking_path, index=False) - logger.info("Linking table saved to '%s'", linking_path) - def function_that_preserves_indentation( - shift_dict: dict[str, pd.Timedelta], + shift_deltas: dict[str, pd.Timedelta], sheet_name: str, sheet_config: dict[str, any], page: Page, @@ -731,11 +719,10 @@ def function_that_preserves_indentation( continue # determine how many days to shift - shift_days = shift_dict.get(pid) - if shift_days is None: + shift_delta = shift_deltas.get(pid) + if shift_delta is None: # all patients need to have a shift value raise UnknownPatient(sheet_name, pid) - shift_delta = pd.Timedelta(days=shift_days) # now scan each column # ... even when page has no date_columns; still check for hidden dates @@ -809,6 +796,90 @@ def function_that_preserves_indentation( logger.info(f"Shifting {sheet_name=} processed {row_idx} rows") +def patient_shift_deltas( + page: Page, + src_linking_table: str | Path, + out_linking_table: str | Path, + patient_id_col_name: str, + patient_header_row: int, + patient_skip_rows: list[int] | None, + seed: int | None, + min_shift_days: int, + max_shift_days: int, +) -> dict[str, pd.Timedelta]: + # find the patient id index + patient_id_col: None | int = None + for col in range(page.columns): + if page[patient_header_row, col].value == patient_id_col_name: + if patient_id_col is not None: + raise RuntimeError(f"multiple columns with {patient_id_col_name=}") + else: + patient_id_col = col + if patient_id_col is None: + patient_id_col = patient_id_col_name + sheet_name = page.name + # it's done like this to get the same error message as above + raise NameError(f"{patient_id_col=} column not in {sheet_name=}") + + # find the/a list of all ids + patient_ids = sorted( + _parse._normalize_patient_id(patient_id) + for patient_id in [ + page[row, patient_id_col].value.strip() + for row in range( + max( + [patient_header_row] + + (patient_skip_rows if patient_skip_rows is not None else []) + ) + + 1, + page.rows, + ) + ] + if patient_id + ) + logger.info(f"Found {len(patient_ids)=} in {page.name=}") + + # load the old ids (should we remove IDs that are "gone"?) + if isinstance(src_linking_table, str): + src_linking_table = Path(src_linking_table) + if not src_linking_table.is_file(): + old_mappings: dict[str, int] = {} + else: + with open(src_linking_table) as file: + old_mappings: dict[str, int] = { + _parse._normalize_patient_id(row[0]): int(row[1]) + for row in csv.reader(file) + if row and row[1] and row[1].strip() != "shift_days" + # if row and row[0] and row[0].strip() != "patient_id" + } + logger.info(f"Loaded {len(old_mappings)=} in {src_linking_table =}") + + # create the list of all mappings + + if seed: + random.seed(seed) + all_mappings: dict[str, int] = { + id: ( + old_mappings[id] + if id in old_mappings + else random.randint(min_shift_days, max_shift_days) + ) + for id in patient_ids + } + removed = len([k for k in all_mappings if k not in patient_ids]) + logger.info(f"Total of {len(all_mappings)} patient(s) (... and {removed=})") + + # store all mappings in the out file + with open(out_linking_table, "w") as file: + csv.writer(file).writerows( + [["patient_id", "shift_days"]] + + [[key, all_mappings[key]] for key in sorted(all_mappings)] + ) + logger.info(f"Saved mappings {out_linking_table}") + + return {k: pd.Timedelta(days=all_mappings[k]) for k in all_mappings} + + # Re-export public API from nuh_helper.date_shift.mappings import ( # noqa: E402 generate_shift_mappings, diff --git a/pyproject.toml b/pyproject.toml index a5eb216..0940fcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,9 @@ select = ["E", "F", "ANN", "B", "UP", "I", "SIM", "C4"] [tool.coverage.run] branch = true +omit = [ + "tests/*", +] [tool.pytest.ini_options] addopts = "-ra" diff --git a/tests/test_hidden_dates.py b/tests/test_hidden_dates.py index 373f516..e3ed551 100644 --- a/tests/test_hidden_dates.py +++ b/tests/test_hidden_dates.py @@ -34,7 +34,6 @@ def test_iso8601(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="args", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -75,7 +74,6 @@ def test_us_date(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="data", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -116,7 +114,6 @@ def test_written(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="yeah", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -157,7 +154,6 @@ def test_hidden_in_patient_id(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="data", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/tests/test_missing_id.py b/tests/test_missing_id.py index 5496e4e..b6eb3a6 100644 --- a/tests/test_missing_id.py +++ b/tests/test_missing_id.py @@ -39,7 +39,6 @@ def test_missing_id(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="patients", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 849c066..04ec291 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -31,7 +31,6 @@ def test_shift_exceptions(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -68,7 +67,6 @@ def test_ptid_in_date_columns(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -105,7 +103,6 @@ def test_ptid_in_text_columns(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -142,7 +139,6 @@ def test_columns_overlap(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -180,7 +176,6 @@ def test_ptid_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -216,7 +211,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -299,7 +293,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-desc", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -309,7 +302,6 @@ def body() -> None: ) if not shift_ignore: - print(output_path) with pytest.raises(HiddenDate) as info: body() assert str(info.value._message) == ( diff --git a/tests/test_structural.py b/tests/test_structural.py index 2de3ea9..c627934 100644 --- a/tests/test_structural.py +++ b/tests/test_structural.py @@ -43,7 +43,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -90,7 +89,6 @@ def test_date_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -127,7 +125,6 @@ def test_extra_column(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -167,7 +164,6 @@ def test_text_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -217,7 +213,6 @@ def test_page_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -255,7 +250,6 @@ def test_page_extra(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/uv.lock b/uv.lock index 6862f4e..0ff3385 100644 --- a/uv.lock +++ b/uv.lock @@ -17,7 +17,7 @@ exclude-newer-span = "P7D" [[package]] name = "book-page" version = "0.1.0" -source = { git = "https://github.com/peterlavalle/book_page.git#15e9f9cb09cc69a2ba74cebc8a921d77ddd56b41" } +source = { git = "https://github.com/peterlavalle/book_page.git#3243604f4d7d3dcae9bf39852aa9b5787dce3052" } dependencies = [ { name = "openpyxl" }, ] From 38ece50f549db752195ec51dd4de222e65e09573 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 1 Oct 2026 16:52:15 +0100 Subject: [PATCH 22/82] improved function and comments --- nuh_helper/date_shift/__init__.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index b791429..5f510da 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -14,7 +14,7 @@ import datefinder import pandas as pd -from book_page import CSVBook, Page, XLSXBook +from book_page import Book, Page, book_copy from nuh_helper.date_shift import _excel, _parse, mappings @@ -159,7 +159,7 @@ def __init__(self, page_name: str) -> None: def _get_patient_ids_and_shift_mappings( - input_file: XLSXBook | CSVBook, + input_file: Path, patient_sheet: str, patient_id_col: str, sheet_configs: dict[str, dict[str, Any]], @@ -513,15 +513,10 @@ def shift_excel_dates_inplace( seed, ) - if str(input_file).endswith(".xlsx"): - book: XLSXBook = XLSXBook.copy(input_file, output_file) - else: - book: CSVBook = CSVBook.copy(input_file, output_file) - - ## ### - ## double check that the parameters match what's in the sheet config + book: Book = book_copy(input_file, output_file) - assert patient_sheet in sheet_configs + if patient_sheet not in sheet_configs: + raise PageMissing(patient_sheet) with book[patient_sheet] as page: shift_deltas = patient_shift_deltas( @@ -556,22 +551,20 @@ def shift_excel_dates_inplace( continue logger.info(f"Shifting {sheet_name=}") - # these will raise errors if the keys are missing - that's fine - with book[sheet_name] as page: - function_that_preserves_indentation( - shift_deltas, sheet_name, sheet_config, page - ) + # shift the page + _shift_book_page(shift_deltas, sheet_name, sheet_config, book) logger.info("Output written to '%s'", output_file) -def function_that_preserves_indentation( +def _shift_book_page( shift_deltas: dict[str, pd.Timedelta], sheet_name: str, sheet_config: dict[str, any], - page: Page, + book: Book, ) -> None: - if True: + """shifts a single page. the nesting/breakout was done to minimize git changes""" + with book[sheet_name] as page: patient_id_col: str = cast(str, sheet_config["patient_id_col"]).strip() date_columns: list[str] = [col.strip() for col in sheet_config["date_columns"]] text_columns: list[str] = [col.strip() for col in sheet_config["text_columns"]] From 724752691b1b090eb72b43bc9f1b1f4a1d87a0bc Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 1 Oct 2026 17:03:36 +0100 Subject: [PATCH 23/82] uses the new exception --- nuh_helper/date_shift/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 5f510da..3c77ca6 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -809,10 +809,7 @@ def patient_shift_deltas( else: patient_id_col = col if patient_id_col is None: - patient_id_col = patient_id_col_name - sheet_name = page.name - # it's done like this to get the same error message as above - raise NameError(f"{patient_id_col=} column not in {sheet_name=}") + raise PatientColumnMissing(page.name, patient_id_col_name) # find the/a list of all ids patient_ids = sorted( From db26a0cf84310d923b647f810a1e3887c51a5bde Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 2 Oct 2026 14:09:58 +0100 Subject: [PATCH 24/82] changes to allow pip to access book_page --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0940fcf..da9350a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,8 +9,10 @@ dependencies = [ "pandas>=3.0.5", "openpyxl>=3.1.0", "datefinder>=1.0.0", - "book-page", + "book_page @ git+https://github.com/peterlavalle/book_page.git@main", ] +[tool.hatch.metadata] +allow-direct-references = true [build-system] requires = ["hatchling", "hatch-vcs"] From 65de773121647cd93adcd5c5410aa858f19be7e9 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 2 Oct 2026 14:54:33 +0100 Subject: [PATCH 25/82] small logging messages --- nuh_helper/date_shift/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 3c77ca6..99b4c08 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -505,7 +505,13 @@ def shift_excel_dates_inplace( linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). seed: Optional random seed for generating shifts. """ # noqa: E501 - logger.info("Shifting dates in-place: '%s' → '%s'", input_file, output_file) + logger.info( + "Shifting dates in-place: '%s' → '%s'", + input_file + if not isinstance(input_file, list) + else [file.name for file in input_file], + output_file, + ) logger.debug( "Shift range: %d to %d days, seed=%s", min_shift_days, @@ -514,6 +520,7 @@ def shift_excel_dates_inplace( ) book: Book = book_copy(input_file, output_file) + logger.info(f"Book has {type(book)=}") if patient_sheet not in sheet_configs: raise PageMissing(patient_sheet) @@ -530,6 +537,7 @@ def shift_excel_dates_inplace( min_shift_days, max_shift_days, ) + logger.info(f"loaded deltas {len(shift_deltas)=}") # check for sheets we didn't have an explanation for for page in book: From 77245b8569d491a5e2fa6c6e4a2b256ca5986d33 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 2 Oct 2026 15:03:35 +0100 Subject: [PATCH 26/82] more logging --- nuh_helper/date_shift/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 99b4c08..6b1d4bf 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -808,6 +808,7 @@ def patient_shift_deltas( min_shift_days: int, max_shift_days: int, ) -> dict[str, pd.Timedelta]: + logger.info("patient_shift_deltas() ;preparing per-patient shifts") # find the patient id index patient_id_col: None | int = None for col in range(page.columns): @@ -818,6 +819,7 @@ def patient_shift_deltas( patient_id_col = col if patient_id_col is None: raise PatientColumnMissing(page.name, patient_id_col_name) + logger.info(f"patient_shift_deltas() ; {patient_id_col=}") # find the/a list of all ids patient_ids = sorted( @@ -835,7 +837,7 @@ def patient_shift_deltas( ] if patient_id ) - logger.info(f"Found {len(patient_ids)=} in {page.name=}") + logger.info(f"patient_shift_deltas() ; Found {len(patient_ids)=} in {page.name=}") # load the old ids (should we remove IDs that are "gone"?) if isinstance(src_linking_table, str): @@ -850,7 +852,9 @@ def patient_shift_deltas( if row and row[1] and row[1].strip() != "shift_days" # if row and row[0] and row[0].strip() != "patient_id" } - logger.info(f"Loaded {len(old_mappings)=} in {src_linking_table =}") + logger.info( + f"patient_shift_deltas() ; Loaded {len(old_mappings)=} in {src_linking_table =}" + ) # create the list of all mappings @@ -865,7 +869,9 @@ def patient_shift_deltas( for id in patient_ids } removed = len([k for k in all_mappings if k not in patient_ids]) - logger.info(f"Total of {len(all_mappings)} patient(s) (... and {removed=})") + logger.info( + f"patient_shift_deltas() ; {len(all_mappings)=} patient(s) (... and {removed=})" + ) # store all mappings in the out file with open(out_linking_table, "w") as file: @@ -873,7 +879,7 @@ def patient_shift_deltas( [["patient_id", "shift_days"]] + [[key, all_mappings[key]] for key in sorted(all_mappings)] ) - logger.info(f"Saved mappings {out_linking_table}") + logger.info(f"patient_shift_deltas() ; Saved mappings {out_linking_table}") return {k: pd.Timedelta(days=all_mappings[k]) for k in all_mappings} From 9e98b1ec294ba84c07641cf7d75f2f102f109a9b Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 2 Oct 2026 15:16:18 +0100 Subject: [PATCH 27/82] log during id generation --- nuh_helper/date_shift/__init__.py | 36 +++++++++++++++++++------------ 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 6b1d4bf..11e2a3b 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -822,21 +822,29 @@ def patient_shift_deltas( logger.info(f"patient_shift_deltas() ; {patient_id_col=}") # find the/a list of all ids - patient_ids = sorted( - _parse._normalize_patient_id(patient_id) - for patient_id in [ - page[row, patient_id_col].value.strip() - for row in range( - max( - [patient_header_row] - + (patient_skip_rows if patient_skip_rows is not None else []) - ) - + 1, - page.rows, - ) - ] - if patient_id + patient_ids = [] + first_row = ( + max( + [patient_header_row] + + (patient_skip_rows if patient_skip_rows is not None else []) + ) + + 1 ) + logger.info(f"patient_shift_deltas() ; {first_row=}") + + for row in range( + first_row, + page.rows, + ): + patient_id = page[row, patient_id_col].value + if not patient_id: + continue + patient_id = _parse._normalize_patient_id(patient_id.strip()) + if patient_id in patient_ids: + continue + patient_ids.append(patient_id) + if (len(patient_ids) % 40) == 0: + logger.info(f"patient_shift_deltas() ; so far {len(patient_ids)=}") logger.info(f"patient_shift_deltas() ; Found {len(patient_ids)=} in {page.name=}") # load the old ids (should we remove IDs that are "gone"?) From dc1b873560150d3a74567e7bd6f960b57142187e Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 2 Oct 2026 15:20:16 +0100 Subject: [PATCH 28/82] remove the pd call --- nuh_helper/date_shift/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 11e2a3b..704e23e 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -839,7 +839,8 @@ def patient_shift_deltas( patient_id = page[row, patient_id_col].value if not patient_id: continue - patient_id = _parse._normalize_patient_id(patient_id.strip()) + # patient_id = _parse._normalize_patient_id(patient_id.strip()) + patient_id = patient_id.strip() if patient_id in patient_ids: continue patient_ids.append(patient_id) From 651c457b04216fde791127fadb79da236fc81947 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 2 Oct 2026 15:25:51 +0100 Subject: [PATCH 29/82] restored it - it wasn't the problem --- nuh_helper/date_shift/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 704e23e..27f9f1c 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -8,6 +8,7 @@ import csv import logging import random +import time from datetime import date, datetime from pathlib import Path from typing import Any, cast @@ -823,6 +824,7 @@ def patient_shift_deltas( # find the/a list of all ids patient_ids = [] + start = time.time() first_row = ( max( [patient_header_row] @@ -839,13 +841,15 @@ def patient_shift_deltas( patient_id = page[row, patient_id_col].value if not patient_id: continue - # patient_id = _parse._normalize_patient_id(patient_id.strip()) - patient_id = patient_id.strip() + patient_id = _parse._normalize_patient_id(patient_id.strip()) if patient_id in patient_ids: continue patient_ids.append(patient_id) if (len(patient_ids) % 40) == 0: - logger.info(f"patient_shift_deltas() ; so far {len(patient_ids)=}") + seconds = time.time() - start + logger.info( + f"patient_shift_deltas() ; so far {len(patient_ids)=} in {seconds=}" + ) logger.info(f"patient_shift_deltas() ; Found {len(patient_ids)=} in {page.name=}") # load the old ids (should we remove IDs that are "gone"?) From 57e1bd2423c458ab966260cdb34f065267915b94 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 10:14:01 +0100 Subject: [PATCH 30/82] ignore the coverage database --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index c7f194b..4cafa08 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ dist/ wheels/ *.egg-info .mypy_cache/ +.coverage # Virtual environments .venv From 117a386ad4f17e5f72c63e24f852f3ed907eda58 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 10:33:29 +0100 Subject: [PATCH 31/82] removes duplicate parameters from _inplace --- nuh_helper/date_shift/__init__.py | 18 ++++++++++++------ tests/test_hidden_dates.py | 4 ---- tests/test_missing_id.py | 1 - tests/test_shift_ignore.py | 7 ------- tests/test_structural.py | 6 ------ 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index e184217..cc67b11 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -195,9 +195,7 @@ def _get_patient_ids_and_shift_mappings( skip_rows_after_header=effective_patient_skip_rows, ) if patient_id_col not in patient_df.columns: - raise ValueError( - f"Patient ID column '{patient_id_col}' not found in sheet '{patient_sheet}'" - ) + raise PatientColumnMissing(patient_sheet, patient_id_col) patient_ids = ( patient_df[patient_id_col] @@ -465,15 +463,12 @@ def shift_excel_dates_inplace( input_file: str, output_file: str, patient_sheet: str, - patient_id_col: str, sheet_configs: dict[str, dict[str, Any]], min_shift_days: int = -15, max_shift_days: int = 15, linking_table_path: str | None = None, linking_table_output: str | None = None, seed: int | None = None, - header_row: int = 0, - skip_rows: list[int] | None = None, ) -> None: """ Shift dates in an Excel file, preserving all cell formatting. @@ -519,6 +514,17 @@ def shift_excel_dates_inplace( seed, ) + # read some parameters from the config (rather than asking for them in the invoke) + if patient_sheet not in sheet_configs: + raise ValueError(f"{patient_sheet=}") + patient_id_col: str = sheet_configs[patient_sheet]["patient_id_col"] + header_row: int = sheet_configs[patient_sheet].get("header_row", 0) + skip_rows: list[int] = sheet_configs[patient_sheet].get( + "skip_rows_after_header", [] + ) + if not skip_rows: + skip_rows = [] + shutil.copy2(input_file, output_file) _patient_ids, shift_mappings = _get_patient_ids_and_shift_mappings( diff --git a/tests/test_hidden_dates.py b/tests/test_hidden_dates.py index 373f516..e3ed551 100644 --- a/tests/test_hidden_dates.py +++ b/tests/test_hidden_dates.py @@ -34,7 +34,6 @@ def test_iso8601(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="args", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -75,7 +74,6 @@ def test_us_date(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="data", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -116,7 +114,6 @@ def test_written(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="yeah", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -157,7 +154,6 @@ def test_hidden_in_patient_id(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="data", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/tests/test_missing_id.py b/tests/test_missing_id.py index 5496e4e..b6eb3a6 100644 --- a/tests/test_missing_id.py +++ b/tests/test_missing_id.py @@ -39,7 +39,6 @@ def test_missing_id(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="patients", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 849c066..df0c79c 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -31,7 +31,6 @@ def test_shift_exceptions(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -68,7 +67,6 @@ def test_ptid_in_date_columns(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -105,7 +103,6 @@ def test_ptid_in_text_columns(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -142,7 +139,6 @@ def test_columns_overlap(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -180,7 +176,6 @@ def test_ptid_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -216,7 +211,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -299,7 +293,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-desc", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/tests/test_structural.py b/tests/test_structural.py index 2de3ea9..c627934 100644 --- a/tests/test_structural.py +++ b/tests/test_structural.py @@ -43,7 +43,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -90,7 +89,6 @@ def test_date_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -127,7 +125,6 @@ def test_extra_column(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -167,7 +164,6 @@ def test_text_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -217,7 +213,6 @@ def test_page_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -255,7 +250,6 @@ def test_page_extra(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, From 3fc3d9d33ebe4aa834d8c674a5dd10d9edba043c Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 10:43:55 +0100 Subject: [PATCH 32/82] update docs (oops) --- README.md | 3 ++- nuh_helper/date_shift/__init__.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c1c8fbc..f939371 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ shift_excel_dates( If your workbook has rich formatting (cell styles, column widths, conditional formatting, etc.) use `shift_excel_dates_inplace` instead. It copies the input file and modifies date cells directly via openpyxl, so all formatting is preserved exactly. +> `shift_excel_dates_inplace()` gets the `patient_id_col` and other `patient_sheet` values from the `sheet_config` rather than duplicate the configuration in the function call + ```python from nuh_helper import shift_excel_dates_inplace @@ -114,7 +116,6 @@ shift_excel_dates_inplace( input_file="input.xlsx", output_file="output.xlsx", patient_sheet="patients", - patient_id_col="patient_id", sheet_configs=sheet_configs, seed=42, linking_table_output="shift_mappings.csv", diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index cc67b11..f733301 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -481,7 +481,6 @@ def shift_excel_dates_inplace( input_file: Path to input Excel file. output_file: Path for the output file (copy of input with shifted dates). patient_sheet: Name of the sheet containing patient IDs. - patient_id_col: Name of the column containing patient IDs in the patient sheet. sheet_configs: Dictionary mapping sheet names to configuration dicts, or, the string 'skip' if that sheet should be skipped but is a valid part of the @@ -503,8 +502,6 @@ def shift_excel_dates_inplace( linking_table_path: Optional path to existing linking table CSV for reproducibility. linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). seed: Optional random seed for generating shifts. - patient_header_row: Zero-based header row index for the patient sheet (default: 0). - patient_skip_rows: Optional zero-based row indices to exclude from patient data. """ # noqa: E501 logger.info("Shifting dates in-place: '%s' → '%s'", input_file, output_file) logger.debug( From 68309d8b2b2503c500760ae82366c4c34625c05a Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 10:33:29 +0100 Subject: [PATCH 33/82] removes duplicate parameters from _inplace --- nuh_helper/date_shift/__init__.py | 18 ++++++++++++------ tests/test_hidden_dates.py | 4 ---- tests/test_missing_id.py | 1 - tests/test_shift_ignore.py | 7 ------- tests/test_structural.py | 6 ------ 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index e184217..cc67b11 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -195,9 +195,7 @@ def _get_patient_ids_and_shift_mappings( skip_rows_after_header=effective_patient_skip_rows, ) if patient_id_col not in patient_df.columns: - raise ValueError( - f"Patient ID column '{patient_id_col}' not found in sheet '{patient_sheet}'" - ) + raise PatientColumnMissing(patient_sheet, patient_id_col) patient_ids = ( patient_df[patient_id_col] @@ -465,15 +463,12 @@ def shift_excel_dates_inplace( input_file: str, output_file: str, patient_sheet: str, - patient_id_col: str, sheet_configs: dict[str, dict[str, Any]], min_shift_days: int = -15, max_shift_days: int = 15, linking_table_path: str | None = None, linking_table_output: str | None = None, seed: int | None = None, - header_row: int = 0, - skip_rows: list[int] | None = None, ) -> None: """ Shift dates in an Excel file, preserving all cell formatting. @@ -519,6 +514,17 @@ def shift_excel_dates_inplace( seed, ) + # read some parameters from the config (rather than asking for them in the invoke) + if patient_sheet not in sheet_configs: + raise ValueError(f"{patient_sheet=}") + patient_id_col: str = sheet_configs[patient_sheet]["patient_id_col"] + header_row: int = sheet_configs[patient_sheet].get("header_row", 0) + skip_rows: list[int] = sheet_configs[patient_sheet].get( + "skip_rows_after_header", [] + ) + if not skip_rows: + skip_rows = [] + shutil.copy2(input_file, output_file) _patient_ids, shift_mappings = _get_patient_ids_and_shift_mappings( diff --git a/tests/test_hidden_dates.py b/tests/test_hidden_dates.py index 373f516..e3ed551 100644 --- a/tests/test_hidden_dates.py +++ b/tests/test_hidden_dates.py @@ -34,7 +34,6 @@ def test_iso8601(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="args", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -75,7 +74,6 @@ def test_us_date(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="data", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -116,7 +114,6 @@ def test_written(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="yeah", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -157,7 +154,6 @@ def test_hidden_in_patient_id(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="data", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/tests/test_missing_id.py b/tests/test_missing_id.py index 5496e4e..b6eb3a6 100644 --- a/tests/test_missing_id.py +++ b/tests/test_missing_id.py @@ -39,7 +39,6 @@ def test_missing_id(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="patients", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 849c066..df0c79c 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -31,7 +31,6 @@ def test_shift_exceptions(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -68,7 +67,6 @@ def test_ptid_in_date_columns(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -105,7 +103,6 @@ def test_ptid_in_text_columns(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -142,7 +139,6 @@ def test_columns_overlap(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -180,7 +176,6 @@ def test_ptid_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -216,7 +211,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-data", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -299,7 +293,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="page-desc", - patient_id_col="pid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, diff --git a/tests/test_structural.py b/tests/test_structural.py index 2de3ea9..c627934 100644 --- a/tests/test_structural.py +++ b/tests/test_structural.py @@ -43,7 +43,6 @@ def body() -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -90,7 +89,6 @@ def test_date_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -127,7 +125,6 @@ def test_extra_column(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -167,7 +164,6 @@ def test_text_column_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -217,7 +213,6 @@ def test_page_missing(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, @@ -255,7 +250,6 @@ def test_page_extra(tmp_path: Path) -> None: input_file=str(source_file), output_file=str(output_path), patient_sheet="paige", - patient_id_col="ptid", sheet_configs=sheet_configs, min_shift_days=-20, max_shift_days=-1, From 700a5684997f382bde6b5fb8983ec4974e88014b Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 10:43:55 +0100 Subject: [PATCH 34/82] update docs (oops) --- README.md | 3 ++- nuh_helper/date_shift/__init__.py | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c1c8fbc..f939371 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ shift_excel_dates( If your workbook has rich formatting (cell styles, column widths, conditional formatting, etc.) use `shift_excel_dates_inplace` instead. It copies the input file and modifies date cells directly via openpyxl, so all formatting is preserved exactly. +> `shift_excel_dates_inplace()` gets the `patient_id_col` and other `patient_sheet` values from the `sheet_config` rather than duplicate the configuration in the function call + ```python from nuh_helper import shift_excel_dates_inplace @@ -114,7 +116,6 @@ shift_excel_dates_inplace( input_file="input.xlsx", output_file="output.xlsx", patient_sheet="patients", - patient_id_col="patient_id", sheet_configs=sheet_configs, seed=42, linking_table_output="shift_mappings.csv", diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index cc67b11..f733301 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -481,7 +481,6 @@ def shift_excel_dates_inplace( input_file: Path to input Excel file. output_file: Path for the output file (copy of input with shifted dates). patient_sheet: Name of the sheet containing patient IDs. - patient_id_col: Name of the column containing patient IDs in the patient sheet. sheet_configs: Dictionary mapping sheet names to configuration dicts, or, the string 'skip' if that sheet should be skipped but is a valid part of the @@ -503,8 +502,6 @@ def shift_excel_dates_inplace( linking_table_path: Optional path to existing linking table CSV for reproducibility. linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). seed: Optional random seed for generating shifts. - patient_header_row: Zero-based header row index for the patient sheet (default: 0). - patient_skip_rows: Optional zero-based row indices to exclude from patient data. """ # noqa: E501 logger.info("Shifting dates in-place: '%s' → '%s'", input_file, output_file) logger.debug( From 83c87ef29197db523fabd832c879d08793c79bf1 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 11:15:59 +0100 Subject: [PATCH 35/82] shift ignore reading from YAML --- nuh_helper/date_shift/__init__.py | 55 +++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index f733301..2e4e197 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -13,6 +13,7 @@ import datefinder import pandas as pd +import yaml from openpyxl import load_workbook from openpyxl.worksheet.worksheet import Worksheet @@ -469,6 +470,7 @@ def shift_excel_dates_inplace( linking_table_path: str | None = None, linking_table_output: str | None = None, seed: int | None = None, + shift_ignore_yaml: None | str | Path = None, ) -> None: """ Shift dates in an Excel file, preserving all cell formatting. @@ -494,14 +496,17 @@ def shift_excel_dates_inplace( - 'skip_rows_after_header': list of zero-based row indices to exclude from data (e.g. a data-type row immediately below the header) - - `shift_ignore`: (Optional) Dict mapping `page:{column:[values]}` to - lists of values that are passed as-is with no manipulation or - checking. min_shift_days: Minimum number of days to shift (default: -15). max_shift_days: Maximum number of days to shift (default: 15). linking_table_path: Optional path to existing linking table CSV for reproducibility. linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). seed: Optional random seed for generating shifts. + shift_ignore_yaml: + path to a .yaml file holding `page:{column:[values]}` lists of cell + values that're ignored and passed as-is with no manipulation. None + and "" are always added, and all values will be .strip() + + (Defaults to linking_table_path.parent / shift_ignore.do-not-commit.yaml) """ # noqa: E501 logger.info("Shifting dates in-place: '%s' → '%s'", input_file, output_file) logger.debug( @@ -511,6 +516,35 @@ def shift_excel_dates_inplace( seed, ) + # TODO; it'd be cool to "normalize" the linking_table_path/linking_table_output here + + # get the shift_ignore: dict[str, dict[str, list[str]]] from shift_ignore_yaml: Path + if not shift_ignore_yaml: + if linking_table_path: + shift_ignore_yaml = ( + Path(linking_table_path).parent / "shift_ignore.do-not-commit.yaml" + ) + else: + raise RuntimeError( + "can't guess shift_ignore_yaml without linking_table_path" + ) + elif not isinstance(shift_ignore_yaml, Path): + shift_ignore_yaml = Path(shift_ignore_yaml) + + if not shift_ignore_yaml.is_file(): + shift_ignore: dict[str, dict[str, set[str]]] = {} + else: + with open(shift_ignore_yaml) as file: + data = yaml.safe_load(file) + shift_ignore: dict[str, dict[str, list[str]]] = { + page.strip(): { + col.strip(): {ignored.strip for ignored in data[page][col]} + + {"", None} + for col in data[page] + } + for page in data + } + # read some parameters from the config (rather than asking for them in the invoke) if patient_sheet not in sheet_configs: raise ValueError(f"{patient_sheet=}") @@ -644,15 +678,6 @@ def shift_excel_dates_inplace( if missing: raise DateColumnsMissing(sheet_name, missing) - # map and strip the shift_ignore values - shift_ignore = { - col_name: [ - ignore.strip() - for ignore in sheet_config.get("shift_ignore", {}).get(col_name, []) - ] - for col_name in col_names - } - ### # process each row of the workbook ## @@ -752,7 +777,11 @@ def shift_excel_dates_inplace( raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) # skip values in shift_ignore - if cell_value in shift_ignore[col_name]: + if sheet_name not in shift_ignore: + shift_ignore[sheet_name] = {} + if col_name not in shift_ignore[sheet_name]: + shift_ignore[sheet_name][col_name] = {None, ""} + if cell_value in shift_ignore[sheet_name][col_name]: continue # for text columns, we now *just* need to check for a hidden date From 5091e8e4a627009134be7b27976a629dfd58ff8c Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 11:36:35 +0100 Subject: [PATCH 36/82] detects old (bad) configs --- nuh_helper/date_shift/__init__.py | 2 ++ tests/test_shift_ignore.py | 54 ++++++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 2e4e197..fa2a5e3 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -777,6 +777,8 @@ def shift_excel_dates_inplace( raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) # skip values in shift_ignore + if "shift_ignore" in sheet_config: + raise RuntimeError("move shift_ignore from sheet_configs to a file") if sheet_name not in shift_ignore: shift_ignore[sheet_name] = {} if col_name not in shift_ignore[sheet_name]: diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index df0c79c..32a3908 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -288,7 +288,7 @@ def test_ignore_in_text_columns(shift_ignore: bool, tmp_path: Path) -> None: "page-data": "skip", } - def body() -> None: + def body(shift_ignore_yaml: None | Path) -> None: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -299,12 +299,12 @@ def body() -> None: seed=14333, linking_table_path=str(linking_table_old), linking_table_output=str(linking_table_out), + shift_ignore_yaml=shift_ignore_yaml, ) if not shift_ignore: - print(output_path) with pytest.raises(HiddenDate) as info: - body() + body(None) assert str(info.value._message) == ( "hidden date in [sheet_name='page-desc', 2, 2 @ glitter] " + 'value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"' @@ -314,6 +314,7 @@ def body() -> None: else: # add the exception + sheet_configs["page-desc"]["shift_ignore"] = { "glitter": [ "can't recall the date but on 12/11/2001 they had an itchy tummy", @@ -324,7 +325,7 @@ def body() -> None: } # run the shift - body() + body(source_file) workbook = load_workbook(output_path) page = workbook["page-desc"] @@ -349,3 +350,48 @@ def body() -> None: ] assert expected == obtained + + +def test_move_to_file(tmp_path: Path) -> None: + + source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" + output_path = tmp_path / "target.xlsx" + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_out = tmp_path / "linking_table_out.csv" + + sheet_configs = { + "page-desc": { + "patient_id_col": "pid", + "date_columns": [], + "text_columns": ["foo", "glitter"], + "header_row": 0, + "skip_rows_after_header": [], + }, + "page-data": "skip", + } + + # add the old config we want to remove + sheet_configs["page-desc"]["shift_ignore"] = { + "glitter": [ + "can't recall the date but on 12/11/2001 they had an itchy tummy", + ], + "foo": [ + "agreed on 21st jul 2017", + ], + } + + with pytest.raises(RuntimeError) as error: + shift_excel_dates_inplace( + input_file=str(source_file), + output_file=str(output_path), + patient_sheet="page-desc", + sheet_configs=sheet_configs, + min_shift_days=-20, + max_shift_days=-1, + seed=14333, + linking_table_path=str(linking_table_old), + linking_table_output=str(linking_table_out), + ) + pytest.fail("execution should throw an exception before now") + + assert str(error.value) == "move shift_ignore from sheet_configs to a file" From 9441cf4fb93abd5b4d4bf1336839efd4e653917e Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 11:54:57 +0100 Subject: [PATCH 37/82] don't count tests/ when checking coverage --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 47ade72..ca4dea5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,9 @@ select = ["E", "F", "ANN", "B", "UP", "I", "SIM", "C4"] [tool.coverage.run] branch = true +omit = [ + "tests/*", +] [tool.pytest.ini_options] addopts = "-ra" From be338b179f6e6c4f91f3038af1e39a4c1507f2f3 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 12:08:15 +0100 Subject: [PATCH 38/82] debug logging for the ignores --- nuh_helper/date_shift/__init__.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index fa2a5e3..1341c47 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -533,13 +533,14 @@ def shift_excel_dates_inplace( if not shift_ignore_yaml.is_file(): shift_ignore: dict[str, dict[str, set[str]]] = {} + logger.warning(f"didn't find shift_ignore file {shift_ignore_yaml=}") else: with open(shift_ignore_yaml) as file: data = yaml.safe_load(file) shift_ignore: dict[str, dict[str, list[str]]] = { page.strip(): { - col.strip(): {ignored.strip for ignored in data[page][col]} - + {"", None} + col.strip(): [ignored.strip() for ignored in data[page][col]] + + ["", None] for col in data[page] } for page in data @@ -790,6 +791,16 @@ def shift_excel_dates_inplace( if col_name in text_columns: # check for dates in non-date columns for found in datefinder.find_dates(str(cell_value)): + logger.debug( + f"found hidden date {sheet_name=}/{col_name=}:{cell_value}" + ) + ignored = shift_ignore[sheet_name][col_name] + logger.debug( + f"shift_ignore[{sheet_name=}][{col_name=}]= {len(ignored)}" + ) + for ig in ignored: + logger.debug(f"- {ig}") + raise HiddenDate( sheet_name, row_idx, From 4215f13883551792a34ca2e34f2511d442185f06 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 12:19:09 +0100 Subject: [PATCH 39/82] tests all pass with the new yaml file approach --- tests/data/shift_ignore/shift-ignore.yaml | 8 ++++++++ tests/test_shift_ignore.py | 21 +++++---------------- 2 files changed, 13 insertions(+), 16 deletions(-) create mode 100644 tests/data/shift_ignore/shift-ignore.yaml diff --git a/tests/data/shift_ignore/shift-ignore.yaml b/tests/data/shift_ignore/shift-ignore.yaml new file mode 100644 index 0000000..94f26d7 --- /dev/null +++ b/tests/data/shift_ignore/shift-ignore.yaml @@ -0,0 +1,8 @@ +page-desc: + glitter: + - can't recall the date but on 12/11/2001 they had an itchy tummy + foo: + - agreed on 21st jul 2017 +page-data: + dob: + - mssing diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 32a3908..cc44664 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -206,7 +206,7 @@ def test_ignore_in_date_columns(shift_ignore: bool, tmp_path: Path) -> None: }, } - def body() -> None: + def body(shift_ignore_yaml: None | Path) -> None: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -217,11 +217,12 @@ def body() -> None: seed=14333, linking_table_path=str(linking_table_old), linking_table_output=str(linking_table_out), + shift_ignore_yaml=shift_ignore_yaml, ) if not shift_ignore: with pytest.raises(ShiftFoundNonDate) as info: - body() + body(None) assert info.value._page == "page-data" assert info.value._row == 5 assert info.value._col == 1 @@ -229,8 +230,7 @@ def body() -> None: assert info.value._value == "mssing" return else: - sheet_configs["page-data"]["shift_ignore"] = {"dob": ["mssing"]} - body() + body(source_file.parent / "shift-ignore.yaml") workbook = load_workbook(output_path) @@ -313,19 +313,8 @@ def body(shift_ignore_yaml: None | Path) -> None: return else: - # add the exception - - sheet_configs["page-desc"]["shift_ignore"] = { - "glitter": [ - "can't recall the date but on 12/11/2001 they had an itchy tummy", - ], - "foo": [ - "agreed on 21st jul 2017", - ], - } - # run the shift - body(source_file) + body(source_file.parent / "shift-ignore.yaml") workbook = load_workbook(output_path) page = workbook["page-desc"] From de2be87c4a2ad4db193b81bf7e877bffbdf8e3ae Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 5 Oct 2026 16:25:30 +0100 Subject: [PATCH 40/82] just-enough to see if it helps --- nuh_helper/date_shift/__init__.py | 66 +++++++++++++------------------ uv.lock | 2 +- 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 49e7dd0..735a5ca 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -16,7 +16,7 @@ import datefinder import pandas as pd import yaml -from book_page import Book, Page, book_copy +from book_page import Book, Page, book_open from nuh_helper.date_shift import _excel, _parse, mappings @@ -523,7 +523,7 @@ def shift_excel_dates_inplace( seed, ) - book: Book = book_copy(input_file, output_file) + book: Book = book_open(input_file) logger.info(f"Book has {type(book)=}") # TODO; it'd be cool to "normalize" the linking_table_path/linking_table_output here @@ -572,6 +572,8 @@ def shift_excel_dates_inplace( ) logger.info(f"loaded deltas {len(shift_deltas)=}") + raise Exception("?? update teh rest") + # check for sheets we didn't have an explanation for for page in book: if page.name not in sheet_configs: @@ -850,49 +852,37 @@ def patient_shift_deltas( max_shift_days: int, ) -> dict[str, pd.Timedelta]: - logger.info("patient_shift_deltas() ;preparing per-patient shifts") + logger.info("patient_shift_deltas() ; preparing per-patient shifts") + # find the patient id index patient_id_col: None | int = None - for col in range(page.columns): - if page[patient_header_row, col].value == patient_id_col_name: - if patient_id_col is not None: - raise RuntimeError(f"multiple columns with {patient_id_col_name=}") - else: - patient_id_col = col - if patient_id_col is None: - raise PatientColumnMissing(page.name, patient_id_col_name) - logger.info(f"patient_shift_deltas() ; {patient_id_col=}") - - # find the/a list of all ids patient_ids = [] start = time.time() - first_row = ( - max( - [patient_header_row] - + (patient_skip_rows if patient_skip_rows is not None else []) - ) - + 1 - ) - logger.info(f"patient_shift_deltas() ; {first_row=}") - - for row in range( - first_row, - page.rows, - ): - patient_id = page[row, patient_id_col].value - if not patient_id: - continue - patient_id = _parse._normalize_patient_id(patient_id.strip()) - if patient_id in patient_ids: + + for row, data in page.stream_rows(): + if row < patient_header_row or row in patient_skip_rows: continue - patient_ids.append(patient_id) - if (len(patient_ids) % 40) == 0: - seconds = time.time() - start - logger.info( - f"patient_shift_deltas() ; so far {len(patient_ids)=} in {seconds=}" - ) + elif row == patient_header_row: + patient_id_col = data.index(patient_id_col_name) + if patient_id_col is None: + raise PatientColumnMissing(page.name, patient_id_col_name) + else: + patient_id = data[patient_id_col] + if not patient_id: + continue + patient_id = _parse._normalize_patient_id(patient_id.strip()) + if patient_id in patient_ids: + continue + patient_ids.append(patient_id) + if (len(patient_ids) % 40) == 0: + seconds = time.time() - start + logger.info( + f"patient_shift_deltas() ; so far {len(patient_ids)=} in {seconds=}" + ) logger.info(f"patient_shift_deltas() ; Found {len(patient_ids)=} in {page.name=}") + raise Exception("TODO ; update the rest of this") + # load the old ids (should we remove IDs that are "gone"?) if isinstance(src_linking_table, str): src_linking_table = Path(src_linking_table) diff --git a/uv.lock b/uv.lock index 0ff3385..295d369 100644 --- a/uv.lock +++ b/uv.lock @@ -17,7 +17,7 @@ exclude-newer-span = "P7D" [[package]] name = "book-page" version = "0.1.0" -source = { git = "https://github.com/peterlavalle/book_page.git#3243604f4d7d3dcae9bf39852aa9b5787dce3052" } +source = { git = "https://github.com/peterlavalle/book_page.git#8bd16f52fb6d5a6fbb19e8a45c9a6a66e3878585" } dependencies = [ { name = "openpyxl" }, ] From 1451435ee4d3953523f28ddc3679075d963fc925 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 8 Oct 2026 14:46:16 +0100 Subject: [PATCH 41/82] test/uv updates --- pyproject.toml | 2 +- tests/data/shift_ignore/page-data.csv | 8 +++ tests/data/shift_ignore/page-desc.csv | 7 ++ tests/test_shift_ignore.py | 100 +++++++++++++++++--------- uv.lock | 2 +- 5 files changed, 83 insertions(+), 36 deletions(-) create mode 100644 tests/data/shift_ignore/page-data.csv create mode 100644 tests/data/shift_ignore/page-desc.csv diff --git a/pyproject.toml b/pyproject.toml index da9350a..63dc9a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "pandas>=3.0.5", "openpyxl>=3.1.0", "datefinder>=1.0.0", - "book_page @ git+https://github.com/peterlavalle/book_page.git@main", + "book-page", ] [tool.hatch.metadata] allow-direct-references = true diff --git a/tests/data/shift_ignore/page-data.csv b/tests/data/shift_ignore/page-data.csv new file mode 100644 index 0000000..43ae253 --- /dev/null +++ b/tests/data/shift_ignore/page-data.csv @@ -0,0 +1,8 @@ +personal id,birthday,,pizza topping +pid,dob,,top +nuh71,12/18/01,,cheese +nuh06,9/28/93,,unknown +nuh23,,,mushrooms +nuh67,mssing,,this can't be a date - sorry 2016 +nuh27,12/7/99,,idk - this can't be a date anymore +nuh65, mssing ,,this one has spaces diff --git a/tests/data/shift_ignore/page-desc.csv b/tests/data/shift_ignore/page-desc.csv new file mode 100644 index 0000000..d4c8e79 --- /dev/null +++ b/tests/data/shift_ignore/page-desc.csv @@ -0,0 +1,7 @@ +foo,pid,glitter +bar,nuh71,road +shoe,nuh06,can't recall the date but on 12/11/2001 they had an itchy tummy +agreed on 21st jul 2017,nuh23,grip +farm,nuh67,grim +cake,nuh27,cheese +glip,nuh65,jkl diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index cc44664..d0489e5 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -6,12 +6,13 @@ from nuh_helper import shift_excel_dates_inplace from nuh_helper.date_shift import HiddenDate, PatientColumnMissing, ShiftFoundNonDate +test_data = Path(__file__).parent / "data/shift_ignore" -def test_shift_exceptions(tmp_path: Path) -> None: + +@pytest.mark.parametrize("use_csv", [False, True]) +def test_shift_exceptions(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" - output_path = tmp_path / "target.xlsx" linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" @@ -26,10 +27,13 @@ def test_shift_exceptions(tmp_path: Path) -> None: "shift_exceptions": {"dob": ["mssing"]}, }, } + + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + with pytest.raises(RuntimeError) as info: shift_excel_dates_inplace( - input_file=str(source_file), - output_file=str(output_path), + input_file=source_file, + output_file=output_path, patient_sheet="page-data", sheet_configs=sheet_configs, min_shift_days=-20, @@ -43,11 +47,10 @@ def test_shift_exceptions(tmp_path: Path) -> None: ) -def test_ptid_in_date_columns(tmp_path: Path) -> None: +@pytest.mark.parametrize("use_csv", [False, True]) +def test_ptid_in_date_columns(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" - output_path = tmp_path / "target.xlsx" linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" @@ -62,9 +65,12 @@ def test_ptid_in_date_columns(tmp_path: Path) -> None: "shift_ignore": {"dob": ["mssing"]}, }, } + + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + with pytest.raises(ValueError) as info: shift_excel_dates_inplace( - input_file=str(source_file), + input_file=source_file, output_file=str(output_path), patient_sheet="page-data", sheet_configs=sheet_configs, @@ -79,11 +85,10 @@ def test_ptid_in_date_columns(tmp_path: Path) -> None: ) -def test_ptid_in_text_columns(tmp_path: Path) -> None: +@pytest.mark.parametrize("use_csv", [False, True]) +def test_ptid_in_text_columns(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" - output_path = tmp_path / "target.xlsx" linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" @@ -98,9 +103,12 @@ def test_ptid_in_text_columns(tmp_path: Path) -> None: "shift_ignore": {"dob": ["mssing"]}, }, } + + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + with pytest.raises(ValueError) as info: shift_excel_dates_inplace( - input_file=str(source_file), + input_file=source_file, output_file=str(output_path), patient_sheet="page-data", sheet_configs=sheet_configs, @@ -115,11 +123,10 @@ def test_ptid_in_text_columns(tmp_path: Path) -> None: ) -def test_columns_overlap(tmp_path: Path) -> None: +@pytest.mark.parametrize("use_csv", [False, True]) +def test_columns_overlap(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" - output_path = tmp_path / "target.xlsx" linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" @@ -134,9 +141,12 @@ def test_columns_overlap(tmp_path: Path) -> None: "shift_ignore": {"dob": ["mssing"]}, }, } + + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + with pytest.raises(ValueError) as info: shift_excel_dates_inplace( - input_file=str(source_file), + input_file=source_file, output_file=str(output_path), patient_sheet="page-data", sheet_configs=sheet_configs, @@ -152,11 +162,10 @@ def test_columns_overlap(tmp_path: Path) -> None: ) -def test_ptid_column_missing(tmp_path: Path) -> None: +@pytest.mark.parametrize("use_csv", [False, True]) +def test_ptid_column_missing(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" - output_path = tmp_path / "target.xlsx" linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" @@ -171,9 +180,12 @@ def test_ptid_column_missing(tmp_path: Path) -> None: "shift_ignore": {"dob": ["mssing"]}, }, } + + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + with pytest.raises(PatientColumnMissing) as info: shift_excel_dates_inplace( - input_file=str(source_file), + input_file=source_file, output_file=str(output_path), patient_sheet="page-data", sheet_configs=sheet_configs, @@ -187,11 +199,12 @@ def test_ptid_column_missing(tmp_path: Path) -> None: assert info.value._column_name == "ptid" +@pytest.mark.parametrize("use_csv", [False, True]) @pytest.mark.parametrize("shift_ignore", [True, False]) -def test_ignore_in_date_columns(shift_ignore: bool, tmp_path: Path) -> None: +def test_ignore_in_date_columns( + use_csv: bool, shift_ignore: bool, tmp_path: Path +) -> None: - source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" - output_path = tmp_path / "target.xlsx" linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" @@ -206,9 +219,11 @@ def test_ignore_in_date_columns(shift_ignore: bool, tmp_path: Path) -> None: }, } + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + def body(shift_ignore_yaml: None | Path) -> None: shift_excel_dates_inplace( - input_file=str(source_file), + input_file=source_file, output_file=str(output_path), patient_sheet="page-data", sheet_configs=sheet_configs, @@ -230,7 +245,7 @@ def body(shift_ignore_yaml: None | Path) -> None: assert info.value._value == "mssing" return else: - body(source_file.parent / "shift-ignore.yaml") + body(test_data / "shift-ignore.yaml") workbook = load_workbook(output_path) @@ -269,11 +284,12 @@ def body(shift_ignore_yaml: None | Path) -> None: assert str(worksheet.cell(7, 2).value) == "1999-11-30 00:00:00" +@pytest.mark.parametrize("use_csv", [False, True]) @pytest.mark.parametrize("shift_ignore", [True, False]) -def test_ignore_in_text_columns(shift_ignore: bool, tmp_path: Path) -> None: +def test_ignore_in_text_columns( + use_csv: bool, shift_ignore: bool, tmp_path: Path +) -> None: - source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" - output_path = tmp_path / "target.xlsx" linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" @@ -288,9 +304,11 @@ def test_ignore_in_text_columns(shift_ignore: bool, tmp_path: Path) -> None: "page-data": "skip", } + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + def body(shift_ignore_yaml: None | Path) -> None: shift_excel_dates_inplace( - input_file=str(source_file), + input_file=source_file, output_file=str(output_path), patient_sheet="page-desc", sheet_configs=sheet_configs, @@ -341,10 +359,9 @@ def body(shift_ignore_yaml: None | Path) -> None: assert expected == obtained -def test_move_to_file(tmp_path: Path) -> None: +@pytest.mark.parametrize("use_csv", [False, True]) +def test_move_to_file(use_csv: bool, tmp_path: Path) -> None: - source_file = Path(__file__).parent / "data/shift_ignore/workbook.xlsx" - output_path = tmp_path / "target.xlsx" linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" @@ -359,6 +376,8 @@ def test_move_to_file(tmp_path: Path) -> None: "page-data": "skip", } + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + # add the old config we want to remove sheet_configs["page-desc"]["shift_ignore"] = { "glitter": [ @@ -371,7 +390,7 @@ def test_move_to_file(tmp_path: Path) -> None: with pytest.raises(RuntimeError) as error: shift_excel_dates_inplace( - input_file=str(source_file), + input_file=source_file, output_file=str(output_path), patient_sheet="page-desc", sheet_configs=sheet_configs, @@ -384,3 +403,16 @@ def test_move_to_file(tmp_path: Path) -> None: pytest.fail("execution should throw an exception before now") assert str(error.value) == "move shift_ignore from sheet_configs to a file" + + +def src_and_out_names( + use_csv: bool, tmp_path: Path, sheet_configs: dict[str, any] +) -> tuple[Path | list[Path], Path]: + # change the source file to point to the csv files + if use_csv: + source_file = [test_data / (name + ".csv") for name in sheet_configs] + output_path = tmp_path + else: + source_file = test_data / "workbook.xlsx" + output_path = tmp_path / "target.xlsx" + return source_file, output_path diff --git a/uv.lock b/uv.lock index 0ff3385..29ff61e 100644 --- a/uv.lock +++ b/uv.lock @@ -17,7 +17,7 @@ exclude-newer-span = "P7D" [[package]] name = "book-page" version = "0.1.0" -source = { git = "https://github.com/peterlavalle/book_page.git#3243604f4d7d3dcae9bf39852aa9b5787dce3052" } +source = { git = "https://github.com/peterlavalle/book_page.git#43abe840796c047d649be1d94de5960011d58e8e" } dependencies = [ { name = "openpyxl" }, ] From 77b54762695cc9580b2d265488b962000dc56b32 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 8 Oct 2026 14:48:29 +0100 Subject: [PATCH 42/82] pointed to the live copy --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 63dc9a5..da9350a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "pandas>=3.0.5", "openpyxl>=3.1.0", "datefinder>=1.0.0", - "book-page", + "book_page @ git+https://github.com/peterlavalle/book_page.git@main", ] [tool.hatch.metadata] allow-direct-references = true From cdc970dd7b0b7d568780ba3fce678381ed0d8d1b Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 8 Oct 2026 16:07:19 +0100 Subject: [PATCH 43/82] WIP 93/111 tests pass --- nuh_helper/date_shift/__init__.py | 239 +++++++++++++++--------------- uv.lock | 2 +- 2 files changed, 123 insertions(+), 118 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 735a5ca..89bca20 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -572,134 +572,138 @@ def shift_excel_dates_inplace( ) logger.info(f"loaded deltas {len(shift_deltas)=}") - raise Exception("?? update teh rest") + # check for missing sheets + for sheet_name in sheet_configs: + if sheet_name not in book: + raise PageMissing(sheet_name) - # check for sheets we didn't have an explanation for - for page in book: - if page.name not in sheet_configs: - raise ExtraPage(page.name) + last_name = None + row_idx = None + for sheet_name, row, cells in book.stream_copy(output_file): + assert isinstance(sheet_name, str) + if row == 0: + if last_name and row_idx: + logger.info(f"Shifting {last_name=} processed {row_idx} rows") + last_name = sheet_name - # start looping through each sheet in the configuration - for sheet_name, sheet_config in sheet_configs.items(): - ### - # load the workbook sheet and configuration - ## + # check that it's in the configs + if sheet_name not in sheet_configs: + raise ExtraPage(sheet_name) - if sheet_name not in book: - raise PageMissing(sheet_name) + sheet_config: dict | str = sheet_configs[sheet_name] + + # do nothing to the skipped ones + if sheet_config == "skip": + continue + + # grab the configuration + patient_id_col: str = cast(str, sheet_config["patient_id_col"]).strip() + date_columns: list[str] = [ + col.strip() for col in sheet_config["date_columns"] + ] + text_columns: list[str] = [ + col.strip() for col in sheet_config["text_columns"] + ] + + header_row: int = cast(int, sheet_config.get("header_row", 0)) + skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) + + ### + # do some checks of the configuration + ## + + # this is an old field i want to be careful about skipping + if "shift_exceptions" in sheet_config: + raise RuntimeError( + f"shift_exceptions was removed, update {sheet_name=}" + ) + + # check patient_id_col isn't re-used as other column types + if patient_id_col in date_columns: + raise ValueError( + f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" + ) + if patient_id_col in text_columns: + raise ValueError( + f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" + ) + + # check that column names don't appear in both + col_names = [col for col in date_columns if col in text_columns] + if col_names: + raise ValueError( + sheet_name + + " has the some columns in both date and text " + + col_names + ) + + col_names = None + patient_id_idx = None - if isinstance(sheet_config, str) and sheet_config == "skip": - # it's a skipped sheet - logger.info(f"Skipping {sheet_name=}") + if (row < header_row) or (row in skip_rows): + # TODO; there's a hole here - if header row is 2 and skip_rows is 4,5,6 + # ... we'll try to run row 3 without being ready continue - logger.info(f"Shifting {sheet_name=}") - # shift the page - _shift_book_page(shift_ignore, shift_deltas, sheet_name, sheet_config, book) + if row == header_row: + # get the real column names now + col_names = cells - logger.info("Output written to '%s'", output_file) + # find patient_id_idx and normalize the column name list + for col_idx in range(len(col_names)): + col_name = col_names[col_idx] + + # if the name is None ... leave it as that (it's fine) + if col_name is None: + continue + # strip the name + col_name = col_name.strip() -def _shift_book_page( - shift_ignore: dict[str, dict[str, set[str]]], - shift_deltas: dict[str, pd.Timedelta], - sheet_name: str, - sheet_config: dict[str, any], - book: Book, -) -> None: - """shifts a single page. the nesting/breakout was done to minimize git changes""" - with book[sheet_name] as page: - patient_id_col: str = cast(str, sheet_config["patient_id_col"]).strip() - date_columns: list[str] = [col.strip() for col in sheet_config["date_columns"]] - text_columns: list[str] = [col.strip() for col in sheet_config["text_columns"]] - - header_row: int = cast(int, sheet_config.get("header_row", 0)) - skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) - - ### - # do some checks of the configuration - ## - - # this is an old field i want to be careful about skipping - if "shift_exceptions" in sheet_config: - raise RuntimeError(f"shift_exceptions was removed, update {sheet_name=}") - - # check patient_id_col isn't re-used as other column types - if patient_id_col in date_columns: - raise ValueError( - f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" - ) - if patient_id_col in text_columns: - raise ValueError( - f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" - ) + # if the name is now blank; store None as the column name + if col_name == "": + col_names[col_idx] = None + continue - # check that column names don't appear in both - col_names = [col for col in date_columns if col in text_columns] - if col_names: - raise ValueError( - f"{sheet_name=} has the some columns in both date and text {col_names=}" - ) - # get teh real column names now - col_names = [page[header_row, c].value for c in range(page.columns)] + # update it to just be the stripped version + col_names[col_idx] = col_name - # find patient_id_idx and normalize the column name list - patient_id_idx = None - for col_idx in range(len(col_names)): - col_name = col_names[col_idx] + # check the config to see if we know what to do with this column + if col_name not in ([patient_id_col] + date_columns + text_columns): + raise ExtraColumn(sheet_name, col_name) - # if the name is None ... leave it as that (it's fine) - if col_name is None: - continue + # select the patient id + if col_name == patient_id_col: + patient_id_idx = col_idx - # strip the name - col_name = col_name.strip() + # check to be sure we found the patient_id_col/patient_id_idx + if patient_id_idx is None: + raise PatientColumnMissing(sheet_name, patient_id_col) - # if the name is now blank; store None as the column name - if col_name == "": - col_names[col_idx] = None - continue + missing = [col for col in text_columns if col not in col_names] + if missing: + raise TextColumnsMissing(sheet_name, missing) + missing = [col for col in date_columns if col not in col_names] + if missing: + raise DateColumnsMissing(sheet_name, missing) - # update it to just be the stripped version - col_names[col_idx] = col_name - - # check the config to see if we know what to do with this column - if col_name not in ([patient_id_col] + date_columns + text_columns): - raise ExtraColumn(sheet_name, col_name) - - # - if col_name == patient_id_col: - patient_id_idx = col_idx - - # check to be sure we found the patient_id_col/patient_id_idx - if patient_id_idx is None: - raise PatientColumnMissing(sheet_name, patient_id_col) - - missing = [col for col in text_columns if col not in col_names] - if missing: - raise TextColumnsMissing(sheet_name, missing) - missing = [col for col in date_columns if col not in col_names] - if missing: - raise DateColumnsMissing(sheet_name, missing) - - ### - # process each row of the workbook - ## - for row_idx in range(page.rows): - # skip all rows that happen before the header row - if row_idx <= header_row: - continue + continue - # skip any skip rows - if row_idx in skip_rows: - continue + if col_names is None: + raise RuntimeError("does the sheet skip all the skip rows?") + else: + row_idx = row + + ### + # process each row of the workbook + ## # write a log message for the user every 40 rows if (row_idx % 40) == 0: logger.info(f"Shifting {sheet_name=} up to row {row_idx}") # get the patient id for this row - cell_value = page[row_idx, patient_id_idx].value + cell_value = cells[patient_id_idx] if cell_value: if not isinstance(cell_value, str): @@ -723,6 +727,7 @@ def _shift_book_page( pid = _parse._normalize_patient_id(cell_value) if pid is None: # if the pid is None; the rest of the row should be None as well + raise RuntimeError("??? check that the empty row is empty") non_blank = [ cell for cell in [ @@ -754,23 +759,22 @@ def _shift_book_page( # now scan each column # ... even when page has no date_columns; still check for hidden dates - for col_idx in range(page.columns): + for col_idx in range(len(cells)): # skip the patient id column (it was already checked anyway) if col_idx == patient_id_idx: continue # get the cell value. replace it with None if it's just whitespace - cell = page[row_idx, col_idx] - cell_value = cell.value + cell_value = cells[col_idx] if isinstance(cell_value, str): cell_value = cell_value.strip() if not cell_value: cell_value = None - cell.value = None # so if the cell is empty; skip the rest of the checks for this cell # ... we go tot he next column in the row if cell_value is None: + cells[col_idx] = None continue # if there's no column name; this column of the row should be None @@ -782,10 +786,13 @@ def _shift_book_page( # skip values in shift_ignore if "shift_ignore" in sheet_config: raise RuntimeError("move shift_ignore from sheet_configs to a file") + if sheet_name not in shift_ignore: shift_ignore[sheet_name] = {} + if col_name not in shift_ignore[sheet_name]: shift_ignore[sheet_name][col_name] = {None, ""} + if cell_value in shift_ignore[sheet_name][col_name]: continue @@ -811,7 +818,7 @@ def _shift_book_page( cell_value, found, ) - cell.value = cell_value + cells[col_idx] = cell_value continue # we've already checked this (sort of) @@ -832,12 +839,12 @@ def _shift_book_page( # controls wether the data is displayed with the 00:00:00 in Excel # ... it might be nice to just drop the time if it's 00:00:00 if isinstance(cell_value, datetime): - cell.value = cast(Any, shifted.to_pydatetime()) + cells[col_idx] = cast(Any, shifted.to_pydatetime()) else: - cell.value = cast(Any, shifted.to_pydatetime().date()) - cell.number_format = "yyyy-mm-dd" + cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) + print('redo cell.number_format = "yyyy-mm-dd"') - logger.info(f"Shifting {sheet_name=} processed {row_idx} rows") + logger.info("Output written to '%s'", output_file) def patient_shift_deltas( @@ -881,8 +888,6 @@ def patient_shift_deltas( ) logger.info(f"patient_shift_deltas() ; Found {len(patient_ids)=} in {page.name=}") - raise Exception("TODO ; update the rest of this") - # load the old ids (should we remove IDs that are "gone"?) if isinstance(src_linking_table, str): src_linking_table = Path(src_linking_table) diff --git a/uv.lock b/uv.lock index 29ff61e..a157526 100644 --- a/uv.lock +++ b/uv.lock @@ -17,7 +17,7 @@ exclude-newer-span = "P7D" [[package]] name = "book-page" version = "0.1.0" -source = { git = "https://github.com/peterlavalle/book_page.git#43abe840796c047d649be1d94de5960011d58e8e" } +source = { git = "https://github.com/peterlavalle/book_page.git#c8246740221cc72cb195f815dd3e97a41e320387" } dependencies = [ { name = "openpyxl" }, ] From 817acb3d0222062607be54cf36660f58205956f9 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 9 Oct 2026 16:09:30 +0100 Subject: [PATCH 44/82] implements "row is blank" test and skipping --- nuh_helper/date_shift/__init__.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 89bca20..768c1bb 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -579,9 +579,13 @@ def shift_excel_dates_inplace( last_name = None row_idx = None + skipping_sheet_name = "" for sheet_name, row, cells in book.stream_copy(output_file): assert isinstance(sheet_name, str) - if row == 0: + if row != 0: + if sheet_name != last_name: + raise RuntimeError(f"didn't see row0 from {sheet_name=}") + else: if last_name and row_idx: logger.info(f"Shifting {last_name=} processed {row_idx} rows") last_name = sheet_name @@ -594,6 +598,7 @@ def shift_excel_dates_inplace( # do nothing to the skipped ones if sheet_config == "skip": + skipping_sheet_name = sheet_name continue # grab the configuration @@ -640,11 +645,17 @@ def shift_excel_dates_inplace( col_names = None patient_id_idx = None + # do nothing to the skipped ones + if sheet_config == "skip": + # "safety" check so that we only skip rows with the correct name + if skipping_sheet_name != sheet_name: + raise RuntimeError("this shouldn't happen") + continue + if (row < header_row) or (row in skip_rows): # TODO; there's a hole here - if header row is 2 and skip_rows is 4,5,6 # ... we'll try to run row 3 without being ready continue - if row == header_row: # get the real column names now col_names = cells @@ -727,20 +738,11 @@ def shift_excel_dates_inplace( pid = _parse._normalize_patient_id(cell_value) if pid is None: # if the pid is None; the rest of the row should be None as well - raise RuntimeError("??? check that the empty row is empty") non_blank = [ cell - for cell in [ - # get all cells - ( - col_idx, - col_names[col_idx], - page[row_idx, col_idx].value, - ) - for col_idx in range(page.columns) - ] + for cell in cells # keep the cells that aren't blank - if cell[2] is not None and cell[2].strip() != "" + if cell is not None and str(cell).strip() != "" ] if non_blank: raise ValueError( From e15626d5d96fc12d9303bee5863e0829c541a4af Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 9 Oct 2026 17:23:55 +0100 Subject: [PATCH 45/82] "cheats" and switches the ambiguous date to iso8061 --- nuh_helper/date_shift/__init__.py | 1 + tests/data/shift_ignore/page-data.csv | 4 +- tests/test_shift_ignore.py | 97 ++++++++++++++------------- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 768c1bb..6e98b1b 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -656,6 +656,7 @@ def shift_excel_dates_inplace( # TODO; there's a hole here - if header row is 2 and skip_rows is 4,5,6 # ... we'll try to run row 3 without being ready continue + if row == header_row: # get the real column names now col_names = cells diff --git a/tests/data/shift_ignore/page-data.csv b/tests/data/shift_ignore/page-data.csv index 43ae253..3dc5175 100644 --- a/tests/data/shift_ignore/page-data.csv +++ b/tests/data/shift_ignore/page-data.csv @@ -4,5 +4,5 @@ nuh71,12/18/01,,cheese nuh06,9/28/93,,unknown nuh23,,,mushrooms nuh67,mssing,,this can't be a date - sorry 2016 -nuh27,12/7/99,,idk - this can't be a date anymore -nuh65, mssing ,,this one has spaces +nuh27,1999-12-07,,idk - this can't be a date anymore +nuh65, mssing ,,this one has spaces diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index d0489e5..5ff6fee 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -1,7 +1,7 @@ from pathlib import Path +import book_page import pytest -from openpyxl import load_workbook from nuh_helper import shift_excel_dates_inplace from nuh_helper.date_shift import HiddenDate, PatientColumnMissing, ShiftFoundNonDate @@ -247,41 +247,23 @@ def body(shift_ignore_yaml: None | Path) -> None: else: body(test_data / "shift-ignore.yaml") - workbook = load_workbook(output_path) - - worksheet = workbook.worksheets[1] - - # first column - assert worksheet.cell(1, 1).value == "personal id" - assert worksheet.cell(2, 1).value == "pid" - assert worksheet.cell(3, 1).value == "nuh71" - assert worksheet.cell(4, 1).value == "nuh06" - assert worksheet.cell(5, 1).value == "nuh23" - assert worksheet.cell(6, 1).value == "nuh67" - assert worksheet.cell(7, 1).value == "nuh27" - - # check column 3 is all none - assert not [ - val for val in [worksheet.cell(r + 1, 3).value for r in range(7)] if val - ] - - # last column - assert worksheet.cell(1, 4).value == "pizza topping" - assert worksheet.cell(2, 4).value == "top" - assert worksheet.cell(3, 4).value == "cheese" - assert worksheet.cell(4, 4).value == "unknown" - assert worksheet.cell(5, 4).value == "mushrooms" - assert worksheet.cell(6, 4).value == "this can't be a date - sorry 2016" - assert worksheet.cell(7, 4).value == "idk - this can't be a date anymore" - - # the important column to check - the dates - assert worksheet.cell(1, 2).value == "birthday" - assert worksheet.cell(2, 2).value == "dob" - assert str(worksheet.cell(3, 2).value) == "2001-12-17 00:00:00" - assert str(worksheet.cell(4, 2).value) == "1993-09-20 00:00:00" - assert worksheet.cell(5, 2).value is None - assert str(worksheet.cell(6, 2).value) == "mssing" # change that's under test - assert str(worksheet.cell(7, 2).value) == "1999-11-30 00:00:00" + obtained = load_workbook(output_path)["page-data"] + expected = [ + ["personal id", "birthday", None, "pizza topping"], + ["pid", "dob", None, "top"], + ["nuh71", "2001-12-17", None, "cheese"], + ["nuh06", "1993-09-20", None, "unknown"], + ["nuh23", None, None, "mushrooms"], + ["nuh67", "mssing", None, "this can't be a date - sorry 2016"], + [ + "nuh27", + "1999-11-30", + None, + "idk - this can't be a date anymore", + ], + ["nuh65", "mssing", None, "this one has spaces"], + ] + assert obtained == expected @pytest.mark.parametrize("use_csv", [False, True]) @@ -332,15 +314,9 @@ def body(shift_ignore_yaml: None | Path) -> None: else: # run the shift - body(source_file.parent / "shift-ignore.yaml") - - workbook = load_workbook(output_path) - page = workbook["page-desc"] + body(test_data / "shift-ignore.yaml") - obtained = [ - [str(page.cell(row + 1, col + 1).value) for col in range(page.max_column)] - for row in range(page.max_row) - ] + obtained = load_workbook(output_path)["page-desc"] expected = [ ["foo", "pid", "glitter"], @@ -359,6 +335,36 @@ def body(shift_ignore_yaml: None | Path) -> None: assert expected == obtained +def load_workbook(path: str | Path) -> dict[str, list[list[str | None]]]: + + book = {} + + for page in book_page.book_open(path): + data = [] + + for row, cells in page.stream_rows(): + assert row == len(data) + for idx, val in enumerate(cells): + if val is None: + continue + val = str(val).strip() + if not val: + cells[idx] = None + continue + + import re + + pattern = r"\d{4}-\d{2}-\d{2} 00:00:00" + if re.fullmatch(pattern, val): + val = val[:10] + cells[idx] = val + data.append(cells) + + book[page.name] = data + + return book + + @pytest.mark.parametrize("use_csv", [False, True]) def test_move_to_file(use_csv: bool, tmp_path: Path) -> None: @@ -411,7 +417,8 @@ def src_and_out_names( # change the source file to point to the csv files if use_csv: source_file = [test_data / (name + ".csv") for name in sheet_configs] - output_path = tmp_path + output_path = tmp_path / "output" + output_path.mkdir() else: source_file = test_data / "workbook.xlsx" output_path = tmp_path / "target.xlsx" From 9266d439402cf9ce41704db551285a92b893222a Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 9 Oct 2026 17:27:36 +0100 Subject: [PATCH 46/82] corrected overlap detection and testing --- nuh_helper/date_shift/__init__.py | 5 ++--- tests/test_shift_ignore.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 6e98b1b..477987b 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -637,9 +637,8 @@ def shift_excel_dates_inplace( col_names = [col for col in date_columns if col in text_columns] if col_names: raise ValueError( - sheet_name - + " has the some columns in both date and text " - + col_names + f"{sheet_name=} has the some columns in both date and text " + + str(col_names) ) col_names = None diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 5ff6fee..d5f51cf 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -157,8 +157,7 @@ def test_columns_overlap(use_csv: bool, tmp_path: Path) -> None: linking_table_output=str(linking_table_out), ) assert str(info.value) == ( - "sheet_name='page-data' has the some columns in both date and text " - + "col_names=['dob']" + "sheet_name='page-data' has the some columns in both date and text ['dob']" ) From 8e044375b42983f78dc49e271f47e212aae034ee Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 9 Oct 2026 17:30:39 +0100 Subject: [PATCH 47/82] fixes missing patient id column detection --- nuh_helper/date_shift/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 477987b..581d8e8 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -872,9 +872,9 @@ def patient_shift_deltas( if row < patient_header_row or row in patient_skip_rows: continue elif row == patient_header_row: - patient_id_col = data.index(patient_id_col_name) - if patient_id_col is None: + if patient_id_col_name not in data: raise PatientColumnMissing(page.name, patient_id_col_name) + patient_id_col = data.index(patient_id_col_name) else: patient_id = data[patient_id_col] if not patient_id: From 1206f2d4c55e9fca456dcbe7c434d22191223d71 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 11:21:32 +0100 Subject: [PATCH 48/82] added comments and checks for "skip holes" --- nuh_helper/date_shift/__init__.py | 89 ++++++++++++++++++++----------- tests/test_shift_ignore.py | 49 +++++++++++++++++ 2 files changed, 108 insertions(+), 30 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 581d8e8..9ed1f08 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -580,9 +580,23 @@ def shift_excel_dates_inplace( last_name = None row_idx = None skipping_sheet_name = "" - for sheet_name, row, cells in book.stream_copy(output_file): + + ## + # this calls the `.stream_copy()` method to copy from one book to another + # WARNING; this works like a callback rather than a loop; + # - each iteration is a callback from copy with a source row + # - each row is passed with a sheet name and a row number + # - modify the values in the row_cells to change what's in the output file(s) + # - no option to add/remove rows is available + # - no option to jump/seek through rows is available + # - ... which is why this acts a bit weird + for sheet_name, row_idx, row_cells in book.stream_copy(output_file): + ##### + # Part-1; check if we've reached a different sheet + ## assert isinstance(sheet_name, str) - if row != 0: + if row_idx != 0: + # SAN/safety check in case something tries to skip showing us rows if sheet_name != last_name: raise RuntimeError(f"didn't see row0 from {sheet_name=}") else: @@ -617,7 +631,20 @@ def shift_excel_dates_inplace( # do some checks of the configuration ## - # this is an old field i want to be careful about skipping + # check for holes in skip_rows (possible issue) + if skip_rows: + missing_skipped = [ + not_skipped + 1 + for not_skipped in range(header_row, max(skip_rows)) + if (not_skipped + 1) not in skip_rows + ] + if missing_skipped: + raise RuntimeError( + f"{sheet_name=} and {missing_skipped=} but" + + f" {header_row=} and {skip_rows=}" + ) + + # check if the config hard-codes patient data to skip if "shift_exceptions" in sheet_config: raise RuntimeError( f"shift_exceptions was removed, update {sheet_name=}" @@ -644,21 +671,28 @@ def shift_excel_dates_inplace( col_names = None patient_id_idx = None - # do nothing to the skipped ones + ##### + # Part-2; skip skipped sheets, and, skip skipped rows + ## + + # do nothing to the skipped sheets if sheet_config == "skip": # "safety" check so that we only skip rows with the correct name if skipping_sheet_name != sheet_name: raise RuntimeError("this shouldn't happen") continue - if (row < header_row) or (row in skip_rows): - # TODO; there's a hole here - if header row is 2 and skip_rows is 4,5,6 - # ... we'll try to run row 3 without being ready + # skip pre-header and skip rows + if (row_idx < header_row) or (row_idx in skip_rows): + # there's an earlier check to find "holes" in the skip-rows continue - if row == header_row: + ##### + # Part-3; handle the header rows + ## + if row_idx == header_row: # get the real column names now - col_names = cells + col_names = row_cells # find patient_id_idx and normalize the column name list for col_idx in range(len(col_names)): @@ -700,47 +734,42 @@ def shift_excel_dates_inplace( continue + ##### + # Part-4; process normal columns + ## if col_names is None: raise RuntimeError("does the sheet skip all the skip rows?") else: - row_idx = row + # code used to be indented like this ### # process each row of the workbook ## - # write a log message for the user every 40 rows + # write a log message every 40 rows (show we're not crashed) if (row_idx % 40) == 0: logger.info(f"Shifting {sheet_name=} up to row {row_idx}") # get the patient id for this row - cell_value = cells[patient_id_idx] - - if cell_value: - if not isinstance(cell_value, str): - raise ValueError( - f"bad val {sheet_name=} {patient_id_col=} ;; {cell_value=}" - ) - - cell_value = cell_value.strip() + pid = row_cells[patient_id_idx] # check for date in the cell_value - for found in datefinder.find_dates(str(cell_value)): + for found in datefinder.find_dates(str(pid)): raise HiddenDate( sheet_name, row_idx, patient_id_idx, patient_id_col, - cell_value, + pid, found, ) - pid = _parse._normalize_patient_id(cell_value) + pid = _parse._normalize_patient_id(pid) if pid is None: # if the pid is None; the rest of the row should be None as well non_blank = [ cell - for cell in cells + for cell in row_cells # keep the cells that aren't blank if cell is not None and str(cell).strip() != "" ] @@ -761,13 +790,13 @@ def shift_excel_dates_inplace( # now scan each column # ... even when page has no date_columns; still check for hidden dates - for col_idx in range(len(cells)): + for col_idx in range(len(row_cells)): # skip the patient id column (it was already checked anyway) if col_idx == patient_id_idx: continue # get the cell value. replace it with None if it's just whitespace - cell_value = cells[col_idx] + cell_value = row_cells[col_idx] if isinstance(cell_value, str): cell_value = cell_value.strip() if not cell_value: @@ -776,7 +805,7 @@ def shift_excel_dates_inplace( # so if the cell is empty; skip the rest of the checks for this cell # ... we go tot he next column in the row if cell_value is None: - cells[col_idx] = None + row_cells[col_idx] = None continue # if there's no column name; this column of the row should be None @@ -820,7 +849,7 @@ def shift_excel_dates_inplace( cell_value, found, ) - cells[col_idx] = cell_value + row_cells[col_idx] = cell_value continue # we've already checked this (sort of) @@ -841,9 +870,9 @@ def shift_excel_dates_inplace( # controls wether the data is displayed with the 00:00:00 in Excel # ... it might be nice to just drop the time if it's 00:00:00 if isinstance(cell_value, datetime): - cells[col_idx] = cast(Any, shifted.to_pydatetime()) + row_cells[col_idx] = cast(Any, shifted.to_pydatetime()) else: - cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) + row_cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) print('redo cell.number_format = "yyyy-mm-dd"') logger.info("Output written to '%s'", output_file) diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index d5f51cf..eacc938 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -410,6 +410,55 @@ def test_move_to_file(use_csv: bool, tmp_path: Path) -> None: assert str(error.value) == "move shift_ignore from sheet_configs to a file" +@pytest.mark.parametrize("use_csv", [False, True]) +def test_skip_hole(use_csv: bool, tmp_path: Path) -> None: + + linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_out = tmp_path / "linking_table_out.csv" + + sheet_configs = { + "page-desc": { + "patient_id_col": "pid", + "date_columns": [], + "text_columns": ["foo", "glitter"], + "header_row": 0, + "skip_rows_after_header": [2], + }, + "page-data": "skip", + } + + source_file, output_path = src_and_out_names(use_csv, tmp_path, sheet_configs) + + # add the old config we want to remove + sheet_configs["page-desc"]["shift_ignore"] = { + "glitter": [ + "can't recall the date but on 12/11/2001 they had an itchy tummy", + ], + "foo": [ + "agreed on 21st jul 2017", + ], + } + + with pytest.raises(RuntimeError) as error: + shift_excel_dates_inplace( + input_file=source_file, + output_file=str(output_path), + patient_sheet="page-desc", + sheet_configs=sheet_configs, + min_shift_days=-20, + max_shift_days=-1, + seed=14333, + linking_table_path=str(linking_table_old), + linking_table_output=str(linking_table_out), + ) + pytest.fail("execution should throw an exception before now") + + assert str(error.value) == ( + "sheet_name='page-desc' and missing_skipped=[1]" + + " but header_row=0 and skip_rows=[2]" + ) + + def src_and_out_names( use_csv: bool, tmp_path: Path, sheet_configs: dict[str, any] ) -> tuple[Path | list[Path], Path]: From 7801176e6f8dea426a76f94d2fd7a6b49151e477 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 12:07:56 +0100 Subject: [PATCH 49/82] corrected the result of a rename --- nuh_helper/date_shift/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index d54e0da..6fe2c8b 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -578,7 +578,7 @@ def shift_excel_dates_inplace( raise PageMissing(sheet_name) last_name = None - row_idx = None + last_row = None skipping_sheet_name = "" ## @@ -599,10 +599,12 @@ def shift_excel_dates_inplace( # SAN/safety check in case something tries to skip showing us rows if sheet_name != last_name: raise RuntimeError(f"didn't see row0 from {sheet_name=}") + last_row = row_idx else: if last_name and row_idx: - logger.info(f"Shifting {last_name=} processed {row_idx} rows") + logger.info(f"Shifting {last_name=} processed {last_row} rows") last_name = sheet_name + last_row = row_idx # check that it's in the configs if sheet_name not in sheet_configs: @@ -876,6 +878,9 @@ def shift_excel_dates_inplace( row_cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) print('redo cell.number_format = "yyyy-mm-dd"') + logger.info( + f"Shifting {last_name=} processed {last_row} rows (and that's the last page)" + ) logger.info("Output written to '%s'", output_file) From bc747431e9615c346652a8c0fd18c06690a62677 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 12:11:17 +0100 Subject: [PATCH 50/82] fix again --- nuh_helper/date_shift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 6fe2c8b..f7890a8 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -601,7 +601,7 @@ def shift_excel_dates_inplace( raise RuntimeError(f"didn't see row0 from {sheet_name=}") last_row = row_idx else: - if last_name and row_idx: + if last_name and last_row: logger.info(f"Shifting {last_name=} processed {last_row} rows") last_name = sheet_name last_row = row_idx From c3baf4ff904a2b460aea461d3baa3007ac58c91b Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 13:01:56 +0100 Subject: [PATCH 51/82] updated many of the unit tests --- nuh_helper/date_shift/__init__.py | 7 +++++-- pyproject.toml | 4 ++-- tests/data/test_date_shift/shift-ignore.yaml | 3 +++ tests/test_date_shift.py | 12 ++++++++---- tests/test_e2e_date_shift.py | 2 +- uv.lock | 4 ++-- 6 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 tests/data/test_date_shift/shift-ignore.yaml diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 5a3908b..3ff1615 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -325,7 +325,10 @@ def shift_excel_dates_inplace( data = yaml.safe_load(file) shift_ignore: dict[str, dict[str, list[str]]] = { page.strip(): { - col.strip(): [ignored.strip() for ignored in data[page][col]] + col.strip(): [ + ignored.strip() if isinstance(ignored, str) else ignored + for ignored in data[page][col] + ] + ["", None] for col in data[page] } @@ -341,7 +344,7 @@ def shift_excel_dates_inplace( linking_table_path, linking_table_output or "shift_mappings.csv", sheet_configs[patient_sheet]["patient_id_col"], - sheet_configs[patient_sheet]["header_row"], + sheet_configs[patient_sheet].get("header_row", 0), sheet_configs[patient_sheet].get("skip_rows", []), seed, min_shift_days, diff --git a/pyproject.toml b/pyproject.toml index da9350a..e8adf03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ dependencies = [ "pandas>=3.0.5", "openpyxl>=3.1.0", "datefinder>=1.0.0", - "book_page @ git+https://github.com/peterlavalle/book_page.git@main", + "book-page", ] [tool.hatch.metadata] allow-direct-references = true @@ -41,7 +41,7 @@ dev = [ exclude-newer = "7 days" [tool.uv.sources] -book-page = { git = "https://github.com/peterlavalle/book_page.git" } +book-page = { git = "https://github.com/peterlavalle/book_page.git", rev = "main" } [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/tests/data/test_date_shift/shift-ignore.yaml b/tests/data/test_date_shift/shift-ignore.yaml new file mode 100644 index 0000000..62ede34 --- /dev/null +++ b/tests/data/test_date_shift/shift-ignore.yaml @@ -0,0 +1,3 @@ +patients: + last_alive: + - "2024-12-31" diff --git a/tests/test_date_shift.py b/tests/test_date_shift.py index 7a20f3d..8d62368 100644 --- a/tests/test_date_shift.py +++ b/tests/test_date_shift.py @@ -18,6 +18,8 @@ ) from nuh_helper.date_shift._parse import _normalize_patient_id, _parse_date_value +test_data = Path(__file__).parent / "data" / __name__ + class TestNormalizePatientId: def test_none_returns_none(self) -> None: @@ -271,8 +273,8 @@ def test_header_row_and_skip_rows_after_header_produce_correct_columns( input_file=str(xlsx), output_file=str(out), patient_sheet="patients", - patient_id_col="patient_id", sheet_configs=sheet_configs, + linking_table_path=tmp_path / "old-linking.csv", linking_table_output=str(tmp_path / "linking.csv"), seed=42, ) @@ -315,8 +317,8 @@ def test_patient_sheet_uses_config_header_when_sheet_in_sheet_configs( input_file=str(xlsx), output_file=str(out), patient_sheet="patients", - patient_id_col="patient_id", sheet_configs=sheet_configs, + linking_table_path=tmp_path / "old-linking.csv", linking_table_output=str(tmp_path / "linking.csv"), seed=42, ) @@ -351,21 +353,23 @@ def test_exception_date_unchanged_other_dates_shifted(self, tmp_path: Path) -> N xlsx = tmp_path / "in.xlsx" self._make_excel(xlsx) out = tmp_path / "out.xlsx" + + # raise Exception(f'{__name__=}') sheet_configs = { "patients": { "patient_id_col": "patient_id", "date_columns": ["last_alive"], "text_columns": [], - "shift_exceptions": {"last_alive": ["2024-12-31"]}, } } shift_excel_dates_inplace( input_file=str(xlsx), output_file=str(out), patient_sheet="patients", - patient_id_col="patient_id", sheet_configs=sheet_configs, + linking_table_path=tmp_path / "does-not-exist.csv", linking_table_output=str(tmp_path / "linking.csv"), + shift_ignore_yaml=test_data / "shift-ignore.yaml", seed=42, ) df = pd.read_excel(str(out), sheet_name="patients") diff --git a/tests/test_e2e_date_shift.py b/tests/test_e2e_date_shift.py index ce0b5ad..7f10acb 100644 --- a/tests/test_e2e_date_shift.py +++ b/tests/test_e2e_date_shift.py @@ -60,8 +60,8 @@ def run_shift(base: Path, **kwargs: Unpack[_ShiftKwargs]) -> tuple[Path, Path]: str(INPUT_FILE), str(output), patient_sheet="patients", - patient_id_col="patient_id", sheet_configs=SHEET_CONFIGS, + linking_table_path=linking.parent / "old-linking.csv", linking_table_output=str(linking), **kwargs, ) diff --git a/uv.lock b/uv.lock index b906311..69878ed 100644 --- a/uv.lock +++ b/uv.lock @@ -17,7 +17,7 @@ exclude-newer-span = "P7D" [[package]] name = "book-page" version = "0.1.0" -source = { git = "https://github.com/peterlavalle/book_page.git#c8246740221cc72cb195f815dd3e97a41e320387" } +source = { git = "https://github.com/peterlavalle/book_page.git?rev=main#ef4758da764c41f5fe2bdb160d0fca60feb848ab" } dependencies = [ { name = "openpyxl" }, ] @@ -222,7 +222,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "book-page", git = "https://github.com/peterlavalle/book_page.git" }, + { name = "book-page", git = "https://github.com/peterlavalle/book_page.git?rev=main" }, { name = "datefinder", specifier = ">=1.0.0" }, { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pandas", specifier = ">=3.0.5" }, From a17e9900d6ac500f6cf368bb5fa27f1640059f12 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 13:05:07 +0100 Subject: [PATCH 52/82] allow header default to 0 in patient page, and, allow date values in the ignore file --- nuh_helper/date_shift/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index f7890a8..b8069bd 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -548,7 +548,10 @@ def shift_excel_dates_inplace( data = yaml.safe_load(file) shift_ignore: dict[str, dict[str, list[str]]] = { page.strip(): { - col.strip(): [ignored.strip() for ignored in data[page][col]] + col.strip(): [ + ignored.strip() if isinstance(ignored, str) else ignored + for ignored in data[page][col] + ] + ["", None] for col in data[page] } @@ -564,7 +567,7 @@ def shift_excel_dates_inplace( linking_table_path, linking_table_output or "shift_mappings.csv", sheet_configs[patient_sheet]["patient_id_col"], - sheet_configs[patient_sheet]["header_row"], + sheet_configs[patient_sheet].get("header_row", 0), sheet_configs[patient_sheet].get("skip_rows", []), seed, min_shift_days, From 17de1f46887283ffc7138b01f07d2f47453ded98 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 14:19:28 +0100 Subject: [PATCH 53/82] refactored the existing functions (as is) into separate files --- nuh_helper/date_shift/__init__.py | 689 ++------------------------ nuh_helper/date_shift/mappings.py | 111 +++++ nuh_helper/date_shift/stream_shift.py | 545 ++++++++++++++++++++ 3 files changed, 684 insertions(+), 661 deletions(-) create mode 100644 nuh_helper/date_shift/stream_shift.py diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index b8069bd..93bf03c 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -5,161 +5,35 @@ in an Excel file, with support for reproducible shifts using a linking table. """ -import csv import logging -import random -import time from datetime import date, datetime from pathlib import Path from typing import Any, cast -import datefinder import pandas as pd -import yaml -from book_page import Book, Page, book_open from nuh_helper.date_shift import _excel, _parse, mappings +from nuh_helper.date_shift.mappings import ( # noqa: E402 + PatientColumnMissing, + UnknownPatient, + generate_shift_mappings, + load_shift_mappings, +) +from nuh_helper.date_shift.stream_shift import ( + BlankColumnHasData, + DateColumnsMissing, + ExtraColumn, + ExtraPage, + HiddenDate, + PageMissing, + ShiftFoundNonDate, + TextColumnsMissing, + shift_excel_dates_inplace, +) logger = logging.getLogger(__name__) -class UnknownPatient(Exception): - def __init__(self, page: str, id: str) -> None: - message = f"Unknown {id=} on {page=}" - super().__init__(message) - self._message = message - - -class ShiftFoundNonDate(Exception): - def __init__( - self, page: str, row: int, col: int, col_name: str, value: str - ) -> None: - # the spaces and newline in the message make the value easier to copy/paste back - message = f"{page=}[{row}, {col} @ {col_name=} ]\n\t'{value}'" - super().__init__(message) - self._message: str = message - self._page: str = page - self._row: int = row - self._col: int = col - self._col_name: str = col_name - self._value: str = value - - -# >> pr 127 Exception goes here -# << end of pr 127 - -# >> pr 128 Exception goes here -# << end of pr 128 - - -class HiddenDate(Exception): - def __init__( - self, - sheet_name: str, - row: int, - col: int, - col_name: str, - value: str, - found: datetime, - ) -> None: - message = ( - f"hidden date in [{sheet_name=}, {row}, {col} @ {col_name}] {value=} " - + f"// {found=}" - ) - super().__init__(message) - self._message = message - self._sheet_name = sheet_name - self._row = row - self._col = col - self._col_name = col_name - self._value = value - self._found = found - - -# >> pr 131 Exception goes here -# << end of pr 131 - - -class DateColumnsMissing(Exception): - def __init__(self, page_name: str, column_names: list[str]) -> None: - message = f"date {column_names=} is missing from the cdm {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_names = column_names - - -class TextColumnsMissing(Exception): - def __init__(self, page_name: str, column_names: list[str]) -> None: - message = f"text {column_names=} is missing from the cdm {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_names = column_names - - -class BlankColumnHasData(Exception): - """raised when a column with a blank name has data. - - prevents data being hidden in the wrong part of the CDM""" - - def __init__(self, page: str, row: int, col: int, value: any) -> None: - message = f"[{page=} @ {row}, {col}] is a blank column with data {value=}" - super().__init__(message) - self._message = message - self._page: str = page - self._row: int = row - self._col: int = col - self._value: any = value - - -class PatientColumnMissing(Exception): - """raised when the patient column is missing from any sheet""" - - def __init__(self, page_name: str, column_name: str) -> None: - message = f"The patient {column_name=} is not present on {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_name = column_name - - -class ExtraColumn(Exception): - """raised when an unknown column appears in a page we're shifting. - - could mean a column is named wrong, or, that the sheet_config is incomplete""" - - def __init__(self, page_name: str, column_name: str) -> None: - message = ( - f"{column_name=} is neither ignored or shifted in the cdm {page_name=}" - ) - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_name = column_name - - -class PageMissing(Exception): - def __init__(self, page_name: str) -> None: - message = f"an expected page was missing from the source cdm {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - - -class ExtraPage(Exception): - def __init__(self, page_name: str) -> None: - message = f"an unexpected page was found in the source cdm {page_name=}" - super().__init__(message) - self._message = message - self._page_name = page_name - - def _get_patient_ids_and_shift_mappings( input_file: Path, patient_sheet: str, @@ -197,7 +71,7 @@ def _get_patient_ids_and_shift_mappings( skip_rows_after_header=effective_patient_skip_rows, ) if patient_id_col not in patient_df.columns: - raise PatientColumnMissing(patient_sheet, patient_id_col) + raise mappings.PatientColumnMissing(patient_sheet, patient_id_col) patient_ids = ( patient_df[patient_id_col] @@ -461,526 +335,19 @@ def shift_excel_dates( logger.info("Linking table saved to '%s'", linking_path) -def shift_excel_dates_inplace( - input_file: str | Path | list[Path], - output_file: str | Path, - patient_sheet: str, - sheet_configs: dict[str, dict[str, Any]], - min_shift_days: int = -15, - max_shift_days: int = 15, - linking_table_path: str | None = None, - linking_table_output: str | None = None, - seed: int | None = None, - shift_ignore_yaml: None | str | Path = None, -) -> None: - """ - Shift dates in an Excel file, preserving all cell formatting. - - Unlike shift_excel_dates(), this function copies the input file and then - modifies date cells directly via openpyxl, so all formatting (cell styles, - merged cells, column widths, conditional formatting, etc.) is preserved. - - Args: - input_file: Path to input Excel file. - output_file: Path for the output file (copy of input with shifted dates). - patient_sheet: Name of the sheet containing patient IDs. - sheet_configs: - Dictionary mapping sheet names to configuration dicts, or, the string - 'skip' if that sheet should be skipped but is a valid part of the - CDM. - Each config dict should have: - - 'patient_id_col': Name of patient ID column in that sheet - - 'date_columns': List of date column names to shift - - 'text_columns': List of non-date columns to ignore - Optional per-sheet header handling: - - 'header_row': zero-based row index of the column names (default 0) - - 'skip_rows_after_header': list of zero-based row indices to - exclude from data (e.g. a data-type row immediately below the - header) - min_shift_days: Minimum number of days to shift (default: -15). - max_shift_days: Maximum number of days to shift (default: 15). - linking_table_path: Optional path to existing linking table CSV for reproducibility. - linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). - seed: Optional random seed for generating shifts. - shift_ignore_yaml: - path to a .yaml file holding `page:{column:[values]}` lists of cell - values that're ignored and passed as-is with no manipulation. None - and "" are always added, and all values will be .strip() - - (Defaults to linking_table_path.parent / shift_ignore.do-not-commit.yaml) - """ # noqa: E501 - logger.info( - "Shifting dates in-place: '%s' → '%s'", - input_file - if not isinstance(input_file, list) - else [file.name for file in input_file], - output_file, - ) - logger.debug( - "Shift range: %d to %d days, seed=%s", - min_shift_days, - max_shift_days, - seed, - ) - - book: Book = book_open(input_file) - logger.info(f"Book has {type(book)=}") - # TODO; it'd be cool to "normalize" the linking_table_path/linking_table_output here - - # get the shift_ignore: dict[str, dict[str, list[str]]] from shift_ignore_yaml: Path - if not shift_ignore_yaml: - if linking_table_path: - shift_ignore_yaml = ( - Path(linking_table_path).parent / "shift_ignore.do-not-commit.yaml" - ) - else: - raise RuntimeError( - "can't guess shift_ignore_yaml without linking_table_path" - ) - elif not isinstance(shift_ignore_yaml, Path): - shift_ignore_yaml = Path(shift_ignore_yaml) - - if not shift_ignore_yaml.is_file(): - shift_ignore: dict[str, dict[str, set[str]]] = {} - logger.warning(f"didn't find shift_ignore file {shift_ignore_yaml=}") - else: - with open(shift_ignore_yaml) as file: - data = yaml.safe_load(file) - shift_ignore: dict[str, dict[str, list[str]]] = { - page.strip(): { - col.strip(): [ - ignored.strip() if isinstance(ignored, str) else ignored - for ignored in data[page][col] - ] - + ["", None] - for col in data[page] - } - for page in data - } - - if patient_sheet not in sheet_configs: - raise PageMissing(patient_sheet) - - with book[patient_sheet] as page: - shift_deltas = patient_shift_deltas( - page, - linking_table_path, - linking_table_output or "shift_mappings.csv", - sheet_configs[patient_sheet]["patient_id_col"], - sheet_configs[patient_sheet].get("header_row", 0), - sheet_configs[patient_sheet].get("skip_rows", []), - seed, - min_shift_days, - max_shift_days, - ) - logger.info(f"loaded deltas {len(shift_deltas)=}") - - # check for missing sheets - for sheet_name in sheet_configs: - if sheet_name not in book: - raise PageMissing(sheet_name) - - last_name = None - last_row = None - skipping_sheet_name = "" - - ## - # this calls the `.stream_copy()` method to copy from one book to another - # WARNING; this works like a callback rather than a loop; - # - each iteration is a callback from copy with a source row - # - each row is passed with a sheet name and a row number - # - modify the values in the row_cells to change what's in the output file(s) - # - no option to add/remove rows is available - # - no option to jump/seek through rows is available - # - ... which is why this acts a bit weird - for sheet_name, row_idx, row_cells in book.stream_copy(output_file): - ##### - # Part-1; check if we've reached a different sheet - ## - assert isinstance(sheet_name, str) - if row_idx != 0: - # SAN/safety check in case something tries to skip showing us rows - if sheet_name != last_name: - raise RuntimeError(f"didn't see row0 from {sheet_name=}") - last_row = row_idx - else: - if last_name and last_row: - logger.info(f"Shifting {last_name=} processed {last_row} rows") - last_name = sheet_name - last_row = row_idx - - # check that it's in the configs - if sheet_name not in sheet_configs: - raise ExtraPage(sheet_name) - - sheet_config: dict | str = sheet_configs[sheet_name] - - # do nothing to the skipped ones - if sheet_config == "skip": - skipping_sheet_name = sheet_name - continue - - # grab the configuration - patient_id_col: str = cast(str, sheet_config["patient_id_col"]).strip() - date_columns: list[str] = [ - col.strip() for col in sheet_config["date_columns"] - ] - text_columns: list[str] = [ - col.strip() for col in sheet_config["text_columns"] - ] - - header_row: int = cast(int, sheet_config.get("header_row", 0)) - skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) - - ### - # do some checks of the configuration - ## - - # check for holes in skip_rows (possible issue) - if skip_rows: - missing_skipped = [ - not_skipped + 1 - for not_skipped in range(header_row, max(skip_rows)) - if (not_skipped + 1) not in skip_rows - ] - if missing_skipped: - raise RuntimeError( - f"{sheet_name=} and {missing_skipped=} but" - + f" {header_row=} and {skip_rows=}" - ) - - # check if the config hard-codes patient data to skip - if "shift_exceptions" in sheet_config: - raise RuntimeError( - f"shift_exceptions was removed, update {sheet_name=}" - ) - - # check patient_id_col isn't re-used as other column types - if patient_id_col in date_columns: - raise ValueError( - f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" - ) - if patient_id_col in text_columns: - raise ValueError( - f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" - ) - - # check that column names don't appear in both - col_names = [col for col in date_columns if col in text_columns] - if col_names: - raise ValueError( - f"{sheet_name=} has the some columns in both date and text " - + str(col_names) - ) - - col_names = None - patient_id_idx = None - - ##### - # Part-2; skip skipped sheets, and, skip skipped rows - ## - - # do nothing to the skipped sheets - if sheet_config == "skip": - # "safety" check so that we only skip rows with the correct name - if skipping_sheet_name != sheet_name: - raise RuntimeError("this shouldn't happen") - continue - logger.info(f"Shifting {sheet_name=}") - - # skip pre-header and skip rows - if (row_idx < header_row) or (row_idx in skip_rows): - # there's an earlier check to find "holes" in the skip-rows - continue - - ##### - # Part-3; handle the header rows - ## - if row_idx == header_row: - # get the real column names now - col_names = row_cells - - # find patient_id_idx and normalize the column name list - for col_idx in range(len(col_names)): - col_name = col_names[col_idx] - - # if the name is None ... leave it as that (it's fine) - if col_name is None: - continue - - # strip the name - col_name = col_name.strip() - - # if the name is now blank; store None as the column name - if col_name == "": - col_names[col_idx] = None - continue - - # update it to just be the stripped version - col_names[col_idx] = col_name - - # check the config to see if we know what to do with this column - if col_name not in ([patient_id_col] + date_columns + text_columns): - raise ExtraColumn(sheet_name, col_name) - - # select the patient id - if col_name == patient_id_col: - patient_id_idx = col_idx - - # check to be sure we found the patient_id_col/patient_id_idx - if patient_id_idx is None: - raise PatientColumnMissing(sheet_name, patient_id_col) - - missing = [col for col in text_columns if col not in col_names] - if missing: - raise TextColumnsMissing(sheet_name, missing) - missing = [col for col in date_columns if col not in col_names] - if missing: - raise DateColumnsMissing(sheet_name, missing) - - continue - - ##### - # Part-4; process normal columns - ## - if col_names is None: - raise RuntimeError("does the sheet skip all the skip rows?") - else: - # code used to be indented like this - - ### - # process each row of the workbook - ## - - # write a log message every 40 rows (show we're not crashed) - if (row_idx % 40) == 0: - logger.info(f"Shifting {sheet_name=} up to row {row_idx}") - - # get the patient id for this row - pid = row_cells[patient_id_idx] - - # check for date in the cell_value - for found in datefinder.find_dates(str(pid)): - raise HiddenDate( - sheet_name, - row_idx, - patient_id_idx, - patient_id_col, - pid, - found, - ) - - pid = _parse._normalize_patient_id(pid) - if pid is None: - # if the pid is None; the rest of the row should be None as well - non_blank = [ - cell - for cell in row_cells - # keep the cells that aren't blank - if cell is not None and str(cell).strip() != "" - ] - if non_blank: - raise ValueError( - f"{row_idx=} has no pid, should be blank but has {non_blank}" - ) - - # we've completed the checks for a None pid - # ... so we need to skip the rest of this loop - continue - - # determine how many days to shift - shift_delta = shift_deltas.get(pid) - if shift_delta is None: - # all patients need to have a shift value - raise UnknownPatient(sheet_name, pid) - - # now scan each column - # ... even when page has no date_columns; still check for hidden dates - for col_idx in range(len(row_cells)): - # skip the patient id column (it was already checked anyway) - if col_idx == patient_id_idx: - continue - - # get the cell value. replace it with None if it's just whitespace - cell_value = row_cells[col_idx] - if isinstance(cell_value, str): - cell_value = cell_value.strip() - if not cell_value: - cell_value = None - - # so if the cell is empty; skip the rest of the checks for this cell - # ... we go tot he next column in the row - if cell_value is None: - row_cells[col_idx] = None - continue - - # if there's no column name; this column of the row should be None - # ... and we win't reach this line if the cell_value was None - col_name = col_names[col_idx] - if col_name is None: - raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) - - # skip values in shift_ignore - if "shift_ignore" in sheet_config: - raise RuntimeError("move shift_ignore from sheet_configs to a file") - - if sheet_name not in shift_ignore: - shift_ignore[sheet_name] = {} - - if col_name not in shift_ignore[sheet_name]: - shift_ignore[sheet_name][col_name] = {None, ""} - - if cell_value in shift_ignore[sheet_name][col_name]: - continue - - # for text columns, we now *just* need to check for a hidden date - if col_name in text_columns: - # check for dates in non-date columns - for found in datefinder.find_dates(str(cell_value)): - logger.debug( - f"found hidden date {sheet_name=}/{col_name=}:{cell_value}" - ) - ignored = shift_ignore[sheet_name][col_name] - logger.debug( - f"shift_ignore[{sheet_name=}][{col_name=}]= {len(ignored)}" - ) - for ig in ignored: - logger.debug(f"- {ig}") - - raise HiddenDate( - sheet_name, - row_idx, - col_idx, - col_name, - cell_value, - found, - ) - row_cells[col_idx] = cell_value - continue - - # we've already checked this (sort of) - # assert col_name in date_columns - - # cell_value can be str | datetime | date - # ... so parsed might not always succeed - parsed = _parse._parse_date_value(cell_value) - if parsed is None: - # raise an error; we already handled values we should pass as-is - raise ShiftFoundNonDate( - sheet_name, row_idx, col_idx, col_name, cell_value - ) - - # perform the actual shifting - shifted = parsed + shift_delta - - # controls wether the data is displayed with the 00:00:00 in Excel - # ... it might be nice to just drop the time if it's 00:00:00 - if isinstance(cell_value, datetime): - row_cells[col_idx] = cast(Any, shifted.to_pydatetime()) - else: - row_cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) - print('redo cell.number_format = "yyyy-mm-dd"') - - logger.info( - f"Shifting {last_name=} processed {last_row} rows (and that's the last page)" - ) - logger.info("Output written to '%s'", output_file) - - -def patient_shift_deltas( - page: Page, - src_linking_table: str | Path, - out_linking_table: str | Path, - patient_id_col_name: str, - patient_header_row: int, - patient_skip_rows: list[int] | None, - seed: int | None, - min_shift_days: int, - max_shift_days: int, -) -> dict[str, pd.Timedelta]: - - logger.info("patient_shift_deltas() ; preparing per-patient shifts") - - # find the patient id index - patient_id_col: None | int = None - patient_ids = [] - start = time.time() - - for row, data in page.stream_rows(): - if row < patient_header_row or row in patient_skip_rows: - continue - elif row == patient_header_row: - if patient_id_col_name not in data: - raise PatientColumnMissing(page.name, patient_id_col_name) - patient_id_col = data.index(patient_id_col_name) - else: - patient_id = data[patient_id_col] - if not patient_id: - continue - patient_id = _parse._normalize_patient_id(patient_id.strip()) - if patient_id in patient_ids: - continue - patient_ids.append(patient_id) - if (len(patient_ids) % 40) == 0: - seconds = time.time() - start - logger.info( - f"patient_shift_deltas() ; so far {len(patient_ids)=} in {seconds=}" - ) - logger.info(f"patient_shift_deltas() ; Found {len(patient_ids)=} in {page.name=}") - - # load the old ids (should we remove IDs that are "gone"?) - if isinstance(src_linking_table, str): - src_linking_table = Path(src_linking_table) - if not src_linking_table.is_file(): - old_mappings: dict[str, int] = {} - else: - with open(src_linking_table) as file: - old_mappings: dict[str, int] = { - _parse._normalize_patient_id(row[0]): int(row[1]) - for row in csv.reader(file) - if row and row[1] and row[1].strip() != "shift_days" - # if row and row[0] and row[0].strip() != "patient_id" - } - logger.info( - f"patient_shift_deltas() ; Loaded {len(old_mappings)=} in {src_linking_table =}" - ) - - # create the list of all mappings - - if seed: - random.seed(seed) - all_mappings: dict[str, int] = { - id: ( - old_mappings[id] - if id in old_mappings - else random.randint(min_shift_days, max_shift_days) - ) - for id in patient_ids - } - removed = len([k for k in all_mappings if k not in patient_ids]) - logger.info( - f"patient_shift_deltas() ; {len(all_mappings)=} patient(s) (... and {removed=})" - ) - - # store all mappings in the out file - with open(out_linking_table, "w") as file: - csv.writer(file).writerows( - [["patient_id", "shift_days"]] - + [[key, all_mappings[key]] for key in sorted(all_mappings)] - ) - logger.info(f"patient_shift_deltas() ; Saved mappings {out_linking_table}") - - return {k: pd.Timedelta(days=all_mappings[k]) for k in all_mappings} - - # Re-export public API -from nuh_helper.date_shift.mappings import ( # noqa: E402 - generate_shift_mappings, - load_shift_mappings, -) - __all__ = [ generate_shift_mappings, load_shift_mappings, shift_excel_dates_inplace, + HiddenDate, + PatientColumnMissing, + ShiftFoundNonDate, + BlankColumnHasData, + DateColumnsMissing, + ExtraColumn, + PageMissing, + ExtraPage, + TextColumnsMissing, + UnknownPatient, ] diff --git a/nuh_helper/date_shift/mappings.py b/nuh_helper/date_shift/mappings.py index 4ee5b81..45b6818 100644 --- a/nuh_helper/date_shift/mappings.py +++ b/nuh_helper/date_shift/mappings.py @@ -1,9 +1,15 @@ """Shift mapping generation and loading for date-shift.""" +import csv import logging import random +import time +from pathlib import Path import pandas as pd +from book_page import Page + +from nuh_helper.date_shift import _parse logger = logging.getLogger(__name__) @@ -49,3 +55,108 @@ def load_shift_mappings(csv_path: str) -> pd.DataFrame: raise ValueError("CSV must contain 'patient_id' and 'shift_days' columns") logger.info("Loaded %d shift mapping(s) from '%s'", len(df), csv_path) return df + + +class PatientColumnMissing(Exception): + """raised when the patient column is missing from any sheet""" + + def __init__(self, page_name: str, column_name: str) -> None: + message = f"The patient {column_name=} is not present on {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_name = column_name + + +class UnknownPatient(Exception): + def __init__(self, page: str, id: str) -> None: + message = f"Unknown {id=} on {page=}" + super().__init__(message) + self._message = message + + +def patient_shift_deltas( + page: Page, + src_linking_table: str | Path, + out_linking_table: str | Path, + patient_id_col_name: str, + patient_header_row: int, + patient_skip_rows: list[int] | None, + seed: int | None, + min_shift_days: int, + max_shift_days: int, +) -> dict[str, pd.Timedelta]: + + logger.info("patient_shift_deltas() ; preparing per-patient shifts") + + # find the patient id index + patient_id_col: None | int = None + patient_ids = [] + start = time.time() + + for row, data in page.stream_rows(): + if row < patient_header_row or row in patient_skip_rows: + continue + elif row == patient_header_row: + if patient_id_col_name not in data: + raise PatientColumnMissing(page.name, patient_id_col_name) + patient_id_col = data.index(patient_id_col_name) + else: + patient_id = data[patient_id_col] + if not patient_id: + continue + patient_id = _parse._normalize_patient_id(patient_id.strip()) + if patient_id in patient_ids: + continue + patient_ids.append(patient_id) + if (len(patient_ids) % 40) == 0: + seconds = time.time() - start + logger.info( + f"patient_shift_deltas() ; so far {len(patient_ids)=} in {seconds=}" + ) + logger.info(f"patient_shift_deltas() ; Found {len(patient_ids)=} in {page.name=}") + + # load the old ids (should we remove IDs that are "gone"?) + if isinstance(src_linking_table, str): + src_linking_table = Path(src_linking_table) + if not src_linking_table.is_file(): + old_mappings: dict[str, int] = {} + else: + with open(src_linking_table) as file: + old_mappings: dict[str, int] = { + _parse._normalize_patient_id(row[0]): int(row[1]) + for row in csv.reader(file) + if row and row[1] and row[1].strip() != "shift_days" + # if row and row[0] and row[0].strip() != "patient_id" + } + logger.info( + f"patient_shift_deltas() ; Loaded {len(old_mappings)=} in {src_linking_table =}" + ) + + # create the list of all mappings + + if seed: + random.seed(seed) + all_mappings: dict[str, int] = { + id: ( + old_mappings[id] + if id in old_mappings + else random.randint(min_shift_days, max_shift_days) + ) + for id in patient_ids + } + removed = len([k for k in all_mappings if k not in patient_ids]) + logger.info( + f"patient_shift_deltas() ; {len(all_mappings)=} patient(s) (... and {removed=})" + ) + + # store all mappings in the out file + with open(out_linking_table, "w") as file: + csv.writer(file).writerows( + [["patient_id", "shift_days"]] + + [[key, all_mappings[key]] for key in sorted(all_mappings)] + ) + logger.info(f"patient_shift_deltas() ; Saved mappings {out_linking_table}") + + return {k: pd.Timedelta(days=all_mappings[k]) for k in all_mappings} diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py new file mode 100644 index 0000000..1ddc71f --- /dev/null +++ b/nuh_helper/date_shift/stream_shift.py @@ -0,0 +1,545 @@ +import logging +from datetime import datetime +from pathlib import Path +from typing import Any, cast + +import datefinder +import yaml +from book_page import Book, book_open + +from nuh_helper.date_shift import _parse, mappings + +logger = logging.getLogger(__name__) + + +class ShiftFoundNonDate(Exception): + def __init__( + self, page: str, row: int, col: int, col_name: str, value: str + ) -> None: + # the spaces and newline in the message make the value easier to copy/paste back + message = f"{page=}[{row}, {col} @ {col_name=} ]\n\t'{value}'" + super().__init__(message) + self._message: str = message + self._page: str = page + self._row: int = row + self._col: int = col + self._col_name: str = col_name + self._value: str = value + + +class HiddenDate(Exception): + def __init__( + self, + sheet_name: str, + row: int, + col: int, + col_name: str, + value: str, + found: datetime, + ) -> None: + message = ( + f"hidden date in [{sheet_name=}, {row}, {col} @ {col_name}] {value=} " + + f"// {found=}" + ) + super().__init__(message) + self._message = message + self._sheet_name = sheet_name + self._row = row + self._col = col + self._col_name = col_name + self._value = value + self._found = found + + +class DateColumnsMissing(Exception): + def __init__(self, page_name: str, column_names: list[str]) -> None: + message = f"date {column_names=} is missing from the cdm {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_names = column_names + + +class TextColumnsMissing(Exception): + def __init__(self, page_name: str, column_names: list[str]) -> None: + message = f"text {column_names=} is missing from the cdm {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_names = column_names + + +class BlankColumnHasData(Exception): + """raised when a column with a blank name has data. + + prevents data being hidden in the wrong part of the CDM""" + + def __init__(self, page: str, row: int, col: int, value: any) -> None: + message = f"[{page=} @ {row}, {col}] is a blank column with data {value=}" + super().__init__(message) + self._message = message + self._page: str = page + self._row: int = row + self._col: int = col + self._value: any = value + + +class ExtraColumn(Exception): + """raised when an unknown column appears in a page we're shifting. + + could mean a column is named wrong, or, that the sheet_config is incomplete""" + + def __init__(self, page_name: str, column_name: str) -> None: + message = ( + f"{column_name=} is neither ignored or shifted in the cdm {page_name=}" + ) + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_name = column_name + + +class PageMissing(Exception): + def __init__(self, page_name: str) -> None: + message = f"an expected page was missing from the source cdm {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + + +class ExtraPage(Exception): + def __init__(self, page_name: str) -> None: + message = f"an unexpected page was found in the source cdm {page_name=}" + super().__init__(message) + self._message = message + self._page_name = page_name + + +def shift_excel_dates_inplace( + input_file: str | Path | list[Path], + output_file: str | Path, + patient_sheet: str, + sheet_configs: dict[str, dict[str, Any]], + min_shift_days: int = -15, + max_shift_days: int = 15, + linking_table_path: str | None = None, + linking_table_output: str | None = None, + seed: int | None = None, + shift_ignore_yaml: None | str | Path = None, +) -> None: + """ + Shift dates in an Excel file, preserving all cell formatting. + + Unlike shift_excel_dates(), this function copies the input file and then + modifies date cells directly via openpyxl, so all formatting (cell styles, + merged cells, column widths, conditional formatting, etc.) is preserved. + + Args: + input_file: Path to input Excel file. + output_file: Path for the output file (copy of input with shifted dates). + patient_sheet: Name of the sheet containing patient IDs. + sheet_configs: + Dictionary mapping sheet names to configuration dicts, or, the string + 'skip' if that sheet should be skipped but is a valid part of the + CDM. + Each config dict should have: + - 'patient_id_col': Name of patient ID column in that sheet + - 'date_columns': List of date column names to shift + - 'text_columns': List of non-date columns to ignore + Optional per-sheet header handling: + - 'header_row': zero-based row index of the column names (default 0) + - 'skip_rows_after_header': list of zero-based row indices to + exclude from data (e.g. a data-type row immediately below the + header) + min_shift_days: Minimum number of days to shift (default: -15). + max_shift_days: Maximum number of days to shift (default: 15). + linking_table_path: Optional path to existing linking table CSV for reproducibility. + linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). + seed: Optional random seed for generating shifts. + shift_ignore_yaml: + path to a .yaml file holding `page:{column:[values]}` lists of cell + values that're ignored and passed as-is with no manipulation. None + and "" are always added, and all values will be .strip() + + (Defaults to linking_table_path.parent / shift_ignore.do-not-commit.yaml) + """ # noqa: E501 + logger.info( + "Shifting dates in-place: '%s' → '%s'", + input_file + if not isinstance(input_file, list) + else [file.name for file in input_file], + output_file, + ) + logger.debug( + "Shift range: %d to %d days, seed=%s", + min_shift_days, + max_shift_days, + seed, + ) + + book: Book = book_open(input_file) + logger.info(f"Book has {type(book)=}") + # TODO; it'd be cool to "normalize" the linking_table_path/linking_table_output here + + # get the shift_ignore: dict[str, dict[str, list[str]]] from shift_ignore_yaml: Path + if not shift_ignore_yaml: + if linking_table_path: + shift_ignore_yaml = ( + Path(linking_table_path).parent / "shift_ignore.do-not-commit.yaml" + ) + else: + raise RuntimeError( + "can't guess shift_ignore_yaml without linking_table_path" + ) + elif not isinstance(shift_ignore_yaml, Path): + shift_ignore_yaml = Path(shift_ignore_yaml) + + if not shift_ignore_yaml.is_file(): + shift_ignore: dict[str, dict[str, set[str]]] = {} + logger.warning(f"didn't find shift_ignore file {shift_ignore_yaml=}") + else: + with open(shift_ignore_yaml) as file: + data = yaml.safe_load(file) + shift_ignore: dict[str, dict[str, list[str]]] = { + page.strip(): { + col.strip(): [ + ignored.strip() if isinstance(ignored, str) else ignored + for ignored in data[page][col] + ] + + ["", None] + for col in data[page] + } + for page in data + } + + if patient_sheet not in sheet_configs: + raise PageMissing(patient_sheet) + + with book[patient_sheet] as page: + shift_deltas = mappings.patient_shift_deltas( + page, + linking_table_path, + linking_table_output or "shift_mappings.csv", + sheet_configs[patient_sheet]["patient_id_col"], + sheet_configs[patient_sheet].get("header_row", 0), + sheet_configs[patient_sheet].get("skip_rows", []), + seed, + min_shift_days, + max_shift_days, + ) + logger.info(f"loaded deltas {len(shift_deltas)=}") + + # check for missing sheets + for sheet_name in sheet_configs: + if sheet_name not in book: + raise PageMissing(sheet_name) + + last_name = None + last_row = None + skipping_sheet_name = "" + + ## + # this calls the `.stream_copy()` method to copy from one book to another + # WARNING; this works like a callback rather than a loop; + # - each iteration is a callback from copy with a source row + # - each row is passed with a sheet name and a row number + # - modify the values in the row_cells to change what's in the output file(s) + # - no option to add/remove rows is available + # - no option to jump/seek through rows is available + # - ... which is why this acts a bit weird + for sheet_name, row_idx, row_cells in book.stream_copy(output_file): + ##### + # Part-1; check if we've reached a different sheet + ## + assert isinstance(sheet_name, str) + if row_idx != 0: + # SAN/safety check in case something tries to skip showing us rows + if sheet_name != last_name: + raise RuntimeError(f"didn't see row0 from {sheet_name=}") + last_row = row_idx + else: + if last_name and last_row: + logger.info(f"Shifting {last_name=} processed {last_row} rows") + last_name = sheet_name + last_row = row_idx + + # check that it's in the configs + if sheet_name not in sheet_configs: + raise ExtraPage(sheet_name) + + sheet_config: dict | str = sheet_configs[sheet_name] + + # do nothing to the skipped ones + if sheet_config == "skip": + skipping_sheet_name = sheet_name + continue + + # grab the configuration + patient_id_col: str = cast(str, sheet_config["patient_id_col"]).strip() + date_columns: list[str] = [ + col.strip() for col in sheet_config["date_columns"] + ] + text_columns: list[str] = [ + col.strip() for col in sheet_config["text_columns"] + ] + + header_row: int = cast(int, sheet_config.get("header_row", 0)) + skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) + + ### + # do some checks of the configuration + ## + + # check for holes in skip_rows (possible issue) + if skip_rows: + missing_skipped = [ + not_skipped + 1 + for not_skipped in range(header_row, max(skip_rows)) + if (not_skipped + 1) not in skip_rows + ] + if missing_skipped: + raise RuntimeError( + f"{sheet_name=} and {missing_skipped=} but" + + f" {header_row=} and {skip_rows=}" + ) + + # check if the config hard-codes patient data to skip + if "shift_exceptions" in sheet_config: + raise RuntimeError( + f"shift_exceptions was removed, update {sheet_name=}" + ) + + # check patient_id_col isn't re-used as other column types + if patient_id_col in date_columns: + raise ValueError( + f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" + ) + if patient_id_col in text_columns: + raise ValueError( + f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" + ) + + # check that column names don't appear in both + col_names = [col for col in date_columns if col in text_columns] + if col_names: + raise ValueError( + f"{sheet_name=} has the some columns in both date and text " + + str(col_names) + ) + + col_names = None + patient_id_idx = None + + ##### + # Part-2; skip skipped sheets, and, skip skipped rows + ## + + # do nothing to the skipped sheets + if sheet_config == "skip": + # "safety" check so that we only skip rows with the correct name + if skipping_sheet_name != sheet_name: + raise RuntimeError("this shouldn't happen") + continue + logger.info(f"Shifting {sheet_name=}") + + # skip pre-header and skip rows + if (row_idx < header_row) or (row_idx in skip_rows): + # there's an earlier check to find "holes" in the skip-rows + continue + + ##### + # Part-3; handle the header rows + ## + if row_idx == header_row: + # get the real column names now + col_names = row_cells + + # find patient_id_idx and normalize the column name list + for col_idx in range(len(col_names)): + col_name = col_names[col_idx] + + # if the name is None ... leave it as that (it's fine) + if col_name is None: + continue + + # strip the name + col_name = col_name.strip() + + # if the name is now blank; store None as the column name + if col_name == "": + col_names[col_idx] = None + continue + + # update it to just be the stripped version + col_names[col_idx] = col_name + + # check the config to see if we know what to do with this column + if col_name not in ([patient_id_col] + date_columns + text_columns): + raise ExtraColumn(sheet_name, col_name) + + # select the patient id + if col_name == patient_id_col: + patient_id_idx = col_idx + + # check to be sure we found the patient_id_col/patient_id_idx + if patient_id_idx is None: + raise mappings.PatientColumnMissing(sheet_name, patient_id_col) + + missing = [col for col in text_columns if col not in col_names] + if missing: + raise TextColumnsMissing(sheet_name, missing) + missing = [col for col in date_columns if col not in col_names] + if missing: + raise DateColumnsMissing(sheet_name, missing) + + continue + + ##### + # Part-4; process normal columns + ## + if col_names is None: + raise RuntimeError("does the sheet skip all the skip rows?") + else: + # code used to be indented like this + + ### + # process each row of the workbook + ## + + # write a log message every 40 rows (show we're not crashed) + if (row_idx % 40) == 0: + logger.info(f"Shifting {sheet_name=} up to row {row_idx}") + + # get the patient id for this row + pid = row_cells[patient_id_idx] + + # check for date in the cell_value + for found in datefinder.find_dates(str(pid)): + raise HiddenDate( + sheet_name, + row_idx, + patient_id_idx, + patient_id_col, + pid, + found, + ) + + pid = _parse._normalize_patient_id(pid) + if pid is None: + # if the pid is None; the rest of the row should be None as well + non_blank = [ + cell + for cell in row_cells + # keep the cells that aren't blank + if cell is not None and str(cell).strip() != "" + ] + if non_blank: + raise ValueError( + f"{row_idx=} has no pid, should be blank but has {non_blank}" + ) + + # we've completed the checks for a None pid + # ... so we need to skip the rest of this loop + continue + + # determine how many days to shift + shift_delta = shift_deltas.get(pid) + if shift_delta is None: + # all patients need to have a shift value + raise mappings.UnknownPatient(sheet_name, pid) + + # now scan each column + # ... even when page has no date_columns; still check for hidden dates + for col_idx in range(len(row_cells)): + # skip the patient id column (it was already checked anyway) + if col_idx == patient_id_idx: + continue + + # get the cell value. replace it with None if it's just whitespace + cell_value = row_cells[col_idx] + if isinstance(cell_value, str): + cell_value = cell_value.strip() + if not cell_value: + cell_value = None + + # so if the cell is empty; skip the rest of the checks for this cell + # ... we go tot he next column in the row + if cell_value is None: + row_cells[col_idx] = None + continue + + # if there's no column name; this column of the row should be None + # ... and we win't reach this line if the cell_value was None + col_name = col_names[col_idx] + if col_name is None: + raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) + + # skip values in shift_ignore + if "shift_ignore" in sheet_config: + raise RuntimeError("move shift_ignore from sheet_configs to a file") + + if sheet_name not in shift_ignore: + shift_ignore[sheet_name] = {} + + if col_name not in shift_ignore[sheet_name]: + shift_ignore[sheet_name][col_name] = {None, ""} + + if cell_value in shift_ignore[sheet_name][col_name]: + continue + + # for text columns, we now *just* need to check for a hidden date + if col_name in text_columns: + # check for dates in non-date columns + for found in datefinder.find_dates(str(cell_value)): + logger.debug( + f"found hidden date {sheet_name=}/{col_name=}:{cell_value}" + ) + ignored = shift_ignore[sheet_name][col_name] + logger.debug( + f"shift_ignore[{sheet_name=}][{col_name=}]= {len(ignored)}" + ) + for ig in ignored: + logger.debug(f"- {ig}") + + raise HiddenDate( + sheet_name, + row_idx, + col_idx, + col_name, + cell_value, + found, + ) + row_cells[col_idx] = cell_value + continue + + # we've already checked this (sort of) + # assert col_name in date_columns + + # cell_value can be str | datetime | date + # ... so parsed might not always succeed + parsed = _parse._parse_date_value(cell_value) + if parsed is None: + # raise an error; we already handled values we should pass as-is + raise ShiftFoundNonDate( + sheet_name, row_idx, col_idx, col_name, cell_value + ) + + # perform the actual shifting + shifted = parsed + shift_delta + + # controls wether the data is displayed with the 00:00:00 in Excel + # ... it might be nice to just drop the time if it's 00:00:00 + if isinstance(cell_value, datetime): + row_cells[col_idx] = cast(Any, shifted.to_pydatetime()) + else: + row_cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) + print('redo cell.number_format = "yyyy-mm-dd"') + + logger.info( + f"Shifting {last_name=} processed {last_row} rows (and that's the last page)" + ) + logger.info("Output written to '%s'", output_file) From 4a5e4076af041906ff6b18791110b611380f614f Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 14:53:46 +0100 Subject: [PATCH 54/82] brings book_page (and tests) into the project ... and updates book_page to pass our quality gates --- nuh_helper/book_page/__init__.py | 11 + nuh_helper/book_page/book.py | 37 +++ nuh_helper/book_page/csv_book.py | 114 ++++++++ nuh_helper/book_page/generic.py | 25 ++ nuh_helper/book_page/xlsx_book.py | 61 +++++ nuh_helper/date_shift/mappings.py | 2 +- nuh_helper/date_shift/stream_shift.py | 2 +- pyproject.toml | 4 - tests/data/test_book_page/book/data.xlsx | Bin 0 -> 9873 bytes tests/data/test_book_page/book/data_3x4.csv | 4 + tests/data/test_book_page/book/data_3x6.tsv | 4 + tests/data/test_book_page/page-desc.csv | 7 + tests/test_book_page.py | 278 ++++++++++++++++++++ tests/test_shift_ignore.py | 3 +- uv.lock | 10 - 15 files changed, 544 insertions(+), 18 deletions(-) create mode 100644 nuh_helper/book_page/__init__.py create mode 100644 nuh_helper/book_page/book.py create mode 100644 nuh_helper/book_page/csv_book.py create mode 100644 nuh_helper/book_page/generic.py create mode 100644 nuh_helper/book_page/xlsx_book.py create mode 100644 tests/data/test_book_page/book/data.xlsx create mode 100644 tests/data/test_book_page/book/data_3x4.csv create mode 100644 tests/data/test_book_page/book/data_3x6.tsv create mode 100644 tests/data/test_book_page/page-desc.csv create mode 100644 tests/test_book_page.py diff --git a/nuh_helper/book_page/__init__.py b/nuh_helper/book_page/__init__.py new file mode 100644 index 0000000..67093f1 --- /dev/null +++ b/nuh_helper/book_page/__init__.py @@ -0,0 +1,11 @@ +from nuh_helper.book_page.csv_book import CSVBook +from nuh_helper.book_page.generic import Book, Page, book_open +from nuh_helper.book_page.xlsx_book import XLSXBook + +__all__ = [ # noqa: PLE0604 + CSVBook, + XLSXBook, + Book, + Page, + book_open, +] diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py new file mode 100644 index 0000000..22988c6 --- /dev/null +++ b/nuh_helper/book_page/book.py @@ -0,0 +1,37 @@ +from abc import ABC, abstractmethod +from collections.abc import Generator as gen +from collections.abc import Iterable as iterator +from functools import cached_property +from pathlib import Path + + +class Page(ABC): + def __init__(self, name: str, page: object) -> None: + self._name = name + self._page = page + + @abstractmethod + def stream_rows(self) -> gen[tuple[int, list[str]]]: ... + + @cached_property + def name(self) -> str: + return self._name + + +class Book(ABC): + def __contains__(self, name: str) -> bool: + return name in self.keys() + + @abstractmethod + def __getitem__(self, name: str) -> gen[Page]: ... + + @abstractmethod + def keys(self) -> iterator[str]: ... + + @abstractmethod + def stream_copy(self, into: Path) -> gen[tuple[str, int, list[str]]]: ... + + def __iter__(self) -> gen[Page]: + for name in self.keys(): + with self[name] as page: + yield page diff --git a/nuh_helper/book_page/csv_book.py b/nuh_helper/book_page/csv_book.py new file mode 100644 index 0000000..ad2e029 --- /dev/null +++ b/nuh_helper/book_page/csv_book.py @@ -0,0 +1,114 @@ +import contextlib +import csv +from _collections_abc import dict_keys +from collections.abc import Generator as gen +from pathlib import Path + +from nuh_helper.book_page.book import Book, Page + + +class CSVBook(Book): + def __init__(self, path: str | Path | list[Path]) -> None: + self._page_files = {page.stem.lower(): page for page in csv_files(path)} + + def keys(self) -> dict_keys: + return self._page_files.keys() + + @contextlib.contextmanager + def __getitem__(self, name: str) -> gen[Page]: + + class CSVPage(Page): + def stream_rows(self) -> gen[tuple[int, list[str]]]: + assert isinstance(self._page, Path) + with self._page.open() as file: + for row, line in enumerate( + csv.reader( + file, + delimiter="\t" if file.name.endswith(".tsv") else ",", + ) + ): + # strip cell strings + line = [ + cell.strip() if isinstance(cell, str) else cell + for cell in line + ] + + # convert empty cells to None + line = [cell if cell else None for cell in line] + + # strip bom + if ( + row == 0 + and isinstance(line[0], str) + and line[0][:1] == "\ufeff" + ): + line[0] = line[0][1:] + + yield ( + row, + line, + ) + + assert isinstance(self._page_files[name], Path) + yield CSVPage(name, self._page_files[name]) + + def stream_copy(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: + if not isinstance(into, Path): + into = Path(into) + + if into.is_file(): + raise RuntimeError(f"can't write csvs to {into} because it's a file") + + into.mkdir(parents=True, exist_ok=True) + + for page in self: + name = page.name + file = self._page_files[name].name + with (into / file).open("w") as data: + writer = csv.writer( + data, delimiter="\t" if file.endswith(".tsv") else "," + ) + + for row, cell in page.stream_rows(): + yield name, row, cell + writer.writerow(cell) + + +def csv_files(path: str | Path | list[Path]) -> list[Path]: + """process whatever form the path takes into a list of Path objects""" + if isinstance(path, str): + path = Path(path) + + # get it to a list + if isinstance(path, Path): + if path.is_file(): + raise RuntimeError(f"need a dir or list of files, but, got a file {path=}") + path = sorted( + list(path.glob("*.csv")) + list(path.glob("*.tsv")), + key=lambda file: file.stem.lower(), + ) + + # compute dupes + collisions = sorted( + [ + page + for page in path + if len([them for them in path if them.stem.lower() == page.stem.lower()]) + != 1 + ], + key=lambda file: file.name.lower(), + ) + # raise error if there are dupes + if collisions: + raise ValueError( + f"the following csv files have name collisions\n { + '\n '.join( + [ + (file.stem.lower() + '\n ' + str(file)) + for file in collisions + ] + ) + }" + ) + + return path diff --git a/nuh_helper/book_page/generic.py b/nuh_helper/book_page/generic.py new file mode 100644 index 0000000..8ccbac7 --- /dev/null +++ b/nuh_helper/book_page/generic.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from .book import Page +from .csv_book import CSVBook +from .xlsx_book import XLSXBook + +# use this or-type instead of the "real" base class since we only have/support these two +Book = CSVBook | XLSXBook + + +def book_open(src: str | Path | list[Path]) -> Book: + if isinstance(src, str): + src = Path(src) + + if isinstance(src, Path) and src.name.endswith(".xlsx"): + return XLSXBook(src) + else: + return CSVBook(src) + + +__all__ = [ # noqa: PLE0604 + Book, + Page, + book_open, +] diff --git a/nuh_helper/book_page/xlsx_book.py b/nuh_helper/book_page/xlsx_book.py new file mode 100644 index 0000000..8c80915 --- /dev/null +++ b/nuh_helper/book_page/xlsx_book.py @@ -0,0 +1,61 @@ +import contextlib +import shutil +from collections.abc import Generator as gen +from pathlib import Path + +import openpyxl + +from .book import Book, Page + + +class XLSXBook(Book): + def __init__(self, file: str | Path, read_only: bool = True) -> None: + if isinstance(file, str): + file = Path(file) + if not file.is_file(): + raise RuntimeError(f"xlsx file not found {file}") + self._file = file + self._book = openpyxl.load_workbook(file, read_only=read_only) + + def keys(self) -> list[str]: + return [page.title for page in self._book.worksheets] + + def stream_copy(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: + if not isinstance(into, Path): + into = Path(into) + + if not into.name.endswith(".xlsx"): + into = into / self._file.name + + shutil.copy2(self._file, into) + the_copy = openpyxl.load_workbook(into) + + for name in the_copy.sheetnames: + page = the_copy[name] + + for r in range(page.max_row): + data = [page.cell(r + 1, c + 1).value for c in range(page.max_column)] + yield name, r, data + for c, val in enumerate(data): + if val != page.cell(r + 1, c + 1).value: + page.cell(r + 1, c + 1).value = val + the_copy.save(into) + + @contextlib.contextmanager + def __getitem__(self, name: str) -> gen[Page]: + + if len(name) > 31: + raise RuntimeError("excel limits you to 31 character page names") + + class XLSXPage(Page): + def stream_rows(self) -> gen[tuple[int, list[str]]]: + for row in range(self._page.max_row): + yield ( + row, + [ + self._page.cell(row + 1, col + 1).value + for col in range(self._page.max_column) + ], + ) + + yield XLSXPage(name, self._book[name]) diff --git a/nuh_helper/date_shift/mappings.py b/nuh_helper/date_shift/mappings.py index 45b6818..6d0a8bc 100644 --- a/nuh_helper/date_shift/mappings.py +++ b/nuh_helper/date_shift/mappings.py @@ -7,8 +7,8 @@ from pathlib import Path import pandas as pd -from book_page import Page +from nuh_helper.book_page import Page from nuh_helper.date_shift import _parse logger = logging.getLogger(__name__) diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index 1ddc71f..02d2d75 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -5,8 +5,8 @@ import datefinder import yaml -from book_page import Book, book_open +from nuh_helper.book_page import Book, book_open from nuh_helper.date_shift import _parse, mappings logger = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index da9350a..624e056 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,6 @@ dependencies = [ "pandas>=3.0.5", "openpyxl>=3.1.0", "datefinder>=1.0.0", - "book_page @ git+https://github.com/peterlavalle/book_page.git@main", ] [tool.hatch.metadata] allow-direct-references = true @@ -40,9 +39,6 @@ dev = [ [tool.uv] exclude-newer = "7 days" -[tool.uv.sources] -book-page = { git = "https://github.com/peterlavalle/book_page.git" } - [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/tests/data/test_book_page/book/data.xlsx b/tests/data/test_book_page/book/data.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..21cbda96bfe2ec1997ede4653c9ad4a62f968e0c GIT binary patch literal 9873 zcmeHN1yfvE8f`2Dx8M-mAwVENLU4!R4#DZ*+PF(_3+|HOG!Wd~EjTppPO#vU(VE_OE1ONaB@Dx^C z#KzhIXzie<;%W;7>AY~UvLyck4@;i~fQ8=w|M5S(0$p(<)*Wo9Jx8$@=&Gg4#4BvF zI02mxB-}aWhkMfRceyYJmrn1V91X@N0%;#dA2@wK4sg0@c^hC{WyB6^>^q`Fe(a z4Z@Qsq>Azi@%7HRHE@&EH#>Z=J z+@pQJGdhAv9zn5e8*vY2#l+{u>v{de%YobwaS<-XXkC>~0(+~?{HQRDBm8cIXmn|T z)MVtUuL)@CS#6j?X2mcCSuwBVdV6E@ZM_FSy@{Z7-sy(%GACPGx#bPvCwI30H~}uRnd0KbV1|!I_29OE zrhb1mf`}`)c*bYAzXGmBLI3D<6~umeq1+ z?Zu9ydP5TOcF_caU%&f4o{ZZEGBlYj_2{#tIKL)RRW^NXR%V!C|Ax|4&)Blnwy!AR4rYGyDH|E)Ja2$O<-Wqc;F%h{YysphS zKVi6NudoA?n4X{hW3z}>MXZZr-7Hbf3y$?p4h|D~9j{r{-15~gOet=LF({TqO~WZo z1(~#4m#(4Wd1F#)+r^gV{Oo!zCSLV7_qBsoR2gKcTeAS4hSVe`hWWON2Fq@yX&p|M zukDqVqTSRBp>$vi1QXZVwfAbdK%?)PxJ>4tD9m6GTgqldpUZ;rekjBqgVLp+dh|$5 zY#z;2)d~H5ch_lyn5J?y;6x)Xz+)^cPZ8LDuDtm4LPL0tca4Y_Mi0XazO((g8Z4w= zNi{ko&m!RmY*q>1ZtNXW%~Mx(zeO{&VUaHhE2+Zg$8FLsnZ}!>WldBB?XBVBf?8Tk zFwc%+O%Hujt#ox>gvBf$n5EGUn_54*eJn4n&ebE-DQ3> zXYuDw#B0AhNIv-6Yvv6Y2=QH{T|&q#;3mb`3Qpya;Yz=*6Yq?2*Z6w*GXC&N`>HCE zCYB!J!>pS&0s~4oaCqjR@4D^cgoU@37=Z@W;kx#HsdX=)aVw*Lh*n zeFisxpTB5-EbrdKLJrJNFswHe9!40fSAN=GaacjkxZo>`G*Hhct$?jp;x&auQphN_nj z%a3?hXces2$?d!M$-ZNd`Rjw^a_)YQlTR7MZ=2E9cdt(-W-V_ZZ?8cb9XDn|0$gut z9m3O%LrLhq1m6@!i8DAwi+B>%bH00e5rxKDJG%wCuPJ;*|I+OiecT3E^oaa+M{hJd za&QS@KKMCbiHexs2n6O}j|`gTe-{Gj!aRc;=r0mbF(3jwg@Fpe?*;SEa`2zx2?OnD zp>p!SJ(^;^nRl_Femn|j@gHlpi+NBOc;n6x&|EW*$X1RWBx`9@DByaV5_iZ|mXkP| zP0e}AKJ?-C_1PLJmE#fveFX`8!(8fcHL~udy!s`1a5YE6B|2`nWLe1zR<4m53psh? zMnTF#9wq(cPX9O+Z4yo(f0j&QPp=S&$cV|m6ay-2Ti5B~t*^3D%tF%@F`c)Fy1Y>P zuWhWBIdvkUTD*uyPY6-^Lr`L~B3UZ@%g%uhLR8&2S zN!i+o+9sW12vzTontRY-^utEFDvm^7TmUuPgZ$bttUoW=L$c+Ht~xee^QZc z;brB$%S@Vvkpp=+>TvPx>`yL&Yx;aw8^6Ja%KZ2wwH2#|K!4BfWGV#D;*4qDvcr2KM z7+QVEZU|brySl5~7j8&I_Q@w_?Cvz;j49+W13b>-r}BQ1 z-R@l~fdWVLi5pq$(em>TGKTcB`{4;+0+X72D4qLnoyVm5=u`owaJc5UvBLYbRm|PU z1qCm~;9iN_Q!o|TF>c;9)#MPPxzuiVvYDdSNY`~9JP>Q%K zT8VfAR{f4wA3BgP!eR8qT*0tcrd02ZqM-`2#{xunTqnd+fR`WGQ`r?2rg!u{Pup9y zaP1v3jlI>EiYbm(Yj0o)^T3l_$o(?)`c=Y5DjQbYIQ!+mjUhx9=Vzx&^fd%sbL%my zz9#I~%`V4Wg_(IA?v|C6So&nfNPl{JDv#Yn|g7cl$sb0($6f~ffPy$oP3{ZfCu~AG8S6@q? zpqKl+QmG@!1oJrrJ}^g+s+Qqy%4h7t%B*$vljjF1d1K{cv|R@DOd)VBy`dY@RZ>7@ z38T;BJ&RV7jg<_I^{tJXDf)b%Z7=*N5F*tzS7Nacri9%1hBgvMQ^FYd=$8u4@+4i* zeWDI2KYLk2Vu<$x?bA+JD#hY+8{r^ixEl{xh)JE%LFs&zp6-*ax<_Q>dq<8qh)_Ka zy$V5A6Ou~i>UC`AUZQ&G>*tTS&vaM6DR?2iVrUWyIEkTC@C@YLh0B!O57sw})Z$Y) z+FPhx~<0RY)DGQ1?mpl3Mw@+J%d5so|N&0H5-%hU2>xjePZhKq<$lcU7t$!G5D>$ zc*<&MX!C<$jG;3t$K(;;!&Y-|FLxB^G8UZl8HJL1QwoA|c=epJKYYZj;=7wX?Yi{5 zh|;QYB%!<{L>p(<)Bb2Z>TnGXH#&*@ll%tEpeAL zagg&sZwc`)@q*Jg$LBWRe=OmLYpW!DYu1xn%wfuT@04UDll6W4bK?ZEIafW#c>j!52YgU$|Og+QC{EAC!1N0ZD=FuW=6D?yTUk4G;W9 z%j)@WB+UWLJiHDayx;PciMEbm5yr|QUhsAM(V6QNcV7|W8v#toJ40zDOOz|xID*%7 zi|Ry07?lGd&!z1;Xi#qrFa>v}?rm>vm?JTszL2ueZ4*62HRG2YhL=@^xC6ABu{Sok zmz!K3&cn7JT~E64Su-9{NXLLVFI5v3DC81=^<)nZy^r_j&O5D-5Azs$5B(8|z&o(_ z!^2eV@_n3}hIs&k?!z@$UC;ac2dnqh1DW2Ns!RBcgjF0OoDjJ_8n+=M7eOQBRDtBs zRH=8A?^uvXe654_qC=Ti`iVI3OXV$b_;Ei0-?!a@P4FOPNPSEMkwa_E}b@cv=CP^V3Z=Foz((g$hI1ox}IM!lDuCSw9z5nQPDOe z~A&)5Cpa^kjE}p5YxUpUMfp7 zJGb&H4q@TtN!>fw9(Y7AJx25a;J!CBJX>v7}jMS;o1kG+?upThplQ zMdqS|jvs1$IV$x9cg;6=4hug@ecGfiRO$@~ScCTg!At59GXo5@W%yRBPw2yo-RScA z_d_#rnBiORT+K5v)XwX0TVy#lJgVSbHKhMn)XUTa{nvX&t}dTgnh ztxd2&NtyTCs>!lz_VfJYmsenr)x@C1mv@45ba2Z|Z)9TrtuLChgbeBHt5bH=l#DpD zchr|-hg4Z#c z8S#(IrRTM7n9#;);I4bY5y{W;R28WNTG#v#`eq}tSf-Prwjbz|(2Nu^Zp;;u;xB5! zaYcmP0J`wfa3Eox)4K0b<;??_YkzHX=k}{TFNGACglAD9zPfpT%K$p>X`Fj~G&OM> z-ami^DCyRbcjkmNN9AX}K&j?Ft`h6up+R9v{df}3aBK{3)9LDvaC<0#Ys-D^m~UGs zCZB3pNrX&PauIY~Ek_Blhm#pd_M_Rso_2W}HY~!C=G2UE7P0zLj>09!>$1ZVDTtMW zrULhotB?j)39UT*Wmsx)>@{6p=+iFKh4Zshn@(RE+7RQ7H)Rgqg&V@nD_&+BZTmHe z{f0sxioRct2iG__p)l~v;0^0VT+RDeQ|tLQj(oYMNvVi;38&~HDiL&dmZKpdgHHx; zKO63>NiGg0Pk0QfL^=Bt&hn=-5>{Dm!XA zXz}_woxpU_D_3r&WhQ2mdT4pG>U50_HpH*i0M_5S4Ify>kir*%Qe7^aTSZHfhQ8zx z9&H(PHA1f6Y7e&7O55nJHd<&Ne{JI&CPLQ}lj{j?>mq^bUUh?>8Mj-t^w+Th zglzE@jN8?1L3uH|(Y_WBf=)9`?d@fOq?2?T^0jZikaWT*@Pfy>!mB0jWkLAW2OILX zXh9|tg!3|dbVh%J(c;NYgUWI1DYtl`8InN0nX1(FrS0bpY%5s;B$2%lW#c4((!)!l zc->a60?zqtFCKL9^UoM79=G*=wFK!Pkak<{s^gIaw2c4#-Mt>K)wmGaq^d)kRJ7j< zJ;=e;5(xU)ndYckSwK6}2mTpv%$vgjr*&qe8kvbng`I&c((hoy!H@#o@SL3Co(Fe% zCtZ>^HDg{O^7#{>kwq$7wq#5QUEdSEt2x3L!mF zk^{JhSRX3x$S|e8o#r7)vk>oH#h@HIk>ZJ0q%D@jaXB#fm=i~shRm5-8cwE�=lk zMs4JdsX-25Z9_^ebSr8d@{OfUU4>ycE-A#2kk%{bv`^)7u-GjJn>~2sElSeFa+l~4 z4Ph0R{bno2S&NRfmu4k43-3c}RD9@qsO4o?s)W#U{D;MsvI~qoyByc6#4sia?$?~n z8S7`x-4|Eoet4E`ko!bYW(++TtzzP@NwQopw?*e?eKr!uEfaLVHG=>U5g;W&yK4_v z%`YAByZcuFyVzU1N~y@DmxT<7>KVvtyyWIr<_E2E+{cZR!|k|z6QL9FYARPsZlAJY zHsjs}OFY7RO`6PDMNEq0jW4Pa2KH{G+OmEA)I7U>juIsxx?e7U8a8)BLWYT0H6lqi zLB;A8NOqTsYwLM|X= zQ?Csb-*xgdUGpWuk|n_`jt(fh=x{LUD?4BZuAC;c72eVK89X5T;g;-NAk%+tn+Z_Cs9QK$G6wUTXn{MV1G5lfIwYLqy%pU zKax1|eD9BUxc|iI{?}g+Ay6(SLVc!Z|K_x*fj!Vz*}>k-+T=HC8;`b2!?UA4S!rD1 zjayobbjOUB!`C_9Xz#oMRO6V0TD{~-?sj!&LKfamV!g%*=JVLh8CKxIzPNFNm?03D z2J>9poS!*3f^oxo+v$XN&XJ2T4u5pK3@g&y(3Ph=ZDuE-?aOfHgqt4bjsKpT0OWj& ze+(xNh>4wbUX1AH|DxuL7w2{<2S!-mhV2!Yu`qf4hUrk0j7gJfIEC!0JDW95%j1N` zJ}l~NdYofbaAr>=6h`YzXw5a7Vqi|rKC2=kidZ<48!zb5Dvy8K6KcKvJ-NSkQqjpk z7r_FW;8;*&&e+CC!QRFe#A;+?5B!`*Bc)e2h_)>4f@A%LtDm{qCt%q#zV4 zL`L^H{T0y@7HVhhaw!pP$iZ@IO{E&$RZ<$LH4P+H!_&{m<5vzWv4183kC5fUB3(T) zcIrepte{V-?5A4oN0Q$DMa<4U)=t*dO5eyFttVa|PF^X&K0VtXlwxRk4MSZS?M%;l zXik-UDpA>+a^hTrn1=bX8=C}YhR) zjqWhe#~|Z8-7-OQn8f13ihdZ&f@n0ssTk|T?(sm-TBxR;$6jKkc>Jh~0I6U3)#Z(ytT!9E0+B%v5hf##M^!d992JHFa{J zF&^y%OY($^I9+V*#XZ@n)#QV)uF{=MS`fo*4}y_9$tt6=mh_(;(M%CWyQuw1TB zJ{iiuH&A|J{+WRWwzmH#Ae4Fk9%->cHuG$#0Y`8bWT=b8^=4S8zQu-;+-GHmFfmTO zzOlO+1hy-M1m ze|ZMn)NEq5Q2xe39`-i+H&B(wczw*N+5sxhLJpM!LZm`}t#5x6P)qK{o-%h4ubt|WyCRAN)ve_5iGAl^^zeOb={VJd&$-6>$yE!&f3BZ zVmP@JkHmtuljJg(Mf3sxq>b%IBo7VSUDKP=t&ceSVJ`RR-ZPT0*ZVuNrdr4O)bFOa z_cM1sF9vDrdn{gDD;$^yYviG%SQ`-)G*<2G+en&?@rJ$9Den_x)%7heJFHGJgg5>(PS$1pb`Up*ixWg9g6_ z{<_`zM`#DMcl%}Y_1ECPZqfV^3IGfu{}%jTHf(;y`PH!h0|^1zJ^nZQ{#TS=ozXu~ z)}ZVD4=BI+q`#v4>QViHl7amj%CC;quK>UHzJCA&5&Z`6M<@Ji=&votAEDZ$zlZ+X zaQuq!*V_C?JOE()9QygSQvVwMS7H5g_!jw}!v8I^3Ni>#j{H2!f&%D)3UxKr&u{+% Dd4)@z literal 0 HcmV?d00001 diff --git a/tests/data/test_book_page/book/data_3x4.csv b/tests/data/test_book_page/book/data_3x4.csv new file mode 100644 index 0000000..e09f8ab --- /dev/null +++ b/tests/data/test_book_page/book/data_3x4.csv @@ -0,0 +1,4 @@ +id,name,age,score +1,Alice,28,87.5 +2,Bob,34,92.1 +3,Carol,22,78.3 diff --git a/tests/data/test_book_page/book/data_3x6.tsv b/tests/data/test_book_page/book/data_3x6.tsv new file mode 100644 index 0000000..0ae2097 --- /dev/null +++ b/tests/data/test_book_page/book/data_3x6.tsv @@ -0,0 +1,4 @@ +id name age score city active +1 Alice 28 87.5 London True +2 Bob 34 92.1 Paris False +3 Carol 22 78.3 Tokyo True diff --git a/tests/data/test_book_page/page-desc.csv b/tests/data/test_book_page/page-desc.csv new file mode 100644 index 0000000..f50a328 --- /dev/null +++ b/tests/data/test_book_page/page-desc.csv @@ -0,0 +1,7 @@ +foo,pid,glitter +bar,nuh71,road +shoe,nuh06,can't recall the date but on 12/11/2001 they had an itchy tummy +agreed on 21st jul 2017,nuh23,grip +farm,nuh67,grim +cake,nuh27,cheese +clip,nuh65,jkl diff --git a/tests/test_book_page.py b/tests/test_book_page.py new file mode 100644 index 0000000..3c302d4 --- /dev/null +++ b/tests/test_book_page.py @@ -0,0 +1,278 @@ +import textwrap +from pathlib import Path +from typing import Literal, Never + +import pytest + +from nuh_helper.book_page import Book, CSVBook, XLSXBook, book_open + +test_data = ( + # i feel like test data should be in a folder following this pattern + Path(__file__).parent / "data" / __name__ +) +test_book = ( + # keep the .xlsx and .csv in their own folder + test_data / "book" +) + +data_3x4 = [ + ["id", "name", "age", "score"], + ["1", "Alice", "28", "87.5"], + ["2", "Bob", "34", "92.1"], + ["3", "Carol", "22", "78.3"], +] +data_3x6 = [ + ["id", "name", "age", "score", "city", "active"], + ["1", "Alice", "28", "87.5", "London", "True"], + ["2", "Bob", "34", "92.1", "Paris", "False"], + ["3", "Carol", "22", "78.3", "Tokyo", "True"], +] + + +def test_csv_name_collision() -> None: + with pytest.raises(ValueError) as error: + CSVBook([test_book / "data_3x4.csv", test_book / "data_3x4.tsv"]) + assert ( + textwrap.dedent( + f""" + the following csv files have name collisions + data_3x4 + {test_book}/data_3x4.csv + data_3x4 + {test_book}/data_3x4.tsv + """ + ).strip() + == str(error.value).strip() + ) + + +@pytest.mark.parametrize( + "kind, path", + [ + (CSVBook, test_book), + (XLSXBook, test_book / "data.xlsx"), + ], +) +@pytest.mark.parametrize( + "to_str", + [False, True], +) +@pytest.mark.parametrize( + "generic", + [False, True], +) +@pytest.mark.parametrize( + "name, data", + [ + ("data_3x4", data_3x4), + ("data_3x6", data_3x6), + ], +) +def test_read_data( + kind: CSVBook | XLSXBook, + path: Path, + to_str: bool, + generic: bool, + name: Literal["data_3x4", "data_3x6"], + data: list[list[str]], +) -> None: + + # arrange + book: Book + + # act + if generic: + book = book_open(str(path) if to_str else path) + else: + book = kind(str(path) if to_str else path) + + # assert + assert isinstance(book, kind) + assert name in book + with book[name] as page: + copy = [] + for _, row in page.stream_rows(): + copy.append([str(c) for c in row]) + + assert data == copy + + +@pytest.mark.parametrize( + "kind, path", + [ + (CSVBook, test_book), + (XLSXBook, test_book / "data.xlsx"), + ], +) +@pytest.mark.parametrize( + "src_str", + [False, True], +) +@pytest.mark.parametrize( + "out_str", + [False, True], +) +@pytest.mark.parametrize( + "end_str", + [False, True], +) +@pytest.mark.parametrize( + "generic_open", + [False, True], +) +@pytest.mark.parametrize( + "name, data", + [ + ("data_3x4", data_3x4), + ("data_3x6", data_3x6), + ], +) +def test_copy_book( + src_str: bool, + out_str: bool, + end_str: bool, + kind: type, + path: Path, + tmp_path: Path, + generic_open: bool, + name: str, + data: list[list[str]], +) -> None: + # arrange + source: Book + changed_data = [ + [f"edit[{r},{c}]{data[r][c]}" for c in range(len(data[0]))] + for r in range(len(data)) + ] + src = str(path) if src_str else path + out = str(tmp_path) if out_str else tmp_path + end: str | Path = (tmp_path / path.name) if path.is_file() else tmp_path + if end_str: + end = str(end) + + # act + source = kind(src) + + for page, row, cells in source.stream_copy(out): + assert page in ("data_3x4", "data_3x6"), f"bad page name {page}" + for col, val in enumerate(cells): + cells[col] = f"edit[{row},{col}]{val}" + + # assert + target: Book + target = book_open(end) if generic_open else kind(end) + + # assert + assert name in source + assert name in target + assert isinstance(source, kind) + assert isinstance(target, kind) + + with target[name] as page: + copy = [] + for row, cell in page.stream_rows(): + assert len(copy) == row + copy.append(cell) + assert copy == changed_data + + +def test_stream_csv_rows() -> None: + book = CSVBook([test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]) + + copy = [] + with book["data_3x4"] as page: + for _, row in page.stream_rows(): + copy.append(row.copy()) + + assert copy == data_3x4 + + +def test_csv_stream_wont_overwrite_file(tmp_path: Path) -> None: + book = CSVBook([test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]) + into = tmp_path / "foo" + into.write_text("foobar") + + with pytest.raises(RuntimeError) as error: + for _ in book.stream_copy(into): + pytest.fail("this should have raised an exception") + + assert str(error.value) == f"can't write csvs to {into} because it's a file" + + +@pytest.mark.parametrize( + "kind, path", + [ + (CSVBook, [test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]), + (XLSXBook, test_book / "data.xlsx"), + ], +) +def test_stream_rows(kind: type, path: Path) -> None: + book = kind(path) + + copy = {} + + # scan each row + for page in book: + copy[page.name] = [] + for idx, row in page.stream_rows(): + assert idx == len(copy[page.name]) + copy[page.name].append([str(c) for c in row]) + + assert copy == { + "data_3x4": data_3x4, + "data_3x6": data_3x6, + } + + +def test_non_file(tmp_path: Path) -> None: + with pytest.raises(RuntimeError) as error: + XLSXBook(tmp_path / "foo.xlsx") + assert f"xlsx file not found {tmp_path}/foo.xlsx" == str(error.value) + with pytest.raises(RuntimeError) as error: + for _ in XLSXBook(tmp_path / "bar.xlsx").stream_copy(tmp_path / "foo.xlsx"): + pytest.fail("this should have raised an exception") + assert f"xlsx file not found {tmp_path}/bar.xlsx" == str(error.value) + + +def test_csv_file() -> None: + path = test_book / "data.xlsx" + + with pytest.raises(RuntimeError) as error: + CSVBook(path) + assert f"need a dir or list of files, but, got a file {path=}" == str(error.value) + + +def test_xlsx_name_cap() -> Never: + book = XLSXBook(test_book / "data.xlsx") + long_name = "name which exceeds 31 characters" + with ( + pytest.raises(RuntimeError) as error, + book[long_name] as page, + ): + raise RuntimeError(f"{long_name=} should have failed {page.name=}") + assert str(error.value) == "excel limits you to 31 character page names" + + +def test_bom_is_removed() -> None: + book = CSVBook([test_data / "page-desc.csv"]) + + obtained = [] + with book["page-desc"] as page: + for idx, row in page.stream_rows(): + assert idx == len(obtained) + obtained.append(row.copy()) + + expected = [ + ["foo", "pid", "glitter"], + ["bar", "nuh71", "road"], + [ + "shoe", + "nuh06", + "can't recall the date but on 12/11/2001 they had an itchy tummy", + ], + ["agreed on 21st jul 2017", "nuh23", "grip"], + ["farm", "nuh67", "grim"], + ["cake", "nuh27", "cheese"], + ["clip", "nuh65", "jkl"], + ] + assert obtained == expected diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index eacc938..bcccd1b 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -1,9 +1,8 @@ from pathlib import Path -import book_page import pytest -from nuh_helper import shift_excel_dates_inplace +from nuh_helper import book_page, shift_excel_dates_inplace from nuh_helper.date_shift import HiddenDate, PatientColumnMissing, ShiftFoundNonDate test_data = Path(__file__).parent / "data/shift_ignore" diff --git a/uv.lock b/uv.lock index b906311..a0479d1 100644 --- a/uv.lock +++ b/uv.lock @@ -14,14 +14,6 @@ resolution-markers = [ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P7D" -[[package]] -name = "book-page" -version = "0.1.0" -source = { git = "https://github.com/peterlavalle/book_page.git#c8246740221cc72cb195f815dd3e97a41e320387" } -dependencies = [ - { name = "openpyxl" }, -] - [[package]] name = "cfgv" version = "3.5.0" @@ -202,7 +194,6 @@ wheels = [ name = "nuh-helper" source = { editable = "." } dependencies = [ - { name = "book-page" }, { name = "datefinder" }, { name = "openpyxl" }, { name = "pandas" }, @@ -222,7 +213,6 @@ dev = [ [package.metadata] requires-dist = [ - { name = "book-page", git = "https://github.com/peterlavalle/book_page.git" }, { name = "datefinder", specifier = ">=1.0.0" }, { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pandas", specifier = ">=3.0.5" }, From 8dd5540a3aea8352e849c8dfda31280df6f0e659 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 14:58:46 +0100 Subject: [PATCH 55/82] raise different exception --- nuh_helper/book_page/xlsx_book.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuh_helper/book_page/xlsx_book.py b/nuh_helper/book_page/xlsx_book.py index 8c80915..2ed363f 100644 --- a/nuh_helper/book_page/xlsx_book.py +++ b/nuh_helper/book_page/xlsx_book.py @@ -45,7 +45,7 @@ def stream_copy(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: def __getitem__(self, name: str) -> gen[Page]: if len(name) > 31: - raise RuntimeError("excel limits you to 31 character page names") + raise KeyError("excel limits you to 31 character page names") class XLSXPage(Page): def stream_rows(self) -> gen[tuple[int, list[str]]]: From cde02e724fe28ff300d17c6163cda1b3f8e0543e Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 15:41:13 +0100 Subject: [PATCH 56/82] refactors the shift_delta generation to be simpler and move checking into main --- nuh_helper/date_shift/__init__.py | 4 +- nuh_helper/date_shift/mappings.py | 77 +++++++-------------------- nuh_helper/date_shift/stream_shift.py | 57 +++++++++++++++++--- tests/test_book_page.py | 6 +-- 4 files changed, 74 insertions(+), 70 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 93bf03c..a7b2d72 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -14,8 +14,6 @@ from nuh_helper.date_shift import _excel, _parse, mappings from nuh_helper.date_shift.mappings import ( # noqa: E402 - PatientColumnMissing, - UnknownPatient, generate_shift_mappings, load_shift_mappings, ) @@ -26,8 +24,10 @@ ExtraPage, HiddenDate, PageMissing, + PatientColumnMissing, ShiftFoundNonDate, TextColumnsMissing, + UnknownPatient, shift_excel_dates_inplace, ) diff --git a/nuh_helper/date_shift/mappings.py b/nuh_helper/date_shift/mappings.py index 6d0a8bc..7bf628b 100644 --- a/nuh_helper/date_shift/mappings.py +++ b/nuh_helper/date_shift/mappings.py @@ -3,12 +3,10 @@ import csv import logging import random -import time from pathlib import Path import pandas as pd -from nuh_helper.book_page import Page from nuh_helper.date_shift import _parse logger = logging.getLogger(__name__) @@ -57,65 +55,29 @@ def load_shift_mappings(csv_path: str) -> pd.DataFrame: return df -class PatientColumnMissing(Exception): - """raised when the patient column is missing from any sheet""" - - def __init__(self, page_name: str, column_name: str) -> None: - message = f"The patient {column_name=} is not present on {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_name = column_name - - -class UnknownPatient(Exception): - def __init__(self, page: str, id: str) -> None: - message = f"Unknown {id=} on {page=}" - super().__init__(message) - self._message = message - - def patient_shift_deltas( - page: Page, + patient_ids: list[str], src_linking_table: str | Path, out_linking_table: str | Path, - patient_id_col_name: str, - patient_header_row: int, - patient_skip_rows: list[int] | None, seed: int | None, min_shift_days: int, max_shift_days: int, ) -> dict[str, pd.Timedelta]: + """: + Generate random shift mappings for patient IDs, reusing known patients and saving + all to a file. - logger.info("patient_shift_deltas() ; preparing per-patient shifts") - - # find the patient id index - patient_id_col: None | int = None - patient_ids = [] - start = time.time() - - for row, data in page.stream_rows(): - if row < patient_header_row or row in patient_skip_rows: - continue - elif row == patient_header_row: - if patient_id_col_name not in data: - raise PatientColumnMissing(page.name, patient_id_col_name) - patient_id_col = data.index(patient_id_col_name) - else: - patient_id = data[patient_id_col] - if not patient_id: - continue - patient_id = _parse._normalize_patient_id(patient_id.strip()) - if patient_id in patient_ids: - continue - patient_ids.append(patient_id) - if (len(patient_ids) % 40) == 0: - seconds = time.time() - start - logger.info( - f"patient_shift_deltas() ; so far {len(patient_ids)=} in {seconds=}" - ) - logger.info(f"patient_shift_deltas() ; Found {len(patient_ids)=} in {page.name=}") + Args: + patient_ids: List of patient IDs to generate shifts for. + src_linking_table: where to load prior values from + out_linking_table: where to save teh full set to + seed: Optional random seed for reproducibility. + min_shift_days: Minimum number of days to shift (default: -15). + max_shift_days: Maximum number of days to shift (default: 15). + + Returns: + dictionary mapping 'patient_id' to a pd.Timedelta 'shift_days' which we use. + """ # load the old ids (should we remove IDs that are "gone"?) if isinstance(src_linking_table, str): @@ -130,14 +92,11 @@ def patient_shift_deltas( if row and row[1] and row[1].strip() != "shift_days" # if row and row[0] and row[0].strip() != "patient_id" } - logger.info( - f"patient_shift_deltas() ; Loaded {len(old_mappings)=} in {src_linking_table =}" - ) - - # create the list of all mappings if seed: random.seed(seed) + + # create the list of all mappings all_mappings: dict[str, int] = { id: ( old_mappings[id] @@ -152,11 +111,11 @@ def patient_shift_deltas( ) # store all mappings in the out file + out_linking_table.parent.mkdir(parents=True, exist_ok=True) with open(out_linking_table, "w") as file: csv.writer(file).writerows( [["patient_id", "shift_days"]] + [[key, all_mappings[key]] for key in sorted(all_mappings)] ) - logger.info(f"patient_shift_deltas() ; Saved mappings {out_linking_table}") return {k: pd.Timedelta(days=all_mappings[k]) for k in all_mappings} diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index 02d2d75..f0b420e 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -12,6 +12,25 @@ logger = logging.getLogger(__name__) +class PatientColumnMissing(Exception): + """raised when the patient column is missing from any sheet""" + + def __init__(self, page_name: str, column_name: str) -> None: + message = f"The patient {column_name=} is not present on {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_name = column_name + + +class UnknownPatient(Exception): + def __init__(self, page: str, id: str) -> None: + message = f"Unknown {id=} on {page=}" + super().__init__(message) + self._message = message + + class ShiftFoundNonDate(Exception): def __init__( self, page: str, row: int, col: int, col_name: str, value: str @@ -183,7 +202,16 @@ def shift_excel_dates_inplace( book: Book = book_open(input_file) logger.info(f"Book has {type(book)=}") - # TODO; it'd be cool to "normalize" the linking_table_path/linking_table_output here + + if not linking_table_path: + linking_table_path = None + elif not isinstance(linking_table_path, Path): + linking_table_path = Path(linking_table_path) + + if not linking_table_output: + linking_table_output = None + elif not isinstance(linking_table_output, Path): + linking_table_output = Path(linking_table_output) # get the shift_ignore: dict[str, dict[str, list[str]]] from shift_ignore_yaml: Path if not shift_ignore_yaml: @@ -220,13 +248,30 @@ def shift_excel_dates_inplace( raise PageMissing(patient_sheet) with book[patient_sheet] as page: + patient_header_row = sheet_configs[patient_sheet].get("header_row", 0) + patient_skip_rows = sheet_configs[patient_sheet].get("skip_rows", []) + patient_id_col_name = sheet_configs[patient_sheet]["patient_id_col"] + patient_ids = [] + for row, data in page.stream_rows(): + if row < patient_header_row or row in patient_skip_rows: + continue + elif row == patient_header_row: + if patient_id_col_name not in data: + raise PatientColumnMissing(page.name, patient_id_col_name) + patient_id_col = data.index(patient_id_col_name) + else: + patient_id = data[patient_id_col] + if not patient_id: + continue + patient_id = _parse._normalize_patient_id(patient_id.strip()) + if patient_id in patient_ids: + continue + patient_ids.append(patient_id) + shift_deltas = mappings.patient_shift_deltas( - page, + patient_ids, linking_table_path, linking_table_output or "shift_mappings.csv", - sheet_configs[patient_sheet]["patient_id_col"], - sheet_configs[patient_sheet].get("header_row", 0), - sheet_configs[patient_sheet].get("skip_rows", []), seed, min_shift_days, max_shift_days, @@ -450,7 +495,7 @@ def shift_excel_dates_inplace( shift_delta = shift_deltas.get(pid) if shift_delta is None: # all patients need to have a shift value - raise mappings.UnknownPatient(sheet_name, pid) + raise UnknownPatient(sheet_name, pid) # now scan each column # ... even when page has no date_columns; still check for hidden dates diff --git a/tests/test_book_page.py b/tests/test_book_page.py index 3c302d4..519dba7 100644 --- a/tests/test_book_page.py +++ b/tests/test_book_page.py @@ -246,11 +246,11 @@ def test_xlsx_name_cap() -> Never: book = XLSXBook(test_book / "data.xlsx") long_name = "name which exceeds 31 characters" with ( - pytest.raises(RuntimeError) as error, + pytest.raises(KeyError) as error, book[long_name] as page, ): - raise RuntimeError(f"{long_name=} should have failed {page.name=}") - assert str(error.value) == "excel limits you to 31 character page names" + pytest.fail(f"{long_name=} should have failed {page.name=}") + assert str(error.value) == "'excel limits you to 31 character page names'" def test_bom_is_removed() -> None: From 7266555753bd8b69b842ac3fb229d1b0b47cc591 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 16:02:02 +0100 Subject: [PATCH 57/82] try to separate the sheet/row checks to be a standalone thing --- nuh_helper/date_shift/stream_shift.py | 146 ++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index f0b420e..54dab28 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -296,7 +296,9 @@ def shift_excel_dates_inplace( # - no option to add/remove rows is available # - no option to jump/seek through rows is available # - ... which is why this acts a bit weird + steate_tool = row_state(sheet_configs) for sheet_name, row_idx, row_cells in book.stream_copy(output_file): + steate_tool.in_body(sheet_name, row_idx, row_cells) ##### # Part-1; check if we've reached a different sheet ## @@ -588,3 +590,147 @@ def shift_excel_dates_inplace( f"Shifting {last_name=} processed {last_row} rows (and that's the last page)" ) logger.info("Output written to '%s'", output_file) + + +class row_state: + def __init__(self, sheet_configs: dict[str, dict]) -> None: + self._sheet_configs = sheet_configs + self._last_name: None | str = None + self._last_row: None | int = None + self._header_row: None | int = None + self._skip_rows: None | list[int] = None + + self._col_names: None | list[str] = None + self._patient_id_idx: None | int = None + self._patient_id_col: None | str = None + self._date_columns: None | list[str] = None + self._text_columns: None | list[str] = None + self._sheet_config: None | dict | str = None + + def in_body(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bool: + # check that it's in the configs + if sheet_name not in self._sheet_configs: + raise ExtraPage(sheet_name) + + if row_idx != 0: + # SAN/safety check in case something tries to skip showing us rows + if sheet_name != self._last_name: + raise RuntimeError(f"didn't see row0 from {sheet_name=}") + self._last_row = row_idx + + if row_idx == 0: + self._last_name = sheet_name + self._last_row = row_idx + + self._sheet_config: dict | str = self._sheet_configs[sheet_name] + + # do nothing to the skipped ones + if self._sheet_config == "skip": + return False + + # grab the configuration + self.pid_column: str = cast( + str, self._sheet_config["patient_id_col"] + ).strip() + self._date_columns: list[str] = [ + col.strip() for col in self._sheet_config["date_columns"] + ] + self._text_columns: list[str] = [ + col.strip() for col in self._sheet_config["text_columns"] + ] + + self._header_row: int = cast(int, self._sheet_config.get("header_row", 0)) + self._skip_rows: list[int] = self._sheet_config.get( + "skip_rows_after_header", [] + ) + + ### + # do some checks of the configuration + ## + + # check for holes in skip_rows (possible issue) + if self._skip_rows: + missing_skipped = [ + not_skipped + 1 + for not_skipped in range(self._header_row, max(self._skip_rows)) + if (not_skipped + 1) not in self._skip_rows + ] + if missing_skipped: + raise RuntimeError( + f"{sheet_name=} and {missing_skipped=} but" + + f" {self._header_row=} and {self._skip_rows=}" + ) + + # check if the config hard-codes patient data to skip + if "shift_exceptions" in self._sheet_configs: + raise RuntimeError( + f"shift_exceptions was removed, update {sheet_name=}" + ) + + # check patient_id_col isn't re-used as other column types + if self.pid_column in self._date_columns: + raise ValueError( + f"{self.pid_column=} shouldn't be in date_columns of {sheet_name=}" + ) + if self.pid_column in self._text_columns: + raise ValueError( + f"{self.pid_column=} shouldn't be in text_columns of {sheet_name=}" + ) + + # check that column names don't appear in both + col_names = [col for col in self._date_columns if col in self._text_columns] + if col_names: + raise ValueError( + f"{sheet_name=} has the some columns in both date and text " + + str(col_names) + ) + + self._col_names = None + self._patient_id_idx = None + + if self._sheet_config == "skip": + return False + + if row_idx < self._header_row: + return False + + if row_idx == self._header_row: + raise Exception("compute columns and indcides") + + if row_idx in self._skip_rows: + return False + + # get the patient id for this row + pid = row_cells[self._patient_id_idx] + + # check for date in the cell_value + for found in datefinder.find_dates(str(pid)): + raise HiddenDate( + sheet_name, + row_idx, + self._patient_id_idx, + self._patient_id_col, + pid, + found, + ) + + pid = _parse._normalize_patient_id(pid) + if pid is None: + # if the pid is None; the rest of the row should be None as well + non_blank = [ + cell + for cell in row_cells + # keep the cells that aren't blank + if cell is not None and str(cell).strip() != "" + ] + if non_blank: + raise ValueError( + f"{row_idx=} has no pid, should be blank but has {non_blank}" + ) + + # we've completed the checks for a None pid + # ... so we need to skip the rest of this loop + return False + + # it's a valid pid - return it + return pid From bca28c5bf2689e1dbc9b32c04af5c3b9025b58ea Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 16:58:28 +0100 Subject: [PATCH 58/82] refactored the loop control and parameter lookups into a scanner class --- nuh_helper/date_shift/stream_shift.py | 462 ++++++------------ .../{ => test_missing_id}/missing-id.xlsx | Bin tests/test_missing_id.py | 7 +- tests/test_shift_ignore.py | 4 - 4 files changed, 157 insertions(+), 316 deletions(-) rename tests/data/{ => test_missing_id}/missing-id.xlsx (100%) diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index 54dab28..fda9f21 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -248,28 +248,16 @@ def shift_excel_dates_inplace( raise PageMissing(patient_sheet) with book[patient_sheet] as page: - patient_header_row = sheet_configs[patient_sheet].get("header_row", 0) - patient_skip_rows = sheet_configs[patient_sheet].get("skip_rows", []) - patient_id_col_name = sheet_configs[patient_sheet]["patient_id_col"] - patient_ids = [] - for row, data in page.stream_rows(): - if row < patient_header_row or row in patient_skip_rows: - continue - elif row == patient_header_row: - if patient_id_col_name not in data: - raise PatientColumnMissing(page.name, patient_id_col_name) - patient_id_col = data.index(patient_id_col_name) - else: - patient_id = data[patient_id_col] - if not patient_id: - continue - patient_id = _parse._normalize_patient_id(patient_id.strip()) - if patient_id in patient_ids: - continue - patient_ids.append(patient_id) - + scanner = row_scanner(sheet_configs) shift_deltas = mappings.patient_shift_deltas( - patient_ids, + [ + pid + for pid in [ + scanner.patient_id(page.name, row, data) + for row, data in page.stream_rows() + ] + if pid + ], linking_table_path, linking_table_output or "shift_mappings.csv", seed, @@ -283,10 +271,6 @@ def shift_excel_dates_inplace( if sheet_name not in book: raise PageMissing(sheet_name) - last_name = None - last_row = None - skipping_sheet_name = "" - ## # this calls the `.stream_copy()` method to copy from one book to another # WARNING; this works like a callback rather than a loop; @@ -296,303 +280,110 @@ def shift_excel_dates_inplace( # - no option to add/remove rows is available # - no option to jump/seek through rows is available # - ... which is why this acts a bit weird - steate_tool = row_state(sheet_configs) + scanner = row_scanner(sheet_configs) for sheet_name, row_idx, row_cells in book.stream_copy(output_file): - steate_tool.in_body(sheet_name, row_idx, row_cells) - ##### - # Part-1; check if we've reached a different sheet - ## - assert isinstance(sheet_name, str) - if row_idx != 0: - # SAN/safety check in case something tries to skip showing us rows - if sheet_name != last_name: - raise RuntimeError(f"didn't see row0 from {sheet_name=}") - last_row = row_idx - else: - if last_name and last_row: - logger.info(f"Shifting {last_name=} processed {last_row} rows") - last_name = sheet_name - last_row = row_idx - - # check that it's in the configs - if sheet_name not in sheet_configs: - raise ExtraPage(sheet_name) - - sheet_config: dict | str = sheet_configs[sheet_name] - - # do nothing to the skipped ones - if sheet_config == "skip": - skipping_sheet_name = sheet_name - continue - - # grab the configuration - patient_id_col: str = cast(str, sheet_config["patient_id_col"]).strip() - date_columns: list[str] = [ - col.strip() for col in sheet_config["date_columns"] - ] - text_columns: list[str] = [ - col.strip() for col in sheet_config["text_columns"] - ] - - header_row: int = cast(int, sheet_config.get("header_row", 0)) - skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) - - ### - # do some checks of the configuration - ## - - # check for holes in skip_rows (possible issue) - if skip_rows: - missing_skipped = [ - not_skipped + 1 - for not_skipped in range(header_row, max(skip_rows)) - if (not_skipped + 1) not in skip_rows - ] - if missing_skipped: - raise RuntimeError( - f"{sheet_name=} and {missing_skipped=} but" - + f" {header_row=} and {skip_rows=}" - ) - - # check if the config hard-codes patient data to skip - if "shift_exceptions" in sheet_config: - raise RuntimeError( - f"shift_exceptions was removed, update {sheet_name=}" - ) - - # check patient_id_col isn't re-used as other column types - if patient_id_col in date_columns: - raise ValueError( - f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" - ) - if patient_id_col in text_columns: - raise ValueError( - f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" - ) - - # check that column names don't appear in both - col_names = [col for col in date_columns if col in text_columns] - if col_names: - raise ValueError( - f"{sheet_name=} has the some columns in both date and text " - + str(col_names) - ) - - col_names = None - patient_id_idx = None - - ##### - # Part-2; skip skipped sheets, and, skip skipped rows - ## - - # do nothing to the skipped sheets - if sheet_config == "skip": - # "safety" check so that we only skip rows with the correct name - if skipping_sheet_name != sheet_name: - raise RuntimeError("this shouldn't happen") - continue - logger.info(f"Shifting {sheet_name=}") - - # skip pre-header and skip rows - if (row_idx < header_row) or (row_idx in skip_rows): - # there's an earlier check to find "holes" in the skip-rows + pid = scanner.patient_id(sheet_name, row_idx, row_cells) + if not pid: continue + # determine how many days to shift + shift_delta = shift_deltas.get(pid) + if shift_delta is None: + # all patients need to have a shift value + raise UnknownPatient(sheet_name, pid) + + # now scan each column + # ... even when page has no date_columns; still check for hidden dates + # for col_idx in range(len(row_cells)): + for col_idx, cell_value in enumerate(row_cells): + # skip the patient id column (it was already checked anyway) + if col_idx == scanner._patient_id_idx: + continue - ##### - # Part-3; handle the header rows - ## - if row_idx == header_row: - # get the real column names now - col_names = row_cells - - # find patient_id_idx and normalize the column name list - for col_idx in range(len(col_names)): - col_name = col_names[col_idx] - - # if the name is None ... leave it as that (it's fine) - if col_name is None: - continue - - # strip the name - col_name = col_name.strip() - - # if the name is now blank; store None as the column name - if col_name == "": - col_names[col_idx] = None - continue - - # update it to just be the stripped version - col_names[col_idx] = col_name + # get the cell value. replace it with None if it's just whitespace + if isinstance(cell_value, str): + cell_value = cell_value.strip() + if not cell_value: + cell_value = None - # check the config to see if we know what to do with this column - if col_name not in ([patient_id_col] + date_columns + text_columns): - raise ExtraColumn(sheet_name, col_name) + # so if the cell is empty; skip the rest of the checks for this cell + # ... we go tot he next column in the row + if cell_value is None: + row_cells[col_idx] = None + continue - # select the patient id - if col_name == patient_id_col: - patient_id_idx = col_idx + # if there's no column name; this column of the row should be None + # ... and we win't reach this line if the cell_value was None + col_name = scanner._col_names[col_idx] + if col_name is None: + raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) - # check to be sure we found the patient_id_col/patient_id_idx - if patient_id_idx is None: - raise mappings.PatientColumnMissing(sheet_name, patient_id_col) + # skip values in shift_ignore - missing = [col for col in text_columns if col not in col_names] - if missing: - raise TextColumnsMissing(sheet_name, missing) - missing = [col for col in date_columns if col not in col_names] - if missing: - raise DateColumnsMissing(sheet_name, missing) + if sheet_name not in shift_ignore: + shift_ignore[sheet_name] = {} - continue + if col_name not in shift_ignore[sheet_name]: + shift_ignore[sheet_name][col_name] = {None, ""} - ##### - # Part-4; process normal columns - ## - if col_names is None: - raise RuntimeError("does the sheet skip all the skip rows?") - else: - # code used to be indented like this - - ### - # process each row of the workbook - ## - - # write a log message every 40 rows (show we're not crashed) - if (row_idx % 40) == 0: - logger.info(f"Shifting {sheet_name=} up to row {row_idx}") - - # get the patient id for this row - pid = row_cells[patient_id_idx] - - # check for date in the cell_value - for found in datefinder.find_dates(str(pid)): - raise HiddenDate( - sheet_name, - row_idx, - patient_id_idx, - patient_id_col, - pid, - found, - ) + if cell_value in shift_ignore[sheet_name][col_name]: + continue - pid = _parse._normalize_patient_id(pid) - if pid is None: - # if the pid is None; the rest of the row should be None as well - non_blank = [ - cell - for cell in row_cells - # keep the cells that aren't blank - if cell is not None and str(cell).strip() != "" - ] - if non_blank: - raise ValueError( - f"{row_idx=} has no pid, should be blank but has {non_blank}" + # for text columns, we now *just* need to check for a hidden date + if col_name in scanner._text_columns: + # check for dates in non-date columns + for found in datefinder.find_dates(str(cell_value)): + logger.debug( + f"found hidden date {sheet_name=}/{col_name=}:{cell_value}" ) - - # we've completed the checks for a None pid - # ... so we need to skip the rest of this loop + ignored = shift_ignore[sheet_name][col_name] + logger.debug( + f"shift_ignore[{sheet_name=}][{col_name=}]= {len(ignored)}" + ) + for ig in ignored: + logger.debug(f"- {ig}") + + raise HiddenDate( + sheet_name, + row_idx, + col_idx, + col_name, + cell_value, + found, + ) + row_cells[col_idx] = cell_value continue - # determine how many days to shift - shift_delta = shift_deltas.get(pid) - if shift_delta is None: - # all patients need to have a shift value - raise UnknownPatient(sheet_name, pid) - - # now scan each column - # ... even when page has no date_columns; still check for hidden dates - for col_idx in range(len(row_cells)): - # skip the patient id column (it was already checked anyway) - if col_idx == patient_id_idx: - continue - - # get the cell value. replace it with None if it's just whitespace - cell_value = row_cells[col_idx] - if isinstance(cell_value, str): - cell_value = cell_value.strip() - if not cell_value: - cell_value = None - - # so if the cell is empty; skip the rest of the checks for this cell - # ... we go tot he next column in the row - if cell_value is None: - row_cells[col_idx] = None - continue - - # if there's no column name; this column of the row should be None - # ... and we win't reach this line if the cell_value was None - col_name = col_names[col_idx] - if col_name is None: - raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) - - # skip values in shift_ignore - if "shift_ignore" in sheet_config: - raise RuntimeError("move shift_ignore from sheet_configs to a file") + # we've already checked this (sort of) + # assert col_name in date_columns - if sheet_name not in shift_ignore: - shift_ignore[sheet_name] = {} - - if col_name not in shift_ignore[sheet_name]: - shift_ignore[sheet_name][col_name] = {None, ""} - - if cell_value in shift_ignore[sheet_name][col_name]: - continue - - # for text columns, we now *just* need to check for a hidden date - if col_name in text_columns: - # check for dates in non-date columns - for found in datefinder.find_dates(str(cell_value)): - logger.debug( - f"found hidden date {sheet_name=}/{col_name=}:{cell_value}" - ) - ignored = shift_ignore[sheet_name][col_name] - logger.debug( - f"shift_ignore[{sheet_name=}][{col_name=}]= {len(ignored)}" - ) - for ig in ignored: - logger.debug(f"- {ig}") - - raise HiddenDate( - sheet_name, - row_idx, - col_idx, - col_name, - cell_value, - found, - ) - row_cells[col_idx] = cell_value - continue - - # we've already checked this (sort of) - # assert col_name in date_columns - - # cell_value can be str | datetime | date - # ... so parsed might not always succeed - parsed = _parse._parse_date_value(cell_value) - if parsed is None: - # raise an error; we already handled values we should pass as-is - raise ShiftFoundNonDate( - sheet_name, row_idx, col_idx, col_name, cell_value - ) + # cell_value can be str | datetime | date + # ... so parsed might not always succeed + parsed = _parse._parse_date_value(cell_value) + if parsed is None: + # raise an error; we already handled values we should pass as-is + raise ShiftFoundNonDate( + sheet_name, row_idx, col_idx, col_name, cell_value + ) - # perform the actual shifting - shifted = parsed + shift_delta + # perform the actual shifting + shifted = parsed + shift_delta - # controls wether the data is displayed with the 00:00:00 in Excel - # ... it might be nice to just drop the time if it's 00:00:00 - if isinstance(cell_value, datetime): - row_cells[col_idx] = cast(Any, shifted.to_pydatetime()) - else: - row_cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) - print('redo cell.number_format = "yyyy-mm-dd"') + # controls wether the data is displayed with the 00:00:00 in Excel + # ... it might be nice to just drop the time if it's 00:00:00 + if isinstance(cell_value, datetime): + row_cells[col_idx] = cast(Any, shifted.to_pydatetime()) + else: + row_cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) + print('redo cell.number_format = "yyyy-mm-dd"') - logger.info( - f"Shifting {last_name=} processed {last_row} rows (and that's the last page)" - ) logger.info("Output written to '%s'", output_file) -class row_state: +class row_scanner: + """builds upon the callbacks from book_page to handle row/page switching. does all + the checking for configs and what page we're on. needed because both patient IDs and + The Big Loop need the same checks""" + def __init__(self, sheet_configs: dict[str, dict]) -> None: self._sheet_configs = sheet_configs self._last_name: None | str = None @@ -607,7 +398,7 @@ def __init__(self, sheet_configs: dict[str, dict]) -> None: self._text_columns: None | list[str] = None self._sheet_config: None | dict | str = None - def in_body(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bool: + def patient_id(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bool: # check that it's in the configs if sheet_name not in self._sheet_configs: raise ExtraPage(sheet_name) @@ -629,7 +420,7 @@ def in_body(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bool: return False # grab the configuration - self.pid_column: str = cast( + self._patient_id_col: str = cast( str, self._sheet_config["patient_id_col"] ).strip() self._date_columns: list[str] = [ @@ -656,25 +447,31 @@ def in_body(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bool: if (not_skipped + 1) not in self._skip_rows ] if missing_skipped: + header_row = self._header_row + skip_rows = self._skip_rows raise RuntimeError( f"{sheet_name=} and {missing_skipped=} but" - + f" {self._header_row=} and {self._skip_rows=}" + + f" {header_row=} and {skip_rows=}" ) # check if the config hard-codes patient data to skip - if "shift_exceptions" in self._sheet_configs: + if "shift_exceptions" in self._sheet_config: raise RuntimeError( f"shift_exceptions was removed, update {sheet_name=}" ) + if "shift_ignore" in self._sheet_config: + raise RuntimeError("move shift_ignore from sheet_configs to a file") # check patient_id_col isn't re-used as other column types - if self.pid_column in self._date_columns: + if self._patient_id_col in self._date_columns: + patient_id_col = self._patient_id_col raise ValueError( - f"{self.pid_column=} shouldn't be in date_columns of {sheet_name=}" + f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" ) - if self.pid_column in self._text_columns: + if self._patient_id_col in self._text_columns: + patient_id_col = self._patient_id_col raise ValueError( - f"{self.pid_column=} shouldn't be in text_columns of {sheet_name=}" + f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" ) # check that column names don't appear in both @@ -695,7 +492,50 @@ def in_body(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bool: return False if row_idx == self._header_row: - raise Exception("compute columns and indcides") + # get the real column names now + self._col_names = row_cells + + # find patient_id_idx and normalize the column name list + for col_idx in range(len(self._col_names)): + col_name = self._col_names[col_idx] + + # if the name is None ... leave it as that (it's fine) + if col_name is None: + continue + + # strip the name + col_name = col_name.strip() + + # if the name is now blank; store None as the column name + if col_name == "": + self._col_names[col_idx] = None + continue + + # update it to just be the stripped version + self._col_names[col_idx] = col_name + + # check the config to see if we know what to do with this column + if col_name not in ( + [self._patient_id_col] + self._date_columns + self._text_columns + ): + raise ExtraColumn(sheet_name, col_name) + + # select the patient id + if col_name == self._patient_id_col: + self._patient_id_idx = col_idx + + # check to be sure we found the patient_id_col/patient_id_idx + if self._patient_id_idx is None: + raise PatientColumnMissing(sheet_name, self._patient_id_col) + + missing = [col for col in self._text_columns if col not in self._col_names] + if missing: + raise TextColumnsMissing(sheet_name, missing) + missing = [col for col in self._date_columns if col not in self._col_names] + if missing: + raise DateColumnsMissing(sheet_name, missing) + + return False if row_idx in self._skip_rows: return False diff --git a/tests/data/missing-id.xlsx b/tests/data/test_missing_id/missing-id.xlsx similarity index 100% rename from tests/data/missing-id.xlsx rename to tests/data/test_missing_id/missing-id.xlsx diff --git a/tests/test_missing_id.py b/tests/test_missing_id.py index b6eb3a6..1b9427c 100644 --- a/tests/test_missing_id.py +++ b/tests/test_missing_id.py @@ -5,6 +5,11 @@ from nuh_helper import shift_excel_dates_inplace from nuh_helper.date_shift import UnknownPatient +test_data = ( + # i feel like test data should be in a folder following this pattern + Path(__file__).parent / "data" / __name__ +) + def test_missing_id(tmp_path: Path) -> None: @@ -29,7 +34,7 @@ def test_missing_id(tmp_path: Path) -> None: }, } - source_file = str(Path(__file__).parent / "data/missing-id.xlsx") + source_file = str(test_data / "missing-id.xlsx") output_path = tmp_path / "target.xlsx" linking_table_old = tmp_path / "linking_table_old.csv" linking_table_out = tmp_path / "linking_table_out.csv" diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index bcccd1b..6b9bd04 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -61,7 +61,6 @@ def test_ptid_in_date_columns(use_csv: bool, tmp_path: Path) -> None: "text_columns": ["top"], "header_row": 1, "skip_rows_after_header": [], - "shift_ignore": {"dob": ["mssing"]}, }, } @@ -99,7 +98,6 @@ def test_ptid_in_text_columns(use_csv: bool, tmp_path: Path) -> None: "text_columns": ["top", "pid"], "header_row": 1, "skip_rows_after_header": [], - "shift_ignore": {"dob": ["mssing"]}, }, } @@ -137,7 +135,6 @@ def test_columns_overlap(use_csv: bool, tmp_path: Path) -> None: "text_columns": ["top", "dob"], "header_row": 1, "skip_rows_after_header": [], - "shift_ignore": {"dob": ["mssing"]}, }, } @@ -175,7 +172,6 @@ def test_ptid_column_missing(use_csv: bool, tmp_path: Path) -> None: "text_columns": ["top", "pid"], "header_row": 1, "skip_rows_after_header": [], - "shift_ignore": {"dob": ["mssing"]}, }, } From 1fc05bf122eade2da672d46666190bef32fa401f Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 17:48:03 +0100 Subject: [PATCH 59/82] extracts the ignore rules to a subroutine --- nuh_helper/date_shift/stream_shift.py | 106 ++++++++++++++++---------- 1 file changed, 65 insertions(+), 41 deletions(-) diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index fda9f21..01318e2 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -248,7 +248,7 @@ def shift_excel_dates_inplace( raise PageMissing(patient_sheet) with book[patient_sheet] as page: - scanner = row_scanner(sheet_configs) + scanner = PatientRowStateMachine(sheet_configs) shift_deltas = mappings.patient_shift_deltas( [ pid @@ -264,6 +264,7 @@ def shift_excel_dates_inplace( min_shift_days, max_shift_days, ) + scanner = None logger.info(f"loaded deltas {len(shift_deltas)=}") # check for missing sheets @@ -280,11 +281,14 @@ def shift_excel_dates_inplace( # - no option to add/remove rows is available # - no option to jump/seek through rows is available # - ... which is why this acts a bit weird - scanner = row_scanner(sheet_configs) + scanner = PatientRowStateMachine(sheet_configs) for sheet_name, row_idx, row_cells in book.stream_copy(output_file): + # check if we're in a patient row + # ... the method will raise exceptions if something is wrong or mis-configured pid = scanner.patient_id(sheet_name, row_idx, row_cells) if not pid: continue + # determine how many days to shift shift_delta = shift_deltas.get(pid) if shift_delta is None: @@ -306,7 +310,7 @@ def shift_excel_dates_inplace( cell_value = None # so if the cell is empty; skip the rest of the checks for this cell - # ... we go tot he next column in the row + # ... we go to the next column in the row if cell_value is None: row_cells[col_idx] = None continue @@ -317,45 +321,18 @@ def shift_excel_dates_inplace( if col_name is None: raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) - # skip values in shift_ignore - - if sheet_name not in shift_ignore: - shift_ignore[sheet_name] = {} - - if col_name not in shift_ignore[sheet_name]: - shift_ignore[sheet_name][col_name] = {None, ""} - - if cell_value in shift_ignore[sheet_name][col_name]: - continue - - # for text columns, we now *just* need to check for a hidden date - if col_name in scanner._text_columns: - # check for dates in non-date columns - for found in datefinder.find_dates(str(cell_value)): - logger.debug( - f"found hidden date {sheet_name=}/{col_name=}:{cell_value}" - ) - ignored = shift_ignore[sheet_name][col_name] - logger.debug( - f"shift_ignore[{sheet_name=}][{col_name=}]= {len(ignored)}" - ) - for ig in ignored: - logger.debug(f"- {ig}") - - raise HiddenDate( - sheet_name, - row_idx, - col_idx, - col_name, - cell_value, - found, - ) - row_cells[col_idx] = cell_value + # skip values in shift_ignore and text columns + # ... the check also looks for hidden dates + if not check_is_shifted( + scanner, + shift_ignore, + row_idx, + col_idx, + cell_value, + row_cells, + ): continue - # we've already checked this (sort of) - # assert col_name in date_columns - # cell_value can be str | datetime | date # ... so parsed might not always succeed parsed = _parse._parse_date_value(cell_value) @@ -379,7 +356,7 @@ def shift_excel_dates_inplace( logger.info("Output written to '%s'", output_file) -class row_scanner: +class PatientRowStateMachine: """builds upon the callbacks from book_page to handle row/page switching. does all the checking for configs and what page we're on. needed because both patient IDs and The Big Loop need the same checks""" @@ -574,3 +551,50 @@ def patient_id(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bo # it's a valid pid - return it return pid + + +def check_is_shifted( + scanner: PatientRowStateMachine, + shift_ignore: list[str], + row_idx: int, + col_idx: int, + cell_value: str, + row_cells: list, +) -> bool: + sheet_name: str = scanner._last_name + col_name: str = scanner._col_names[col_idx] + + if sheet_name not in shift_ignore: + shift_ignore[sheet_name] = {} + + if col_name not in shift_ignore[sheet_name]: + shift_ignore[sheet_name][col_name] = {None, ""} + + if cell_value in shift_ignore[sheet_name][col_name]: + return False + + # patient_id_col has already been checked + + if col_name in scanner._date_columns: + return True + + # for text columns, we now *just* need to check for a hidden date + if col_name in scanner._text_columns: + # check for dates in non-date columns + for found in datefinder.find_dates(str(cell_value)): + logger.debug(f"found hidden date {sheet_name=}/{col_name=}:{cell_value}") + ignored = shift_ignore[sheet_name][col_name] + logger.debug(f"shift_ignore[{sheet_name=}][{col_name=}]= {len(ignored)}") + for ig in ignored: + logger.debug(f"- {ig}") + + raise HiddenDate( + sheet_name, + row_idx, + col_idx, + col_name, + cell_value, + found, + ) + row_cells[col_idx] = cell_value + return False From 7f75ba7a1c88e7fdff0c125651f9182354dd1231 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Mon, 12 Oct 2026 17:50:17 +0100 Subject: [PATCH 60/82] idea for how to fixit --- nuh_helper/date_shift/stream_shift.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index 01318e2..e2e3400 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -375,6 +375,8 @@ def __init__(self, sheet_configs: dict[str, dict]) -> None: self._text_columns: None | list[str] = None self._sheet_config: None | dict | str = None + print("TODO; this should use a dataclass for the row-data that shifts") + def patient_id(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bool: # check that it's in the configs if sheet_name not in self._sheet_configs: From 0f064124014615fe46e6fc1c01cfae7949ca61f2 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 13 Oct 2026 12:59:03 +0100 Subject: [PATCH 61/82] more cleanup --- nuh_helper/date_shift/stream_shift.py | 379 +++++++++++++------------- 1 file changed, 190 insertions(+), 189 deletions(-) diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index e2e3400..9047020 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -1,7 +1,9 @@ import logging +from dataclasses import dataclass, replace from datetime import datetime +from enum import StrEnum from pathlib import Path -from typing import Any, cast +from typing import Any, Literal, cast import datefinder import yaml @@ -9,6 +11,13 @@ from nuh_helper.book_page import Book, book_open from nuh_helper.date_shift import _parse, mappings + +class ColumnType(StrEnum): + TEXT = "text" + DATE = "date" + IGNORE = "skip" + + logger = logging.getLogger(__name__) @@ -251,9 +260,9 @@ def shift_excel_dates_inplace( scanner = PatientRowStateMachine(sheet_configs) shift_deltas = mappings.patient_shift_deltas( [ - pid + pid.patient_id for pid in [ - scanner.patient_id(page.name, row, data) + scanner.next_details(page.name, row, data) for row, data in page.stream_rows() ] if pid @@ -285,75 +294,119 @@ def shift_excel_dates_inplace( for sheet_name, row_idx, row_cells in book.stream_copy(output_file): # check if we're in a patient row # ... the method will raise exceptions if something is wrong or mis-configured - pid = scanner.patient_id(sheet_name, row_idx, row_cells) - if not pid: + detail = scanner.next_details(sheet_name, row_idx, row_cells) + if not detail: continue # determine how many days to shift - shift_delta = shift_deltas.get(pid) + shift_delta = shift_deltas.get(detail.patient_id) if shift_delta is None: # all patients need to have a shift value - raise UnknownPatient(sheet_name, pid) + raise UnknownPatient(sheet_name, detail.patient_id) # now scan each column # ... even when page has no date_columns; still check for hidden dates # for col_idx in range(len(row_cells)): for col_idx, cell_value in enumerate(row_cells): # skip the patient id column (it was already checked anyway) - if col_idx == scanner._patient_id_idx: + if col_idx == detail.patient_id_idx: continue # get the cell value. replace it with None if it's just whitespace if isinstance(cell_value, str): cell_value = cell_value.strip() - if not cell_value: - cell_value = None - # so if the cell is empty; skip the rest of the checks for this cell - # ... we go to the next column in the row - if cell_value is None: - row_cells[col_idx] = None - continue + if not cell_value: + cell_value = None - # if there's no column name; this column of the row should be None - # ... and we win't reach this line if the cell_value was None - col_name = scanner._col_names[col_idx] - if col_name is None: + col_name = detail.col_names[col_idx] + + # with no column name; this column of the row should have been None + if cell_value and not col_name: raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) - # skip values in shift_ignore and text columns - # ... the check also looks for hidden dates - if not check_is_shifted( - scanner, - shift_ignore, - row_idx, - col_idx, - cell_value, - row_cells, - ): - continue + # choose what to do based on the column/cell type + match detail.column_type(shift_ignore, col_name, cell_value): + case ColumnType.IGNORE: + # "skip" is sheet-wide, but, ignore is for a single cell of data + pass + case ColumnType.TEXT: + # check for dates in non-date columns + for found in datefinder.find_dates(str(cell_value)): + logger.debug( + f"found hidden date {sheet_name=}/{col_name=}" + + f":{cell_value}" + ) + ignored = shift_ignore[sheet_name][col_name] + logger.debug( + f"shift_ignore[{sheet_name=}][{col_name=}]=" + + f" {len(ignored)}" + ) + for ig in ignored: + logger.debug(f"- {ig}") + + raise HiddenDate( + sheet_name, + row_idx, + col_idx, + col_name, + cell_value, + found, + ) + case ColumnType.DATE: + # cell_value can be str | datetime | date + # ... so parsed might not always succeed + parsed = _parse._parse_date_value(cell_value) + if parsed is None: + # raise an error; we already handled ignored values + raise ShiftFoundNonDate( + sheet_name, row_idx, col_idx, col_name, cell_value + ) + + # perform the actual shifting + shifted = parsed + shift_delta + + # would might be nice to always drop the 00:00:00 + if isinstance(cell_value, datetime): + cell_value = cast(Any, shifted.to_pydatetime()) + else: + cell_value = cast(Any, shifted.to_pydatetime().date()) + row_cells[col_idx] = cell_value - # cell_value can be str | datetime | date - # ... so parsed might not always succeed - parsed = _parse._parse_date_value(cell_value) - if parsed is None: - # raise an error; we already handled values we should pass as-is - raise ShiftFoundNonDate( - sheet_name, row_idx, col_idx, col_name, cell_value - ) + logger.info("Output written to '%s'", output_file) - # perform the actual shifting - shifted = parsed + shift_delta - # controls wether the data is displayed with the 00:00:00 in Excel - # ... it might be nice to just drop the time if it's 00:00:00 - if isinstance(cell_value, datetime): - row_cells[col_idx] = cast(Any, shifted.to_pydatetime()) - else: - row_cells[col_idx] = cast(Any, shifted.to_pydatetime().date()) - print('redo cell.number_format = "yyyy-mm-dd"') +@dataclass +class PatientPageDetail: + page_name: str + col_names: tuple[str, ...] + patient_id_idx: int + patient_id_col: str + patient_id: str + date_columns: tuple[str, ...] + text_columns: tuple[str, ...] - logger.info("Output written to '%s'", output_file) + def column_type( + self, + shift_ignore: dict[str, dict[str, list[str]]], + col_name: str, + cell_value: str, + ) -> bool: + sheet_name: str = self.page_name + + if sheet_name not in shift_ignore: + shift_ignore[sheet_name] = {} + + if col_name not in shift_ignore[sheet_name]: + shift_ignore[sheet_name][col_name] = {None, ""} + + return ( + ColumnType.IGNORE + if cell_value in shift_ignore[sheet_name][col_name] + else ColumnType.TEXT + if col_name in self.text_columns + else ColumnType.DATE + ) class PatientRowStateMachine: @@ -363,60 +416,34 @@ class PatientRowStateMachine: def __init__(self, sheet_configs: dict[str, dict]) -> None: self._sheet_configs = sheet_configs - self._last_name: None | str = None - self._last_row: None | int = None self._header_row: None | int = None self._skip_rows: None | list[int] = None + self._page_detail: None | PatientPageDetail = None - self._col_names: None | list[str] = None - self._patient_id_idx: None | int = None - self._patient_id_col: None | str = None - self._date_columns: None | list[str] = None - self._text_columns: None | list[str] = None - self._sheet_config: None | dict | str = None - - print("TODO; this should use a dataclass for the row-data that shifts") + def next_details( + self, sheet_name: str, row_idx: int, row_cells: list + ) -> PatientPageDetail | Literal[False]: + """determine if we should do a shift/check based on where we are in the loop.""" - def patient_id(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bool: # check that it's in the configs if sheet_name not in self._sheet_configs: raise ExtraPage(sheet_name) - if row_idx != 0: - # SAN/safety check in case something tries to skip showing us rows - if sheet_name != self._last_name: - raise RuntimeError(f"didn't see row0 from {sheet_name=}") - self._last_row = row_idx + sheet_config: dict | str = self._sheet_configs[sheet_name] if row_idx == 0: - self._last_name = sheet_name - self._last_row = row_idx + ### + # process the "which row is header and which do we skip" configuration + ## - self._sheet_config: dict | str = self._sheet_configs[sheet_name] + self._page_detail = None # do nothing to the skipped ones - if self._sheet_config == "skip": + if sheet_config == "skip": return False - # grab the configuration - self._patient_id_col: str = cast( - str, self._sheet_config["patient_id_col"] - ).strip() - self._date_columns: list[str] = [ - col.strip() for col in self._sheet_config["date_columns"] - ] - self._text_columns: list[str] = [ - col.strip() for col in self._sheet_config["text_columns"] - ] - - self._header_row: int = cast(int, self._sheet_config.get("header_row", 0)) - self._skip_rows: list[int] = self._sheet_config.get( - "skip_rows_after_header", [] - ) - - ### - # do some checks of the configuration - ## + self._header_row: int = cast(int, sheet_config.get("header_row", 0)) + self._skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) # check for holes in skip_rows (possible issue) if self._skip_rows: @@ -434,107 +461,112 @@ def patient_id(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bo ) # check if the config hard-codes patient data to skip - if "shift_exceptions" in self._sheet_config: + if "shift_exceptions" in sheet_config: raise RuntimeError( f"shift_exceptions was removed, update {sheet_name=}" ) - if "shift_ignore" in self._sheet_config: + if "shift_ignore" in sheet_config: raise RuntimeError("move shift_ignore from sheet_configs to a file") + if ( + (sheet_config == "skip") + or (row_idx < self._header_row) + or (row_idx in self._skip_rows) + ): + return False + + if row_idx == self._header_row: + ### + # now read the "which column" configuration + ## + + # grab the configuration + patient_id_col: str = str(sheet_config["patient_id_col"]).strip() + date_columns: list[str] = normalize_inplace(sheet_config["date_columns"]) + text_columns: list[str] = normalize_inplace(sheet_config["text_columns"]) + # check patient_id_col isn't re-used as other column types - if self._patient_id_col in self._date_columns: - patient_id_col = self._patient_id_col + if patient_id_col in date_columns: raise ValueError( f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" ) - if self._patient_id_col in self._text_columns: - patient_id_col = self._patient_id_col + if patient_id_col in text_columns: raise ValueError( f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" ) # check that column names don't appear in both - col_names = [col for col in self._date_columns if col in self._text_columns] - if col_names: + overlapping_columns = [col for col in date_columns if col in text_columns] + if overlapping_columns: raise ValueError( f"{sheet_name=} has the some columns in both date and text " - + str(col_names) + + str(overlapping_columns) ) - self._col_names = None - self._patient_id_idx = None - - if self._sheet_config == "skip": - return False - - if row_idx < self._header_row: - return False - - if row_idx == self._header_row: - # get the real column names now - self._col_names = row_cells - - # find patient_id_idx and normalize the column name list - for col_idx in range(len(self._col_names)): - col_name = self._col_names[col_idx] + # normalize the column names + col_names = normalize_inplace(row_cells) - # if the name is None ... leave it as that (it's fine) - if col_name is None: + # find patient_id_idx and check columns + patient_id_idx = None + for col_idx, col_name in enumerate(col_names): + if not col_name: continue - # strip the name - col_name = col_name.strip() - - # if the name is now blank; store None as the column name - if col_name == "": - self._col_names[col_idx] = None - continue - - # update it to just be the stripped version - self._col_names[col_idx] = col_name - # check the config to see if we know what to do with this column - if col_name not in ( - [self._patient_id_col] + self._date_columns + self._text_columns - ): + if col_name not in ([patient_id_col] + date_columns + text_columns): raise ExtraColumn(sheet_name, col_name) # select the patient id - if col_name == self._patient_id_col: - self._patient_id_idx = col_idx + if col_name == patient_id_col: + if patient_id_idx is not None: + raise KeyError( + f"{sheet_name=} multiple columns {patient_id_col=}" + ) + patient_id_idx = col_idx # check to be sure we found the patient_id_col/patient_id_idx - if self._patient_id_idx is None: - raise PatientColumnMissing(sheet_name, self._patient_id_col) + if patient_id_idx is None: + raise PatientColumnMissing(sheet_name, patient_id_col) - missing = [col for col in self._text_columns if col not in self._col_names] + # check for columns in the config and not in the sheet + missing = [col for col in text_columns if col not in col_names] if missing: raise TextColumnsMissing(sheet_name, missing) - missing = [col for col in self._date_columns if col not in self._col_names] + + missing = [col for col in date_columns if col not in col_names] if missing: raise DateColumnsMissing(sheet_name, missing) - return False + # we've completed the header_row; save the archetype of the page detail + self._page_detail = PatientPageDetail( + page_name=sheet_name, + col_names=col_names, + patient_id_idx=patient_id_idx, + patient_id_col=patient_id_col, + patient_id="

", + date_columns=tuple(date_columns), + text_columns=tuple(text_columns), + ) - if row_idx in self._skip_rows: + # we don't shift anything in the header_row return False # get the patient id for this row - pid = row_cells[self._patient_id_idx] + patient_id = row_cells[self._page_detail.patient_id_idx] - # check for date in the cell_value - for found in datefinder.find_dates(str(pid)): + # check for date in the patient_id + for found in datefinder.find_dates(str(patient_id)): raise HiddenDate( sheet_name, row_idx, - self._patient_id_idx, - self._patient_id_col, - pid, + self._page_detail.patient_id_idx, + self._page_detail.patient_id_col, + patient_id, found, ) - pid = _parse._normalize_patient_id(pid) - if pid is None: + patient_id = _parse._normalize_patient_id(patient_id) + if patient_id is None: # if the pid is None; the rest of the row should be None as well non_blank = [ cell @@ -551,52 +583,21 @@ def patient_id(self, sheet_name: str, row_idx: int, row_cells: list) -> str | bo # ... so we need to skip the rest of this loop return False - # it's a valid pid - return it - return pid - - -def check_is_shifted( - scanner: PatientRowStateMachine, - shift_ignore: list[str], - row_idx: int, - col_idx: int, - cell_value: str, - row_cells: list, -) -> bool: - sheet_name: str = scanner._last_name - col_name: str = scanner._col_names[col_idx] - - if sheet_name not in shift_ignore: - shift_ignore[sheet_name] = {} - - if col_name not in shift_ignore[sheet_name]: - shift_ignore[sheet_name][col_name] = {None, ""} - - if cell_value in shift_ignore[sheet_name][col_name]: - return False + # it's a valid pid - return a page detail object + # ... so that we don't need to examine this scanner + return replace( + self._page_detail, + patient_id=patient_id, + ) - # patient_id_col has already been checked - if col_name in scanner._date_columns: - return True +def normalize_inplace(row: list[str]) -> list[str]: + for idx, val in enumerate(row): + if isinstance(val, str): + val = val.strip() - # for text columns, we now *just* need to check for a hidden date - if col_name in scanner._text_columns: - # check for dates in non-date columns - for found in datefinder.find_dates(str(cell_value)): - logger.debug(f"found hidden date {sheet_name=}/{col_name=}:{cell_value}") - ignored = shift_ignore[sheet_name][col_name] - logger.debug(f"shift_ignore[{sheet_name=}][{col_name=}]= {len(ignored)}") - for ig in ignored: - logger.debug(f"- {ig}") + if not val: + val = None - raise HiddenDate( - sheet_name, - row_idx, - col_idx, - col_name, - cell_value, - found, - ) - row_cells[col_idx] = cell_value - return False + row[idx] = val + return row From 93da71de3637ca30d1db8f803397108441c84adb Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 13 Oct 2026 13:24:53 +0100 Subject: [PATCH 62/82] comments and removed a redundant check --- nuh_helper/date_shift/stream_shift.py | 32 +++++++++++++-------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index 9047020..0976b88 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -337,14 +337,6 @@ def shift_excel_dates_inplace( f"found hidden date {sheet_name=}/{col_name=}" + f":{cell_value}" ) - ignored = shift_ignore[sheet_name][col_name] - logger.debug( - f"shift_ignore[{sheet_name=}][{col_name=}]=" - + f" {len(ignored)}" - ) - for ig in ignored: - logger.debug(f"- {ig}") - raise HiddenDate( sheet_name, row_idx, @@ -378,6 +370,8 @@ def shift_excel_dates_inplace( @dataclass class PatientPageDetail: + """carries the data needed to process a row from the sheet config and header_row""" + page_name: str col_names: tuple[str, ...] patient_id_idx: int @@ -388,21 +382,19 @@ class PatientPageDetail: def column_type( self, - shift_ignore: dict[str, dict[str, list[str]]], + shift_ignore: dict[str, dict[str, set[None | str]]], col_name: str, cell_value: str, ) -> bool: - sheet_name: str = self.page_name + """determine what we should do for a given column""" - if sheet_name not in shift_ignore: - shift_ignore[sheet_name] = {} - - if col_name not in shift_ignore[sheet_name]: - shift_ignore[sheet_name][col_name] = {None, ""} + ignored: set[None | str] = shift_ignore.get(self.page_name, {}).get( + col_name, {None, ""} + ) return ( ColumnType.IGNORE - if cell_value in shift_ignore[sheet_name][col_name] + if cell_value in ignored else ColumnType.TEXT if col_name in self.text_columns else ColumnType.DATE @@ -423,7 +415,9 @@ def __init__(self, sheet_configs: dict[str, dict]) -> None: def next_details( self, sheet_name: str, row_idx: int, row_cells: list ) -> PatientPageDetail | Literal[False]: - """determine if we should do a shift/check based on where we are in the loop.""" + """need to call this every step of the loop. it returns False to indicate that + this row doesn't need further processing, or a page-description that includes + the located patient id""" # check that it's in the configs if sheet_name not in self._sheet_configs: @@ -592,6 +586,9 @@ def next_details( def normalize_inplace(row: list[str]) -> list[str]: + """modifies (and returns) the passed array to drop extra whitespace and replace + fully blank entries with None. doesn't touch non-strings""" + for idx, val in enumerate(row): if isinstance(val, str): val = val.strip() @@ -600,4 +597,5 @@ def normalize_inplace(row: list[str]) -> list[str]: val = None row[idx] = val + return row From 15171ae9be8da8d4ce5cc90781d6a4f747a9bd9c Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 13 Oct 2026 15:01:32 +0100 Subject: [PATCH 63/82] all old tests pass --- tests/data/test_e2e_date_shift/shift-ignore.yaml | 3 +++ tests/test_e2e_date_shift.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/data/test_e2e_date_shift/shift-ignore.yaml diff --git a/tests/data/test_e2e_date_shift/shift-ignore.yaml b/tests/data/test_e2e_date_shift/shift-ignore.yaml new file mode 100644 index 0000000..4ebcc47 --- /dev/null +++ b/tests/data/test_e2e_date_shift/shift-ignore.yaml @@ -0,0 +1,3 @@ +results: + date_result: + - unknown diff --git a/tests/test_e2e_date_shift.py b/tests/test_e2e_date_shift.py index 7f10acb..5cb6fbc 100644 --- a/tests/test_e2e_date_shift.py +++ b/tests/test_e2e_date_shift.py @@ -27,6 +27,8 @@ class _ShiftKwargs(TypedDict, total=False): DATA_DIR = Path(__file__).parent / "data" INPUT_FILE = DATA_DIR / "patients.xlsx" +test_data = Path(__file__).parent / "data" / __name__ + # Config matching the structure of patients.xlsx SHEET_CONFIGS = { "patients": { @@ -63,6 +65,7 @@ def run_shift(base: Path, **kwargs: Unpack[_ShiftKwargs]) -> tuple[Path, Path]: sheet_configs=SHEET_CONFIGS, linking_table_path=linking.parent / "old-linking.csv", linking_table_output=str(linking), + shift_ignore_yaml=test_data / ("shift-ignore.yaml"), **kwargs, ) return output, linking @@ -204,10 +207,10 @@ def test_linking_table_reproduces_same_shifts_on_new_file( str(INPUT_FILE), str(output2), patient_sheet="patients", - patient_id_col="patient_id", sheet_configs=SHEET_CONFIGS, linking_table_path=str(linking), linking_table_output=str(output_path / "linking2.csv"), + shift_ignore_yaml=test_data / ("shift-ignore.yaml"), ) for sheet in SHEET_CONFIGS: From ae4100e7372d0b6c48f3f452358c907be34ed097 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 13 Oct 2026 15:07:35 +0100 Subject: [PATCH 64/82] don't need that --- nuh_helper/date_shift/__init__.py | 308 ------------------------------ nuh_helper/date_shift/_excel.py | 97 +--------- 2 files changed, 1 insertion(+), 404 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index a7b2d72..4e74384 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -6,13 +6,7 @@ """ import logging -from datetime import date, datetime -from pathlib import Path -from typing import Any, cast -import pandas as pd - -from nuh_helper.date_shift import _excel, _parse, mappings from nuh_helper.date_shift.mappings import ( # noqa: E402 generate_shift_mappings, load_shift_mappings, @@ -33,308 +27,6 @@ logger = logging.getLogger(__name__) - -def _get_patient_ids_and_shift_mappings( - input_file: Path, - patient_sheet: str, - patient_id_col: str, - sheet_configs: dict[str, dict[str, Any]], - min_shift_days: int, - max_shift_days: int, - linking_table_path: str | None, - seed: int | None, - patient_header_row: int, - patient_skip_rows: list[int] | None, -) -> tuple[list[str], pd.DataFrame]: - """ - Read patient IDs from the patient sheet and resolve shift mappings - (load from CSV or generate). Returns (patient_ids, shift_mappings) - with shift_mappings already normalized and deduplicated. - """ - effective_patient_header_row = patient_header_row - effective_patient_skip_rows = patient_skip_rows - if patient_sheet in sheet_configs: - cfg = sheet_configs[patient_sheet] - effective_patient_header_row = cast( - int, cfg.get("header_row", patient_header_row) - ) - effective_patient_skip_rows = cfg.get( - "skip_rows_after_header", patient_skip_rows - ) - - patient_excel = pd.ExcelFile(input_file, engine="openpyxl") - patient_df, _, _, _ = _excel._read_sheet_with_structure( - patient_excel, - sheet_name=patient_sheet, - header_row=effective_patient_header_row, - input_file=input_file, - skip_rows_after_header=effective_patient_skip_rows, - ) - if patient_id_col not in patient_df.columns: - raise mappings.PatientColumnMissing(patient_sheet, patient_id_col) - - patient_ids = ( - patient_df[patient_id_col] - .apply(_parse._normalize_patient_id) - .dropna() - .unique() - .tolist() - ) - logger.info("Found %d patient(s) in sheet '%s'", len(patient_ids), patient_sheet) - - if linking_table_path and Path(linking_table_path).exists(): - logger.info("Loading shift mappings from '%s'", linking_table_path) - shift_mappings = mappings.load_shift_mappings(linking_table_path) - shift_mappings = shift_mappings[shift_mappings["patient_id"].isin(patient_ids)] - existing_ids = set(shift_mappings["patient_id"]) - missing_ids = [pid for pid in patient_ids if pid not in existing_ids] - if missing_ids: - logger.warning( - "%d patient(s) had no entry in the linking table; new shifts generated", - len(missing_ids), - ) - new_shifts = mappings.generate_shift_mappings( - missing_ids, min_shift_days, max_shift_days, seed - ) - shift_mappings = pd.concat([shift_mappings, new_shifts], ignore_index=True) - else: - logger.info("Generating shift mappings for %d patient(s)", len(patient_ids)) - shift_mappings = mappings.generate_shift_mappings( - patient_ids, min_shift_days, max_shift_days, seed - ) - - shift_mappings["patient_id"] = shift_mappings["patient_id"].apply( - _parse._normalize_patient_id - ) - shift_mappings = shift_mappings.dropna(subset=["patient_id"]).drop_duplicates( - subset=["patient_id"], keep="first" - ) - return (patient_ids, shift_mappings) - - -def apply_date_shifts( - df: pd.DataFrame, - patient_id_col: str, - date_columns: list[str], - shift_mappings: pd.DataFrame, - date_format: str | None = None, - shift_exceptions: dict[str, list[str]] | None = None, -) -> pd.DataFrame: - """ - Apply date shifts to specified columns in a DataFrame. - - Args: - df: pd.DataFrame containing patient data. - patient_id_col: Name of the column containing patient IDs. - date_columns: List of column names containing dates to shift. - shift_mappings: DataFrame with 'patient_id' and 'shift_days' columns. - date_format: Optional date format string (e.g., 'YYYY-MM-DD'). - Note: This parameter is kept for API compatibility but formatting - is applied at the Excel cell level, not in the DataFrame. - shift_exceptions: Optional dict mapping column names to lists of date strings - that should never be shifted (e.g. a fixed end-of-study date). - - Returns: - DataFrame with shifted dates. - """ - df = df.copy() - - # Normalize patient IDs in the working DataFrame to align with mapping keys - df[patient_id_col] = df[patient_id_col].apply(_parse._normalize_patient_id) - - shift_dict = dict( - zip( - shift_mappings["patient_id"], - shift_mappings["shift_days"], - strict=True, - ) - ) - - # Pre-parse exception dates once per column - parsed_exceptions: dict[str, set[date]] = {} - if shift_exceptions: - for col, exc_values in shift_exceptions.items(): - parsed_set: set[date] = set() - for v in exc_values: - ts = _parse._parse_date_value(v) - if ts is not None: - parsed_set.add(ts.date()) - if parsed_set: - parsed_exceptions[col] = parsed_set - - for date_col in date_columns: - if date_col not in df.columns: - logger.warning( - "Date column '%s' not found in DataFrame, skipping", date_col - ) - continue - - # Parse flexible date strings (handles YYYY-DD-MM and placeholders "Unknown") - non_null_before = df[date_col].notna().sum() - df[date_col] = df[date_col].apply(_parse._parse_date_value) - parse_failures = non_null_before - sum(x is not None for x in df[date_col]) - if parse_failures > 0: - logger.debug( - "Column '%s': %d value(s) could not be parsed as dates", - date_col, - parse_failures, - ) - - # Apply shifts - exc_dates = parsed_exceptions.get(date_col, set()) - df[date_col] = df.apply( - lambda row: ( - row[date_col] # noqa: B023 - + pd.Timedelta(days=shift_dict.get(row[patient_id_col], 0)) - if row[date_col] is not None # noqa: B023 - and row[patient_id_col] in shift_dict # noqa: B023 - and row[date_col].date() not in exc_dates # noqa: B023 - else row[date_col] # noqa: B023 - ), - axis=1, - ) - - # Convert back to date-only format (removes time component) - df[date_col] = df[date_col].apply( - lambda x: ( - x.date() if isinstance(x, pd.Timestamp | datetime | date) else None - ), - ) - - return df - - -def shift_excel_dates( - input_file: str, - output_file: str, - patient_sheet: str, - patient_id_col: str, - sheet_configs: dict[str, dict[str, Any]], - min_shift_days: int = -15, - max_shift_days: int = 15, - linking_table_path: str | None = None, - linking_table_output: str | None = None, - seed: int | None = None, - patient_header_row: int = 0, - patient_skip_rows: list[int] | None = None, - date_format: str | None = None, -) -> None: - """ - Shift dates in an Excel file for patient IDs consistently across sheets. - - Args: - input_file: Path to input Excel file. - output_file: Path to output Excel file with shifted dates. - patient_sheet: Name of the sheet containing patient IDs. - patient_id_col: Name of the column containing patient IDs in the patient sheet. - sheet_configs: Dictionary mapping sheet names to configuration dicts. - Each config dict should have: - - 'patient_id_col': Name of patient ID column in that sheet - - 'date_columns': List of date column names to shift - Optional per-sheet header handling: - - 'header_row': zero-based row index of the column names (default 0) - - 'skip_rows_after_header': list of zero-based row indices to exclude - from data (e.g. a data-type row immediately below the header) - min_shift_days: Minimum number of days to shift (default: -15). - max_shift_days: Maximum number of days to shift (default: 15). - linking_table_path: Optional path to existing linking table CSV for reproducibility. - linking_table_output: Path to save the linking table CSV (default: 'shift_mappings.csv'). - seed: Optional random seed for generating shifts. - patient_header_row: Zero-based header row index for the patient sheet (default: 0). - patient_skip_rows: Optional zero-based row indices to exclude from patient data - (e.g. a data-type row immediately below the header). - date_format: Optional Excel date format string (e.g., 'YYYY-MM-DD', 'yyyy-mm-dd'). - If None, Excel's default date format is used. - Common formats: 'YYYY-MM-DD', 'MM/DD/YYYY', 'DD-MM-YYYY', etc. - """ # noqa: E501 - logger.info("Shifting dates: '%s' → '%s'", input_file, output_file) - logger.debug( - "Shift range: %d to %d days, seed=%s", - min_shift_days, - max_shift_days, - seed, - ) - - _patient_ids, shift_mappings = _get_patient_ids_and_shift_mappings( - input_file=input_file, - patient_sheet=patient_sheet, - patient_id_col=patient_id_col, - sheet_configs=sheet_configs, - min_shift_days=min_shift_days, - max_shift_days=max_shift_days, - linking_table_path=linking_table_path, - seed=seed, - patient_header_row=patient_header_row, - patient_skip_rows=patient_skip_rows, - ) - - with pd.ExcelWriter(output_file, engine="openpyxl") as writer: - excel_file = pd.ExcelFile(input_file, engine="openpyxl") - - for sheet_name in excel_file.sheet_names: - default_header_row = 0 - header_row = default_header_row - sheet_date_columns: list[str] | None = None - skip_rows_after_header: list[int] | None = None - sheet_shift_exceptions: dict[str, list[str]] | None = None - - if sheet_name in sheet_configs: - config = sheet_configs[cast(str, sheet_name)] - sheet_patient_id_col: str = cast(str, config["patient_id_col"]) - date_columns: list[str] = cast(list[str], config["date_columns"]) - header_row = cast(int, config.get("header_row", header_row)) - skip_rows_after_header = config.get("skip_rows_after_header") - sheet_shift_exceptions = config.get("shift_exceptions") - sheet_date_columns = date_columns - logger.info( - "Shifting %d date column(s) in sheet '%s'", - len(date_columns), - sheet_name, - ) - - df, _description_df, description_rows, description_merged_ranges = ( - _excel._read_sheet_with_structure( - excel_file, - sheet_name=cast(str, sheet_name), - header_row=header_row, - input_file=input_file, - skip_rows_after_header=skip_rows_after_header, - ) - ) - - if sheet_name in sheet_configs: - if sheet_patient_id_col not in df.columns: - raise ValueError( - f"Patient ID column '{sheet_patient_id_col}' not found in sheet '{sheet_name}'" # noqa: E501 - ) - - df = apply_date_shifts( - df, - sheet_patient_id_col, - date_columns, - shift_mappings, - date_format=None, - shift_exceptions=sheet_shift_exceptions, - ) - - _excel._write_sheet_with_structure( - writer, - sheet_name=cast(str, sheet_name), - data_df=df, - description_rows=description_rows, - header_row=header_row, - date_columns=sheet_date_columns, - date_format=date_format, - description_merged_ranges=description_merged_ranges, - ) - - logger.info("Output written to '%s'", output_file) - - linking_path = linking_table_output or "shift_mappings.csv" - shift_mappings.to_csv(linking_path, index=False) - logger.info("Linking table saved to '%s'", linking_path) - - # Re-export public API __all__ = [ generate_shift_mappings, diff --git a/nuh_helper/date_shift/_excel.py b/nuh_helper/date_shift/_excel.py index 204b4a5..3782704 100644 --- a/nuh_helper/date_shift/_excel.py +++ b/nuh_helper/date_shift/_excel.py @@ -1,10 +1,9 @@ """Excel sheet structure helpers for date-shift.""" import contextlib -from typing import Any, cast +from typing import Any import pandas as pd -from openpyxl import load_workbook from openpyxl.cell.cell import MergedCell from openpyxl.worksheet.worksheet import Worksheet @@ -48,100 +47,6 @@ def _description_merged_ranges( return result -def _read_sheet_with_structure( - excel_file: pd.ExcelFile, - sheet_name: str, - header_row: int = 0, - input_file: str | None = None, - skip_rows_after_header: list[int] | None = None, -) -> tuple[pd.DataFrame, pd.DataFrame, list[list[Any]], list[str]]: - """ - Read a sheet preserving description rows and structure. - - Uses openpyxl to resolve merged cells in the header row so column names - are correct when the sheet has merged cells. Rows whose indices are in - skip_rows_after_header (0-based) are excluded from the data (e.g. a - data-type row immediately below the header). - - Returns: - Tuple of (data_df, description_df, description_rows, - description_merged_ranges) - - data_df: DataFrame with header row as column names and data rows - - description_df: DataFrame with description rows (if any) - - description_rows: List of description row data (for writing back) - - description_merged_ranges: Merged range refs to preserve when writing - """ - # Read entire sheet without header to preserve all rows - full_df = pd.read_excel(excel_file, sheet_name=sheet_name, header=None) - max_col = full_df.shape[1] - description_merged_ranges: list[str] = [] - - # Resolve header row (and optional description merged ranges) via openpyxl - if input_file and max_col: - wb = load_workbook(input_file, read_only=False, data_only=True) - if sheet_name in wb.sheetnames: - ws = wb[sheet_name] - # Header row in 1-based openpyxl - header_row_1based = header_row + 1 - header_values = _get_row_values_resolving_merged( - ws, header_row_1based, max_col - ) - # Pandas-friendly column names (no None) - columns = [] - for i, v in enumerate(header_values): - if v is None or (isinstance(v, float) and pd.isna(cast("float", v))): - columns.append(f"Unnamed: {i}") - else: - columns.append(str(v).strip() or f"Unnamed: {i}") - - if header_row > 0: - description_merged_ranges = _description_merged_ranges(ws, header_row) - - # Description rows (rows before header) - description_rows = ( - full_df.iloc[:header_row].values.tolist() if header_row > 0 else [] - ) - # Data rows: everything after header, optionally excluding some rows - data_block = full_df.iloc[header_row + 1 :] - if skip_rows_after_header: - # Drop by 0-based index (data_block index = original row) - to_drop = [i for i in data_block.index if i in skip_rows_after_header] - data_block = data_block.drop(index=to_drop, errors="ignore") - - data_df = pd.DataFrame(data_block.values, columns=columns) - description_df = ( - full_df.iloc[:header_row].copy() if header_row > 0 else pd.DataFrame() - ) - wb.close() - return ( - data_df, - description_df, - description_rows, - description_merged_ranges, - ) - - # Fallback when no openpyxl path or empty sheet - if header_row == 0: - description_rows = [] - description_df = pd.DataFrame() - data_df = pd.read_excel(excel_file, sheet_name=sheet_name, header=0) - if skip_rows_after_header: - data_df = data_df.drop(index=skip_rows_after_header, errors="ignore") - else: - description_rows = full_df.iloc[:header_row].values.tolist() - description_df = full_df.iloc[:header_row].copy() - data_df = pd.read_excel(excel_file, sheet_name=sheet_name, header=header_row) - if skip_rows_after_header: - data_df = data_df.drop(index=skip_rows_after_header, errors="ignore") - - return ( - data_df, - description_df, - description_rows, - description_merged_ranges, - ) - - def _write_sheet_with_structure( writer: pd.ExcelWriter, sheet_name: str, From a45f7d6f2e81e975f2987683e3f6ea84f749a6a1 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 13 Oct 2026 15:11:34 +0100 Subject: [PATCH 65/82] out with the old --- nuh_helper/date_shift/_excel.py | 136 -------------------------------- tests/test_date_shift.py | 80 ------------------- 2 files changed, 216 deletions(-) delete mode 100644 nuh_helper/date_shift/_excel.py diff --git a/nuh_helper/date_shift/_excel.py b/nuh_helper/date_shift/_excel.py deleted file mode 100644 index 3782704..0000000 --- a/nuh_helper/date_shift/_excel.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Excel sheet structure helpers for date-shift.""" - -import contextlib -from typing import Any - -import pandas as pd -from openpyxl.cell.cell import MergedCell -from openpyxl.worksheet.worksheet import Worksheet - - -def _get_row_values_resolving_merged( - sheet: Worksheet, row_1based: int, max_col: int -) -> list[Any]: - """ - Return values for one row, resolving merged cells to the top-left cell value. - - openpyxl stores the value only in the first cell of a merged range; - other cells are MergedCell and have no value. This walks the row and - fills in the value from the merge range's top-left for each column. - """ - result: list[Any] = [] - for col in range(1, max_col + 1): - cell = sheet.cell(row=row_1based, column=col) - if isinstance(cell, MergedCell): - for merged_range in sheet.merged_cells.ranges: - if cell.coordinate in merged_range: - top_left = sheet.cell( - row=merged_range.min_row, column=merged_range.min_col - ) - result.append(top_left.value) - break - else: - result.append(None) - else: - result.append(cell.value) - return result - - -def _description_merged_ranges( - sheet: Worksheet, num_description_rows: int -) -> list[str]: - """Merged range refs (e.g. 'A1:D1') in the first num_description_rows.""" - result: list[str] = [] - for merged_range in sheet.merged_cells.ranges: - if merged_range.max_row <= num_description_rows: - result.append(str(merged_range)) - return result - - -def _write_sheet_with_structure( - writer: pd.ExcelWriter, - sheet_name: str, - data_df: pd.DataFrame, - description_rows: list[list[Any]], - header_row: int, - date_columns: list[str] | None = None, - date_format: str | None = None, - description_merged_ranges: list[str] | None = None, -) -> None: - """ - Write a sheet preserving description rows and structure. - - Args: - writer: ExcelWriter instance. - sheet_name: Name of the sheet to write. - data_df: DataFrame with data to write. - description_rows: List of description rows to write at the top. - header_row: Row index where header should be written. - date_columns: Optional list of date column names to format. - date_format: Optional Excel date format string (e.g., 'yyyy-mm-dd'). - description_merged_ranges: Optional merged cell refs to restore - (e.g. 'A1:D1'). - """ - # Convert Python date format to Excel format - excel_date_format = None - if date_format: - # Convert common Python format strings to Excel format - # Excel uses lowercase: yyyy (year), mm (month), dd (day) - excel_date_format = ( - date_format.replace("YYYY", "yyyy") - .replace("YY", "yy") - .replace("DD", "dd") - .replace("MM", "mm") - ) - - # Calculate where to start writing data (after description rows + header row) - data_start_row = len(description_rows) + 1 if description_rows else 1 - - # Write data without header first (header=False), then we'll add header manually - data_df.to_excel( - writer, - sheet_name=sheet_name, - index=False, - header=False, - startrow=data_start_row, - ) - - # Get the workbook and worksheet - workbook = writer.book - worksheet = workbook[sheet_name] - - # Write description rows at the top - if description_rows: - for row_idx, desc_row in enumerate(description_rows, start=1): - for col_idx, value in enumerate(desc_row, start=1): - cell_value = value - # Handle NaN values - if pd.isna(cell_value): - cell_value = None - worksheet.cell(row=row_idx, column=col_idx, value=cell_value) - # Restore merged cells in the description area - if description_merged_ranges: - for range_ref in description_merged_ranges: - with contextlib.suppress(Exception): - worksheet.merge_cells(range_ref) - - # Write header row (after description rows) - header_row_idx = len(description_rows) + 1 if description_rows else 1 - for col_idx, col_name in enumerate(data_df.columns, start=1): - worksheet.cell(row=header_row_idx, column=col_idx, value=col_name) - - # Apply date formatting to date columns if specified - if excel_date_format and date_columns: - # Find column indices for date columns - col_map = {col: idx + 1 for idx, col in enumerate(data_df.columns)} - for date_col in date_columns: - if date_col in col_map: - col_idx = col_map[date_col] - # Apply format to all data rows (skip description and header rows) - for row_idx in range( - data_start_row + 1, data_start_row + len(data_df) + 1 - ): - cell = worksheet.cell(row=row_idx, column=col_idx) - # Only format if cell has a date value - if cell.value is not None: - cell.number_format = excel_date_format diff --git a/tests/test_date_shift.py b/tests/test_date_shift.py index 8d62368..4bb4ca8 100644 --- a/tests/test_date_shift.py +++ b/tests/test_date_shift.py @@ -12,10 +12,6 @@ load_shift_mappings, shift_excel_dates_inplace, ) -from nuh_helper.date_shift._excel import ( - _description_merged_ranges, - _get_row_values_resolving_merged, -) from nuh_helper.date_shift._parse import _normalize_patient_id, _parse_date_value test_data = Path(__file__).parent / "data" / __name__ @@ -136,82 +132,6 @@ def test_raises_on_missing_columns(self, tmp_path: Path) -> None: load_shift_mappings(str(csv_file)) -class TestGetRowValuesResolvingMerged: - """Unit tests for _get_row_values_resolving_merged (merged cell header handling).""" - - def test_merged_row_returns_value_repeated_for_each_column(self) -> None: - wb = Workbook() - ws = wb.active - assert ws is not None - ws["A1"] = "crazy" - ws.merge_cells("A1:D1") - result = _get_row_values_resolving_merged(ws, 1, 4) - assert result == ["crazy", "crazy", "crazy", "crazy"] - - def test_normal_row_returns_cell_values(self) -> None: - wb = Workbook() - ws = wb.active - assert ws is not None - ws["A2"] = "patient_id" - ws["B2"] = "measurement" - ws["C2"] = "date_result" - ws["D2"] = "type" - result = _get_row_values_resolving_merged(ws, 2, 4) - assert result == ["patient_id", "measurement", "date_result", "type"] - - def test_partial_merge_second_column_gets_top_left_value(self) -> None: - wb = Workbook() - ws = wb.active - assert ws is not None - ws["A1"] = "a" - ws["B1"] = "b" - ws.merge_cells("B1:C1") - ws["D1"] = "d" - result = _get_row_values_resolving_merged(ws, 1, 4) - assert result == ["a", "b", "b", "d"] - - -class TestDescriptionMergedRanges: - """Unit tests for _description_merged_ranges.""" - - def test_returns_merged_ranges_within_description_rows(self) -> None: - wb = Workbook() - ws = wb.active - assert ws is not None - ws["A1"] = "title" - ws.merge_cells("A1:D1") - result = _description_merged_ranges(ws, 1) - assert result == ["A1:D1"] - - def test_excludes_merged_ranges_below_description_rows(self) -> None: - wb = Workbook() - ws = wb.active - assert ws is not None - ws["A1"] = "title" - ws.merge_cells("A1:D1") - ws["A2"] = "x" - ws.merge_cells("A2:B2") - result = _description_merged_ranges(ws, 1) - assert result == ["A1:D1"] - - def test_empty_when_no_merged_cells(self) -> None: - wb = Workbook() - ws = wb.active - assert ws is not None - ws["A1"] = "a" - result = _description_merged_ranges(ws, 2) - assert result == [] - - def test_empty_when_num_description_rows_zero(self) -> None: - wb = Workbook() - ws = wb.active - assert ws is not None - ws["A1"] = "x" - ws.merge_cells("A1:B1") - result = _description_merged_ranges(ws, 0) - assert result == [] - - class TestShiftExcelDatesWithComplexLayout: """Integration tests: header_row, skip_rows_after_header, merged cells, patient sheet from config.""" From f191501b25e28da192af3daf2dc5aa71871fe7d9 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 13 Oct 2026 16:50:15 +0100 Subject: [PATCH 66/82] remove unused log --- nuh_helper/date_shift/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 4e74384..1b73227 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -5,8 +5,6 @@ in an Excel file, with support for reproducible shifts using a linking table. """ -import logging - from nuh_helper.date_shift.mappings import ( # noqa: E402 generate_shift_mappings, load_shift_mappings, @@ -25,8 +23,6 @@ shift_excel_dates_inplace, ) -logger = logging.getLogger(__name__) - # Re-export public API __all__ = [ generate_shift_mappings, From e36a4fdc0ecbb4afc7b8666810e4e1656e1034fc Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Tue, 13 Oct 2026 17:30:19 +0100 Subject: [PATCH 67/82] removed more unused functions and tests for what was left --- nuh_helper/__init__.py | 4 --- nuh_helper/date_shift/__init__.py | 2 -- nuh_helper/date_shift/_parse.py | 19 ++++++------ nuh_helper/date_shift/mappings.py | 18 ----------- tests/test_date_shift.py | 16 ---------- tests/test_parse.py | 51 +++++++++++++++++++++++++++++++ 6 files changed, 60 insertions(+), 50 deletions(-) create mode 100644 tests/test_parse.py diff --git a/nuh_helper/__init__.py b/nuh_helper/__init__.py index 2edb216..e020d8b 100644 --- a/nuh_helper/__init__.py +++ b/nuh_helper/__init__.py @@ -11,15 +11,11 @@ logging.getLogger(__name__).addHandler(logging.NullHandler()) from nuh_helper.date_shift import ( # noqa: E402 - generate_shift_mappings, - load_shift_mappings, shift_excel_dates_inplace, ) from nuh_helper.profile import generate_scan_report # noqa: E402 __all__ = [ generate_scan_report, - generate_shift_mappings, - load_shift_mappings, shift_excel_dates_inplace, ] diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index 1b73227..33b4382 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -7,7 +7,6 @@ from nuh_helper.date_shift.mappings import ( # noqa: E402 generate_shift_mappings, - load_shift_mappings, ) from nuh_helper.date_shift.stream_shift import ( BlankColumnHasData, @@ -26,7 +25,6 @@ # Re-export public API __all__ = [ generate_shift_mappings, - load_shift_mappings, shift_excel_dates_inplace, HiddenDate, PatientColumnMissing, diff --git a/nuh_helper/date_shift/_parse.py b/nuh_helper/date_shift/_parse.py index ee8c494..d71433c 100644 --- a/nuh_helper/date_shift/_parse.py +++ b/nuh_helper/date_shift/_parse.py @@ -18,14 +18,12 @@ def _parse_date_value( return result if pd.notna(result) else None if isinstance(value, str): - v = value.strip() - if not v or v.lower() in {"unknown", "unk", "unkown", "n/a", "none", "null"}: - return None + value = value.strip() # Try a handful of common formats, including YYYY-DD-MM found in some feeds for fmt in ("%Y-%m-%d", "%Y-%d-%m", "%d-%m-%Y", "%m-%d-%Y"): try: - parsed = pd.to_datetime(v, format=fmt, errors="coerce") + parsed = pd.to_datetime(value, format=fmt, errors="coerce") if pd.notna(parsed): return parsed except (TypeError, ValueError): @@ -33,7 +31,7 @@ def _parse_date_value( pass # Fallback: let pandas try with dayfirst to handle ambiguous strings - parsed = pd.to_datetime(v, errors="coerce", dayfirst=True) + parsed = pd.to_datetime(value, errors="coerce", dayfirst=True) return parsed if pd.notna(parsed) else None # Anything else: no parse @@ -45,9 +43,10 @@ def _normalize_patient_id(value: object) -> str | None: if value is None or (isinstance(value, float) and pd.isna(value)): return None - if isinstance(value, str): - v = value.strip() - return v if v else None + if value and not isinstance(value, str): + value = str(value) + + if value: + value = value.strip() - v = str(value).strip() - return v if v else None + return value if value else None diff --git a/nuh_helper/date_shift/mappings.py b/nuh_helper/date_shift/mappings.py index 7bf628b..87d1d0b 100644 --- a/nuh_helper/date_shift/mappings.py +++ b/nuh_helper/date_shift/mappings.py @@ -37,24 +37,6 @@ def generate_shift_mappings( return pd.DataFrame({"patient_id": patient_ids, "shift_days": shifts}) -def load_shift_mappings(csv_path: str) -> pd.DataFrame: - """ - Load shift mappings from a CSV file. - - Args: - csv_path: Path to the CSV file containing shift mappings. - Expected columns: 'patient_id' and 'shift_days'. - - Returns: - DataFrame with shift mappings. - """ - df = pd.read_csv(csv_path) - if "patient_id" not in df.columns or "shift_days" not in df.columns: - raise ValueError("CSV must contain 'patient_id' and 'shift_days' columns") - logger.info("Loaded %d shift mapping(s) from '%s'", len(df), csv_path) - return df - - def patient_shift_deltas( patient_ids: list[str], src_linking_table: str | Path, diff --git a/tests/test_date_shift.py b/tests/test_date_shift.py index 4bb4ca8..dd65234 100644 --- a/tests/test_date_shift.py +++ b/tests/test_date_shift.py @@ -9,7 +9,6 @@ from nuh_helper.date_shift import ( generate_shift_mappings, - load_shift_mappings, shift_excel_dates_inplace, ) from nuh_helper.date_shift._parse import _normalize_patient_id, _parse_date_value @@ -117,21 +116,6 @@ def test_empty_patient_list(self) -> None: assert len(result) == 0 -class TestLoadShiftMappings: - def test_loads_valid_csv(self, tmp_path: Path) -> None: - csv_file = tmp_path / "shifts.csv" - csv_file.write_text("patient_id,shift_days\nP001,5\nP002,-3\n") - result = load_shift_mappings(str(csv_file)) - assert list(result["patient_id"]) == ["P001", "P002"] - assert list(result["shift_days"]) == [5, -3] - - def test_raises_on_missing_columns(self, tmp_path: Path) -> None: - csv_file = tmp_path / "bad.csv" - csv_file.write_text("id,days\nP001,5\n") - with pytest.raises(ValueError, match="patient_id.*shift_days"): - load_shift_mappings(str(csv_file)) - - class TestShiftExcelDatesWithComplexLayout: """Integration tests: header_row, skip_rows_after_header, merged cells, patient sheet from config.""" diff --git a/tests/test_parse.py b/tests/test_parse.py new file mode 100644 index 0000000..bdcaf16 --- /dev/null +++ b/tests/test_parse.py @@ -0,0 +1,51 @@ +import datetime +from typing import Never + +import pandas as pd +import pytest + +from nuh_helper.date_shift._parse import _normalize_patient_id, _parse_date_value + + +@pytest.mark.parametrize( + "source, expected", + [ + ("12-31-2004", "2004-12-31 00:00:00"), + ("12/03/2016", "2016-03-12 00:00:00"), + ("2016-12-03", "2016-12-03 00:00:00"), + ("2016-13-03", "2016-03-13 00:00:00"), + ("31-12-2004", "2004-12-31 00:00:00"), + ("not-a-date", "None"), + (3.9, "None"), + (None, "None"), + (datetime.date(2026, 9, 12), "2026-09-12 00:00:00"), + (datetime.datetime(2026, 9, 12, 13, 14, 15), "2026-09-12 13:14:15"), + (float("nan"), "None"), + (pd.NaT, "None"), + (pd.Timestamp("2026-09-12 13:14:15"), "2026-09-12 13:14:15"), + ], +) +def test_date_value(source: str, expected: str) -> Never: + obtained: str = str(_parse_date_value(source)) + + assert obtained == expected + + +@pytest.mark.parametrize( + "source, expected", + [ + (" ", "None"), + (" nuh007 ", "nuh007"), + ("", "None"), + ("2016-13-03", "2016-13-03"), + (0, "None"), + (3.9, "3.9"), + (None, "None"), + (datetime.date(2026, 9, 12), "2026-09-12"), + (float("nan"), "None"), + ], +) +def test_patient_id(source: str, expected: str) -> Never: + obtained: str = str(_normalize_patient_id(source)) + + assert obtained == expected From fe70510a4d540dacd08f122438dbf4be49622537 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Wed, 14 Oct 2026 14:09:04 +0100 Subject: [PATCH 68/82] "orchestration" changes - shift_excel_inplace is 15 lines - ... ignoring logs/docs, and multi-line calls - extracts the shifting logic itself - the handling of page/row changes is now a FSM - moved shared definitions into their own file - merges the shift-delta loading and generation - annotated with sheet_configs_t - added pyyaml dependency --- .vscode/settings.json | 11 +- README.md | 107 +++-- nuh_helper/book_page/csv_book.py | 13 + nuh_helper/date_shift/__init__.py | 20 - nuh_helper/date_shift/defs.py | 162 +++++++ nuh_helper/date_shift/ignore.py | 39 ++ nuh_helper/date_shift/mappings.py | 80 +++- nuh_helper/date_shift/row_machine.py | 260 +++++++++++ nuh_helper/date_shift/stream_shift.py | 632 +++++--------------------- pyproject.toml | 1 + tests/test_hidden_dates.py | 2 +- tests/test_missing_id.py | 2 +- tests/test_shift_ignore.py | 6 +- tests/test_structural.py | 20 +- uv.lock | 2 + 15 files changed, 750 insertions(+), 607 deletions(-) create mode 100644 nuh_helper/date_shift/defs.py create mode 100644 nuh_helper/date_shift/ignore.py create mode 100644 nuh_helper/date_shift/row_machine.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 4911468..4efb4a9 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,5 +3,14 @@ "tests" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, + "cSpell.words": [ + "inplace", + "mapstream", + "mssing", + "openpyxl", + "ptid", + "pydatetime", + "ufeff" + ] } diff --git a/README.md b/README.md index f939371..d5703c5 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ Helper library for enabling data studies: utilities for study enablement such as - **`nuh_helper.date_shift`** — Date shifting for patient data in Excel/DataFrames (consistent shifts per patient ID, reproducible via linking tables). - **`nuh_helper.profile`** - Profile a dataset into a Scan Report +- **`nuh_helper.book_page`** - a library for manipulating `.xlsx` files and collections of `.csv` (or `.tsv`) files with an identical API -## Usage +## Usage - date_shift ### Date shifting (basic example) @@ -130,65 +131,79 @@ The function accepts the same parameters as `shift_excel_dates` except; - a per-sheet list of columns that have text only and shouldn't have dates - these need to be explicitly specified to prevent extra columns in the CDM -### Passing Non Dates (in `date_columns`) +### Allowing Non Dates (in `date_columns`) and Faux Dates (data that looks like dates in text columns) Studies frequently include data that's not parsable as a date in the date columns. -Rarely is this a typo, it can be text like `Record missing` or `2007-09-UN` to signify that information is only partially available. -This has previously been one or two dozen entries across hundreds of cells that need to be copied to the final output unchanged. -To accommodate this, each column in each sheet can have a fixed set of strings that are passed through as-is +This seems to change for each study, it has been text like "`_ missing _ _`" with spaces or incomplete values like `2007-09-UN` to signify that information is only partially available. +Studies may also include things like `120/80 mmHg` which can look like dates to the safety checks. + +To accommodate this, teh technician can specify a **shift_ignore** file when invoking `shift_excel_dates_inplace()` ```python -sheet_configs = { - "page-data": { - "patient_id_col": "pid", - "date_columns": ["dob"], - "text_columns": [ - "food", - ], - "header_row": 1, - "skip_rows_after_header": [], - "shift_ignore": { # the parameter is here - "dob": [ # any column can have "as is" values added - "missing", # the values are each listed here - ] - }, - } -} +from nuh_helper import shift_excel_dates_inplace + +shift_excel_dates_inplace( + input_file="input.xlsx", + output_file="output.xlsx", + + shift_ignore = "path/to/shift_ignore.yaml", + + patient_sheet="patients", + sheet_configs=sheet_configs, + seed=42, + linking_table_output="shift_mappings.csv", +) +``` +This is a simple `.yaml` file with `page_name: column_name: [ignore_entry, "ignored entry2"]` format like this ... + +```yaml +page-desc: + glitter: + - can't recall the date but on 12/11/2001 they had an itchy tummy + foo: + - agreed on 21st jul 2017 +page-data: + dob: + - mssing ``` -The intended workflow is ... +When running the shift, it's expected that the technician will ... -1. create sheet configurations -2. execute the date shifting and note non-date values detected -3. gradually build up the list of approved values per column +1. execute with an empty file +2. encounter a halt because of a value in the CDM +3. determine if it's safe to ignore or not and + - add add the value to the ignore list + - **or** remove, modify, or otherwise handle the value +4. gradually build up the list of approved values per column While this does require repeated manual intervention ... - It's faster than searching and restoring the fields manually - It doesn't sacrifice any control or the ability to inspect/detect problematic values +- It keeps any potential patient data out of the configuration file + - ... as the ignore list will presumably be deleted or stored with the linking_table.csv files ### Key parameters (date shifting) - `input_file`: Path to input Excel file - `output_file`: Path to output Excel file with shifted dates - `patient_sheet`: Name of the sheet containing patient IDs -- `patient_id_col`: Name of the column containing patient IDs -- `sheet_configs`: Dictionary mapping sheet names to configuration dicts, or, the string 'skip' if that sheet should be skipped but is a valid part of the CDM. - - `patient_id_col`: Patient ID column name in that sheet - - `date_columns`: List of date column names to shift - - `text_columns`: List of non-date columns to ignore - - `header_row`: (Optional) Zero-based row index for the row that contains column names - - `skip_rows_after_header`: (Optional) List of zero-based row indices to exclude from data (e.g. a data-type row immediately below the header) - - `shift_exceptions`: (Optional) Dict mapping column names to lists of date strings that should never be shifted (e.g. a fixed end-of-study date). Dates are parsed using the same flexible parser as regular date values. - - `shift_ignore`: (Optional) Dict mapping `page:{column:[values]}` to lists of values that `shift_excel_dates_inplace()` passes as-is with no manipulation or checking. -- `patient_header_row`: (Optional) Zero-based header row for the patient sheet (default: 0). If the patient sheet is in `sheet_configs`, that sheet’s `header_row` is used instead. -- `patient_skip_rows`: (Optional) Zero-based row indices to exclude from patient data (e.g. a data-type row). If the patient sheet is in `sheet_configs`, that sheet’s `skip_rows_after_header` is used instead. +- `patient_id_col`: Name of the column containing patient IDs (not used with `shift_excel_dates_inplace()`) +- `patient_header_row`: (Optional) Zero-based header row for the patient sheet (default: 0). If the patient sheet is in `sheet_configs`, that sheet’s `header_row` is used instead. (not used with `shift_excel_dates_inplace()`) +- `patient_skip_rows`: (Optional) Zero-based row indices to exclude from patient data (e.g. a data-type row). If the patient sheet is in `sheet_configs`, that sheet’s `skip_rows_after_header` is used instead. (not used with `shift_excel_dates_inplace()`) - `min_shift_days` / `max_shift_days`: Range of days to shift (default: -15 to 15) - `linking_table_path`: (Optional) Path to existing linking table CSV for reproducibility - `linking_table_output`: (Optional) Path to save the linking table CSV - `seed`: (Optional) Random seed for generating shifts - `date_format`: (Optional, `shift_excel_dates` only) Excel date format string (e.g., ‘YYYY-MM-DD’) +- `sheet_configs`: Dictionary mapping sheet names to configuration dicts, or, the string 'skip' if that sheet should be skipped but is a valid part of the CDM. + - `patient_id_col`: Patient ID column name in that sheet + - `date_columns`: List of date column names to shift + - `text_columns`: List of non-date columns to ignore + - `header_row`: (Optional) Zero-based row index for the row that contains column names (default to 0 with `shift_excel_dates_inplace()`) + - `skip_rows_after_header`: (Optional) List of zero-based row indices to exclude from data (e.g. a data-type row immediately below the header) (default to [] with `shift_excel_dates_inplace()`) + - `shift_exceptions`: (Optional) Dict mapping column names to lists of date strings that should never be shifted (e.g. a fixed end-of-study date). Dates are parsed using the same flexible parser as regular date values. (not allowed with `shift_excel_dates_inplace()`) ### Excel layout (header row and merged cells) @@ -207,8 +222,26 @@ Sheets can have a non-standard layout: e.g. a merged title row, then a descripti - Optional skip of rows after the header (e.g. data-type row) via `skip_rows_after_header` - Supports flexible date parsing (handles various formats and placeholders like "Unknown") - Reproducible shifts via linking tables +- `shift_excel_dates_inplace`: can date-shift `.csv` files by streaming the rows rather than loading the whole file. + +### Date Shifting `.csv` Files + +In most cases, the process for using `shift_excel_dates_inplace()` with `.csv` or `.tsv` files is identical to working with a whole `.xlsx` file. + +- `input_file: str | Path | list[Path]` is read as ... + 1. a `str` that's changed to a `Path` + 2. a `Path` (or a `str`) that's checked for `.tsv` and `.csv` files + 3. a `list[Path]` of `.csv` and `.tsv` files that are treated as workbook pages + - each must have a different stem; `foo.tsv` and `foo.csv` will cause an error +- `output_file =` is a folder where all of the final `.csv` pages will be placed + - if the folder/parents are missing;l they will be created + - it **must not** have any of the input files + - the original `.csv` or `.tsv` format/extension will be used for the output pages + +There is technically no requirement that all of the `.csv` files be in the same folder. +If you're date-shifting carrot-transform output, you will want to use the list of files and remove the `person_ids.tsv` and `summary_mapstream.tsv` files. -### Dataset Profile +## Usage - profile Profile a dataset and generate a Scan Report. diff --git a/nuh_helper/book_page/csv_book.py b/nuh_helper/book_page/csv_book.py index ad2e029..c558294 100644 --- a/nuh_helper/book_page/csv_book.py +++ b/nuh_helper/book_page/csv_book.py @@ -55,10 +55,23 @@ def stream_rows(self) -> gen[tuple[int, list[str]]]: def stream_copy(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: if not isinstance(into, Path): into = Path(into) + into = into.absolute() if into.is_file(): raise RuntimeError(f"can't write csvs to {into} because it's a file") + # don't overwrite input + if inputs_in_there := sorted( + [ + input + for input in self._page_files.values() + if str(input.absolute()).startswith(str(into)) + ] + ): + raise RuntimeError( + f"can't write to {into} - there are inputs there {inputs_in_there=}" + ) + into.mkdir(parents=True, exist_ok=True) for page in self: diff --git a/nuh_helper/date_shift/__init__.py b/nuh_helper/date_shift/__init__.py index a7b2d72..c05223d 100644 --- a/nuh_helper/date_shift/__init__.py +++ b/nuh_helper/date_shift/__init__.py @@ -18,16 +18,6 @@ load_shift_mappings, ) from nuh_helper.date_shift.stream_shift import ( - BlankColumnHasData, - DateColumnsMissing, - ExtraColumn, - ExtraPage, - HiddenDate, - PageMissing, - PatientColumnMissing, - ShiftFoundNonDate, - TextColumnsMissing, - UnknownPatient, shift_excel_dates_inplace, ) @@ -340,14 +330,4 @@ def shift_excel_dates( generate_shift_mappings, load_shift_mappings, shift_excel_dates_inplace, - HiddenDate, - PatientColumnMissing, - ShiftFoundNonDate, - BlankColumnHasData, - DateColumnsMissing, - ExtraColumn, - PageMissing, - ExtraPage, - TextColumnsMissing, - UnknownPatient, ] diff --git a/nuh_helper/date_shift/defs.py b/nuh_helper/date_shift/defs.py new file mode 100644 index 0000000..b8576ea --- /dev/null +++ b/nuh_helper/date_shift/defs.py @@ -0,0 +1,162 @@ +""" +collects the exceptions and sheet_configs_t. these are used across the package and need +to be in one "root" file. +""" + +from datetime import datetime + +# each entry in a sheet config will match this +_sheet_config_row_t = str | int | list[str] | dict[str, set[str] | list[str]] + +# a single sheet will consist of this config +_sheet_config_single_t = dict[str, _sheet_config_row_t] + +# the whole configuration for a shift workbook will have this type +sheet_configs_t = dict[str, _sheet_config_single_t] + + +class PatientColumnMissing(Exception): + """raised when the patient column is missing from any sheet""" + + def __init__(self, page_name: str, column_name: str) -> None: + message = f"The patient {column_name=} is not present on {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_name = column_name + + +class DateColumnsMissing(Exception): + def __init__(self, page_name: str, column_names: list[str]) -> None: + message = f"date {column_names=} is missing from the cdm {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_names = column_names + + +class TextColumnsMissing(Exception): + def __init__(self, page_name: str, column_names: list[str]) -> None: + message = f"text {column_names=} is missing from the cdm {page_name=}" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_names = column_names + + +class ExtraColumns(Exception): + """raised when unknown columns appear in a page we're shifting. + + could mean a column is named wrong, or, that the sheet_config is incomplete""" + + def __init__(self, page_name: str, column_names: list[str]) -> None: + message = f"{page_name=} has {column_names=} that're neither ignored or shifted" + super().__init__(message) + self._message = message + + self._page_name = page_name + self._column_names = column_names + + +class UnknownPatient(Exception): + def __init__(self, page: str, id: str) -> None: + message = f"Unknown {id=} on {page=}" + super().__init__(message) + self._message = message + + +class ShiftFoundNonDate(Exception): + def __init__( + self, page: str, row: int, col: int, col_name: str, value: str + ) -> None: + # the spaces and newline in the message make the value easier to copy/paste back + message = f"{page=}[{row}, {col} @ {col_name=} ]\n\t'{value}'" + super().__init__(message) + self._message: str = message + + self._page: str = page + self._row: int = row + self._col: int = col + self._col_name: str = col_name + self._value: str = value + + +class HiddenDate(ValueError): + def __init__( + self, + sheet_name: str, + row: int, + col: int, + col_name: str, + value: str, + found: datetime, + ) -> None: + message = ( + f"hidden date in [{sheet_name=}, {row}, {col} @ {col_name}] {value=} " + + f"// {found=}" + ) + super().__init__(message) + self._message = message + + self._sheet_name = sheet_name + self._row = row + self._col = col + self._col_name = col_name + self._value = value + self._found = found + + +class BlankColumnHasData(Exception): + """raised when a column with a blank name has data. + + prevents data being hidden in the wrong part of the CDM""" + + def __init__(self, page: str, row: int, col: int, value: any) -> None: + message = f"[{page=} @ {row}, {col}] is a blank column with data {value=}" + super().__init__(message) + self._message = message + + self._page: str = page + self._row: int = row + self._col: int = col + self._value: any = value + + +class FoundMissingPages(Exception): + def __init__(self, page_names: list[str]) -> None: + message = f"page missing from the source cdm {page_names=}" + super().__init__(message) + self._message = message + + self._page_name = page_names + + +class FoundExtraPages(Exception): + def __init__(self, page_names: list[str], expected: list[str]) -> None: + message = ( + f"unexpected page(s) found in the source cdm {page_names=} ({expected=})" + ) + super().__init__(message) + self._message = message + + self._page_names: list[str] = page_names + self._config_names: list[str] = expected + + +def normalize_inplace(row: list[str]) -> list[str]: + """modifies (and returns) the passed array to drop extra whitespace and replace + fully blank entries with None. doesn't touch non-strings""" + + for idx, val in enumerate(row): + if isinstance(val, str): + val = val.strip() + + if not val: + val = None + + row[idx] = val + + return row diff --git a/nuh_helper/date_shift/ignore.py b/nuh_helper/date_shift/ignore.py new file mode 100644 index 0000000..4ddeedf --- /dev/null +++ b/nuh_helper/date_shift/ignore.py @@ -0,0 +1,39 @@ +import logging +from pathlib import Path + +import yaml + +from nuh_helper.date_shift.defs import normalize_inplace + +shift_ignore_t = dict[str, dict[str, set[str]]] + +logger = logging.getLogger(__name__) + + +def shift_ignore_load(shift_ignore_yaml: None | Path | str) -> shift_ignore_t: + """load the ignore rules. if file or value is missing, just use {} + + - shift_ignore_yaml; path to the file or None. + """ + if shift_ignore_yaml is None: + return {} + + if not isinstance(shift_ignore_yaml, Path): + shift_ignore_yaml = Path(shift_ignore_yaml) + + if not shift_ignore_yaml.is_file(): + logger.warning(f"didn't find shift_ignore file {shift_ignore_yaml=}") + return {} + + with open(shift_ignore_yaml) as file: + data: dict = yaml.safe_load(file) + shift_ignore_full = {} + for page_name, page_data in data.items(): + shift_ignore_page = {} + + for col_name, col_ignored in page_data.items(): + shift_ignore_page[col_name.strip()] = set( + normalize_inplace(col_ignored + [None]) + ) + shift_ignore_full[page_name.strip()] = shift_ignore_page + return shift_ignore_full diff --git a/nuh_helper/date_shift/mappings.py b/nuh_helper/date_shift/mappings.py index 7bf628b..52a4f93 100644 --- a/nuh_helper/date_shift/mappings.py +++ b/nuh_helper/date_shift/mappings.py @@ -6,8 +6,16 @@ from pathlib import Path import pandas as pd +from pandas import Timedelta +from nuh_helper.book_page import Book from nuh_helper.date_shift import _parse +from nuh_helper.date_shift.defs import ( + sheet_configs_t, +) +from nuh_helper.date_shift.row_machine import ( + PatientRowStateMachine, +) logger = logging.getLogger(__name__) @@ -55,37 +63,63 @@ def load_shift_mappings(csv_path: str) -> pd.DataFrame: return df -def patient_shift_deltas( - patient_ids: list[str], - src_linking_table: str | Path, - out_linking_table: str | Path, - seed: int | None, +def load_shift_deltas( + book: Book, + patient_sheet: str, + sheet_configs: sheet_configs_t, min_shift_days: int, max_shift_days: int, -) -> dict[str, pd.Timedelta]: - """: - Generate random shift mappings for patient IDs, reusing known patients and saving - all to a file. - - Args: - patient_ids: List of patient IDs to generate shifts for. - src_linking_table: where to load prior values from - out_linking_table: where to save teh full set to - seed: Optional random seed for reproducibility. - min_shift_days: Minimum number of days to shift (default: -15). - max_shift_days: Maximum number of days to shift (default: 15). + load_linking_table: str | Path | None, + save_linking_table: str | Path | None, + seed: int | None, +) -> dict[str, Timedelta]: + """load the day-offsets for patients. + + Args + - book: Bool = the CSV or XLSX workbook to read the patient ids from + - patient_sheet: str = the authoritatively page name with a list of patients + * we assert that it exists in the book + * rows don't need to be unique + - sheet_configs: the full sheet configs + - min_shift_days = dates are shifted up to this far back (probably negative) + - max_shift_days = dates can be shifted at most this forward (should be > min) + - linking_table_path: str | None = None, + - linking_table_output: str | None = None, + - seed = an optional seed to pass to the shift generation Returns: dictionary mapping 'patient_id' to a pd.Timedelta 'shift_days' which we use. + """ + if not load_linking_table: + load_linking_table = None + elif not isinstance(load_linking_table, Path): + load_linking_table = Path(load_linking_table) + + if not save_linking_table: + save_linking_table = None + elif not isinstance(save_linking_table, Path): + save_linking_table = Path(save_linking_table) + + with book[patient_sheet] as page: + scanner = PatientRowStateMachine(sheet_configs) + patient_ids: list[str] = [ + pid.patient_id + for pid in [ + scanner.next_details(page.name, row, data) + for row, data in page.stream_rows() + ] + if pid + ] + # load the old ids (should we remove IDs that are "gone"?) - if isinstance(src_linking_table, str): - src_linking_table = Path(src_linking_table) - if not src_linking_table.is_file(): + if isinstance(load_linking_table, str): + load_linking_table = Path(load_linking_table) + if not load_linking_table.is_file(): old_mappings: dict[str, int] = {} else: - with open(src_linking_table) as file: + with open(load_linking_table) as file: old_mappings: dict[str, int] = { _parse._normalize_patient_id(row[0]): int(row[1]) for row in csv.reader(file) @@ -111,8 +145,8 @@ def patient_shift_deltas( ) # store all mappings in the out file - out_linking_table.parent.mkdir(parents=True, exist_ok=True) - with open(out_linking_table, "w") as file: + save_linking_table.parent.mkdir(parents=True, exist_ok=True) + with open(save_linking_table, "w") as file: csv.writer(file).writerows( [["patient_id", "shift_days"]] + [[key, all_mappings[key]] for key in sorted(all_mappings)] diff --git a/nuh_helper/date_shift/row_machine.py b/nuh_helper/date_shift/row_machine.py new file mode 100644 index 0000000..cee85e2 --- /dev/null +++ b/nuh_helper/date_shift/row_machine.py @@ -0,0 +1,260 @@ +"""implements a Finite State Machine to handle calls from the streaming. + +While the book_page API is meant to be generic, this is certainly not + + +""" + +import logging +from collections.abc import Generator as gen +from dataclasses import dataclass, replace +from enum import StrEnum +from pathlib import Path +from typing import Literal, cast + +import datefinder +from book_page import Book + +from nuh_helper.date_shift import _parse +from nuh_helper.date_shift.defs import ( + DateColumnsMissing, + ExtraColumns, + HiddenDate, + PatientColumnMissing, + TextColumnsMissing, + normalize_inplace, + sheet_configs_t, +) + +logger = logging.getLogger(__name__) + + +class ColumnType(StrEnum): + TEXT = "text" + DATE = "date" + IGNORE = "skip" + + +@dataclass +class PatientRow: + """carries the data needed to process a row from the sheet config and header_row""" + + page_name: str + col_names: tuple[str, ...] + patient_id_idx: int + patient_id_col: str + patient_id: str + date_columns: tuple[str, ...] + text_columns: tuple[str, ...] + row_idx: int + row_cells: list[str] + + def column_type( + self, + shift_ignore: dict[str, dict[str, set[None | str]]], + col_name: str, + cell_value: str, + ) -> bool: + """determine what we should do for a given column""" + + ignored: set[None | str] = shift_ignore.get(self.page_name, {}).get( + col_name, {None, ""} + ) + + return ( + ColumnType.IGNORE + if cell_value in ignored + else ColumnType.TEXT + if col_name in self.text_columns + else ColumnType.DATE + ) + + +class PatientRowStateMachine: + """builds upon the callbacks from book_page to handle row/page switching. does all + the checking for configs and what page we're on. needed because both patient IDs and + The Big Loop need the same checks""" + + def __init__(self, sheet_configs: sheet_configs_t) -> None: + self._sheet_configs: sheet_configs_t = sheet_configs + self._header_row: None | int = None + self._skip_rows: None | list[int] = None + self._detail_archetype: None | PatientRow = None + + def next_details( + self, sheet_name: str, row_idx: int, row_cells: list + ) -> PatientRow | Literal[False]: + """need to call this every step of the loop. it returns False to indicate that + this row doesn't need further processing, or a page-description that includes + the located patient id""" + + row_cells = normalize_inplace(row_cells) + + sheet_config: dict | str = self._sheet_configs[sheet_name] + + if row_idx == 0: + ### + # process the "which row is header and which do we skip" configuration + ## + + self._detail_archetype = None + + # do nothing to the skipped ones + if sheet_config == "skip": + return False + + self._header_row: int = cast(int, sheet_config.get("header_row", 0)) + self._skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) + + # check for holes in skip_rows (possible issue) + if self._skip_rows: + missing_skipped = [ + not_skipped + 1 + for not_skipped in range(self._header_row, max(self._skip_rows)) + if (not_skipped + 1) not in self._skip_rows + ] + if missing_skipped: + header_row = self._header_row + skip_rows = self._skip_rows + raise RuntimeError( + f"{sheet_name=} and {missing_skipped=} but" + + f" {header_row=} and {skip_rows=}" + ) + + # check if the config hard-codes patient data to skip + if "shift_exceptions" in sheet_config: + raise RuntimeError( + f"shift_exceptions was removed, update {sheet_name=}" + ) + if "shift_ignore" in sheet_config: + raise RuntimeError("move shift_ignore from sheet_configs to a file") + + if ( + (sheet_config == "skip") + or (row_idx < self._header_row) + or (row_idx in self._skip_rows) + ): + return False + + if row_idx == self._header_row: + ### + # now read the "which column" configuration + ## + + # grab the configuration + patient_id_col: str = str(sheet_config["patient_id_col"]).strip() + date_columns: list[str] = normalize_inplace(sheet_config["date_columns"]) + text_columns: list[str] = normalize_inplace(sheet_config["text_columns"]) + + # check patient_id_col isn't re-used as other column types + if patient_id_col in date_columns: + raise ValueError( + f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" + ) + if patient_id_col in text_columns: + raise ValueError( + f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" + ) + + # check that column names don't appear in both + overlapping_columns = [col for col in date_columns if col in text_columns] + if overlapping_columns: + raise ValueError( + f"{sheet_name=} has the some columns in both date and text " + + str(overlapping_columns) + ) + + # detect extra columns in this sheet + known = [patient_id_col] + date_columns + text_columns + if extra_columns := [col for col in row_cells if col and col not in known]: + raise ExtraColumns(sheet_name, extra_columns) + + # find patient_id_idx and check columns + patient_id_idx = None + for col_idx, col_name in enumerate(row_cells): + if not col_name: + continue + + # select the patient id + if col_name == patient_id_col: + if patient_id_idx is not None: + raise KeyError( + f"{sheet_name=} multiple columns {patient_id_col=}" + ) + patient_id_idx = col_idx + + # check to be sure we found the patient_id_col/patient_id_idx + if patient_id_idx is None: + raise PatientColumnMissing(sheet_name, patient_id_col) + + # check for columns in the config and not in the sheet + + if missing := [col for col in text_columns if col not in row_cells]: + raise TextColumnsMissing(sheet_name, missing) + + if missing := [col for col in date_columns if col not in row_cells]: + raise DateColumnsMissing(sheet_name, missing) + + # we've completed the header_row; save the archetype of the page detail + self._detail_archetype = PatientRow( + page_name=sheet_name, + col_names=row_cells, + patient_id_idx=patient_id_idx, + patient_id_col=patient_id_col, + date_columns=tuple(date_columns), + text_columns=tuple(text_columns), + # these three need to be replaced + patient_id="", + row_idx=-1, + row_cells=[], + ) + + # we don't shift anything in the header_row + return False + + # get the patient id for this row + patient_id = row_cells[self._detail_archetype.patient_id_idx] + + # check for date in the patient_id + for found in datefinder.find_dates(str(patient_id)): + raise HiddenDate( + sheet_name, + row_idx, + self._detail_archetype.patient_id_idx, + self._detail_archetype.patient_id_col, + patient_id, + found, + ) + + patient_id = _parse._normalize_patient_id(patient_id) + if patient_id is None: + # if the pid is None; the rest of the row should be None as well + if non_blank := [ + cell + for cell in row_cells + # keep the cells that aren't blank + if cell is not None and str(cell).strip() != "" + ]: + raise ValueError( + f"{row_idx=} has no pid, should be blank but has {non_blank}" + ) + + # we've completed the checks for a None pid + # ... so we need to skip the rest of this loop + return False + + # it's a valid pid - return a page detail object + # ... so that we don't need to examine this scanner + return replace( + self._detail_archetype, + patient_id=patient_id, + row_idx=row_idx, + row_cells=row_cells, + ) + + def book_copy(scanner, book: Book, output_file: Path) -> gen[PatientRow]: + for sheet_name, row_idx, row_cells in book.stream_copy(output_file): + # check if we're in a patient row, and, yield the row details if so + detail = scanner.next_details(sheet_name, row_idx, row_cells) + if detail: + yield detail diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index 0976b88..dfe5bad 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -1,157 +1,37 @@ import logging -from dataclasses import dataclass, replace from datetime import datetime -from enum import StrEnum from pathlib import Path -from typing import Any, Literal, cast +from typing import Any, cast import datefinder -import yaml +from pandas import Timedelta from nuh_helper.book_page import Book, book_open from nuh_helper.date_shift import _parse, mappings - - -class ColumnType(StrEnum): - TEXT = "text" - DATE = "date" - IGNORE = "skip" - +from nuh_helper.date_shift.defs import ( + BlankColumnHasData, + FoundExtraPages, + FoundMissingPages, + HiddenDate, + ShiftFoundNonDate, + UnknownPatient, + sheet_configs_t, +) +from nuh_helper.date_shift.ignore import shift_ignore_load, shift_ignore_t +from nuh_helper.date_shift.row_machine import ( + ColumnType, + PatientRow, + PatientRowStateMachine, +) logger = logging.getLogger(__name__) -class PatientColumnMissing(Exception): - """raised when the patient column is missing from any sheet""" - - def __init__(self, page_name: str, column_name: str) -> None: - message = f"The patient {column_name=} is not present on {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_name = column_name - - -class UnknownPatient(Exception): - def __init__(self, page: str, id: str) -> None: - message = f"Unknown {id=} on {page=}" - super().__init__(message) - self._message = message - - -class ShiftFoundNonDate(Exception): - def __init__( - self, page: str, row: int, col: int, col_name: str, value: str - ) -> None: - # the spaces and newline in the message make the value easier to copy/paste back - message = f"{page=}[{row}, {col} @ {col_name=} ]\n\t'{value}'" - super().__init__(message) - self._message: str = message - self._page: str = page - self._row: int = row - self._col: int = col - self._col_name: str = col_name - self._value: str = value - - -class HiddenDate(Exception): - def __init__( - self, - sheet_name: str, - row: int, - col: int, - col_name: str, - value: str, - found: datetime, - ) -> None: - message = ( - f"hidden date in [{sheet_name=}, {row}, {col} @ {col_name}] {value=} " - + f"// {found=}" - ) - super().__init__(message) - self._message = message - self._sheet_name = sheet_name - self._row = row - self._col = col - self._col_name = col_name - self._value = value - self._found = found - - -class DateColumnsMissing(Exception): - def __init__(self, page_name: str, column_names: list[str]) -> None: - message = f"date {column_names=} is missing from the cdm {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_names = column_names - - -class TextColumnsMissing(Exception): - def __init__(self, page_name: str, column_names: list[str]) -> None: - message = f"text {column_names=} is missing from the cdm {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_names = column_names - - -class BlankColumnHasData(Exception): - """raised when a column with a blank name has data. - - prevents data being hidden in the wrong part of the CDM""" - - def __init__(self, page: str, row: int, col: int, value: any) -> None: - message = f"[{page=} @ {row}, {col}] is a blank column with data {value=}" - super().__init__(message) - self._message = message - self._page: str = page - self._row: int = row - self._col: int = col - self._value: any = value - - -class ExtraColumn(Exception): - """raised when an unknown column appears in a page we're shifting. - - could mean a column is named wrong, or, that the sheet_config is incomplete""" - - def __init__(self, page_name: str, column_name: str) -> None: - message = ( - f"{column_name=} is neither ignored or shifted in the cdm {page_name=}" - ) - super().__init__(message) - self._message = message - - self._page_name = page_name - self._column_name = column_name - - -class PageMissing(Exception): - def __init__(self, page_name: str) -> None: - message = f"an expected page was missing from the source cdm {page_name=}" - super().__init__(message) - self._message = message - - self._page_name = page_name - - -class ExtraPage(Exception): - def __init__(self, page_name: str) -> None: - message = f"an unexpected page was found in the source cdm {page_name=}" - super().__init__(message) - self._message = message - self._page_name = page_name - - def shift_excel_dates_inplace( input_file: str | Path | list[Path], output_file: str | Path, patient_sheet: str, - sheet_configs: dict[str, dict[str, Any]], + sheet_configs: sheet_configs_t, min_shift_days: int = -15, max_shift_days: int = 15, linking_table_path: str | None = None, @@ -167,7 +47,7 @@ def shift_excel_dates_inplace( merged cells, column widths, conditional formatting, etc.) is preserved. Args: - input_file: Path to input Excel file. + input_file: Path to input Excel file, or, folder of .csv files, or list of .csv files output_file: Path for the output file (copy of input with shifted dates). patient_sheet: Name of the sheet containing patient IDs. sheet_configs: @@ -195,91 +75,38 @@ def shift_excel_dates_inplace( (Defaults to linking_table_path.parent / shift_ignore.do-not-commit.yaml) """ # noqa: E501 + logger.info( - "Shifting dates in-place: '%s' → '%s'", - input_file - if not isinstance(input_file, list) - else [file.name for file in input_file], - output_file, - ) - logger.debug( - "Shift range: %d to %d days, seed=%s", - min_shift_days, - max_shift_days, - seed, + f"Shifting dates in-place: '{input_file=}' -> '{output_file=}'" + + f"{min_shift_days=} {max_shift_days=} {seed=}" ) book: Book = book_open(input_file) logger.info(f"Book has {type(book)=}") - if not linking_table_path: - linking_table_path = None - elif not isinstance(linking_table_path, Path): - linking_table_path = Path(linking_table_path) - - if not linking_table_output: - linking_table_output = None - elif not isinstance(linking_table_output, Path): - linking_table_output = Path(linking_table_output) - - # get the shift_ignore: dict[str, dict[str, list[str]]] from shift_ignore_yaml: Path - if not shift_ignore_yaml: - if linking_table_path: - shift_ignore_yaml = ( - Path(linking_table_path).parent / "shift_ignore.do-not-commit.yaml" - ) - else: - raise RuntimeError( - "can't guess shift_ignore_yaml without linking_table_path" - ) - elif not isinstance(shift_ignore_yaml, Path): - shift_ignore_yaml = Path(shift_ignore_yaml) + # check for missing sheets + if sheet_names := [name for name in sheet_configs if name not in book]: + raise FoundMissingPages(sheet_names) - if not shift_ignore_yaml.is_file(): - shift_ignore: dict[str, dict[str, set[str]]] = {} - logger.warning(f"didn't find shift_ignore file {shift_ignore_yaml=}") - else: - with open(shift_ignore_yaml) as file: - data = yaml.safe_load(file) - shift_ignore: dict[str, dict[str, list[str]]] = { - page.strip(): { - col.strip(): [ - ignored.strip() if isinstance(ignored, str) else ignored - for ignored in data[page][col] - ] - + ["", None] - for col in data[page] - } - for page in data - } + # check that there aren't any extra pages in config + if sheet_names := [page.name for page in book if page.name not in sheet_configs]: + raise FoundExtraPages(sheet_names, list(sheet_configs)) - if patient_sheet not in sheet_configs: - raise PageMissing(patient_sheet) + # load the shift_ignore rules + shift_ignore: shift_ignore_t = shift_ignore_load(shift_ignore_yaml) - with book[patient_sheet] as page: - scanner = PatientRowStateMachine(sheet_configs) - shift_deltas = mappings.patient_shift_deltas( - [ - pid.patient_id - for pid in [ - scanner.next_details(page.name, row, data) - for row, data in page.stream_rows() - ] - if pid - ], - linking_table_path, - linking_table_output or "shift_mappings.csv", - seed, - min_shift_days, - max_shift_days, - ) - scanner = None - logger.info(f"loaded deltas {len(shift_deltas)=}") + shift_deltas = mappings.load_shift_deltas( + book, + patient_sheet, + sheet_configs, + min_shift_days, + max_shift_days, + load_linking_table=linking_table_path, + save_linking_table=linking_table_output, + seed=seed, + ) - # check for missing sheets - for sheet_name in sheet_configs: - if sheet_name not in book: - raise PageMissing(sheet_name) + logger.info("loaded shift_ignore and shift_deltas") ## # this calls the `.stream_copy()` method to copy from one book to another @@ -290,312 +117,91 @@ def shift_excel_dates_inplace( # - no option to add/remove rows is available # - no option to jump/seek through rows is available # - ... which is why this acts a bit weird - scanner = PatientRowStateMachine(sheet_configs) - for sheet_name, row_idx, row_cells in book.stream_copy(output_file): - # check if we're in a patient row - # ... the method will raise exceptions if something is wrong or mis-configured - detail = scanner.next_details(sheet_name, row_idx, row_cells) - if not detail: - continue - + for detail in PatientRowStateMachine(sheet_configs).book_copy(book, output_file): # determine how many days to shift - shift_delta = shift_deltas.get(detail.patient_id) + shift_delta: Timedelta = shift_deltas.get(detail.patient_id) if shift_delta is None: # all patients need to have a shift value - raise UnknownPatient(sheet_name, detail.patient_id) - - # now scan each column - # ... even when page has no date_columns; still check for hidden dates - # for col_idx in range(len(row_cells)): - for col_idx, cell_value in enumerate(row_cells): - # skip the patient id column (it was already checked anyway) - if col_idx == detail.patient_id_idx: - continue - - # get the cell value. replace it with None if it's just whitespace - if isinstance(cell_value, str): - cell_value = cell_value.strip() - - if not cell_value: - cell_value = None - - col_name = detail.col_names[col_idx] - - # with no column name; this column of the row should have been None - if cell_value and not col_name: - raise BlankColumnHasData(sheet_name, row_idx, col_idx, cell_value) - - # choose what to do based on the column/cell type - match detail.column_type(shift_ignore, col_name, cell_value): - case ColumnType.IGNORE: - # "skip" is sheet-wide, but, ignore is for a single cell of data - pass - case ColumnType.TEXT: - # check for dates in non-date columns - for found in datefinder.find_dates(str(cell_value)): - logger.debug( - f"found hidden date {sheet_name=}/{col_name=}" - + f":{cell_value}" - ) - raise HiddenDate( - sheet_name, - row_idx, - col_idx, - col_name, - cell_value, - found, - ) - case ColumnType.DATE: - # cell_value can be str | datetime | date - # ... so parsed might not always succeed - parsed = _parse._parse_date_value(cell_value) - if parsed is None: - # raise an error; we already handled ignored values - raise ShiftFoundNonDate( - sheet_name, row_idx, col_idx, col_name, cell_value - ) - - # perform the actual shifting - shifted = parsed + shift_delta - - # would might be nice to always drop the 00:00:00 - if isinstance(cell_value, datetime): - cell_value = cast(Any, shifted.to_pydatetime()) - else: - cell_value = cast(Any, shifted.to_pydatetime().date()) - row_cells[col_idx] = cell_value - - logger.info("Output written to '%s'", output_file) - - -@dataclass -class PatientPageDetail: - """carries the data needed to process a row from the sheet config and header_row""" - - page_name: str - col_names: tuple[str, ...] - patient_id_idx: int - patient_id_col: str - patient_id: str - date_columns: tuple[str, ...] - text_columns: tuple[str, ...] - - def column_type( - self, - shift_ignore: dict[str, dict[str, set[None | str]]], - col_name: str, - cell_value: str, - ) -> bool: - """determine what we should do for a given column""" - - ignored: set[None | str] = shift_ignore.get(self.page_name, {}).get( - col_name, {None, ""} - ) - - return ( - ColumnType.IGNORE - if cell_value in ignored - else ColumnType.TEXT - if col_name in self.text_columns - else ColumnType.DATE - ) - - -class PatientRowStateMachine: - """builds upon the callbacks from book_page to handle row/page switching. does all - the checking for configs and what page we're on. needed because both patient IDs and - The Big Loop need the same checks""" - - def __init__(self, sheet_configs: dict[str, dict]) -> None: - self._sheet_configs = sheet_configs - self._header_row: None | int = None - self._skip_rows: None | list[int] = None - self._page_detail: None | PatientPageDetail = None - - def next_details( - self, sheet_name: str, row_idx: int, row_cells: list - ) -> PatientPageDetail | Literal[False]: - """need to call this every step of the loop. it returns False to indicate that - this row doesn't need further processing, or a page-description that includes - the located patient id""" - - # check that it's in the configs - if sheet_name not in self._sheet_configs: - raise ExtraPage(sheet_name) - - sheet_config: dict | str = self._sheet_configs[sheet_name] - - if row_idx == 0: - ### - # process the "which row is header and which do we skip" configuration - ## - - self._page_detail = None - - # do nothing to the skipped ones - if sheet_config == "skip": - return False - - self._header_row: int = cast(int, sheet_config.get("header_row", 0)) - self._skip_rows: list[int] = sheet_config.get("skip_rows_after_header", []) - - # check for holes in skip_rows (possible issue) - if self._skip_rows: - missing_skipped = [ - not_skipped + 1 - for not_skipped in range(self._header_row, max(self._skip_rows)) - if (not_skipped + 1) not in self._skip_rows - ] - if missing_skipped: - header_row = self._header_row - skip_rows = self._skip_rows - raise RuntimeError( - f"{sheet_name=} and {missing_skipped=} but" - + f" {header_row=} and {skip_rows=}" - ) - - # check if the config hard-codes patient data to skip - if "shift_exceptions" in sheet_config: - raise RuntimeError( - f"shift_exceptions was removed, update {sheet_name=}" - ) - if "shift_ignore" in sheet_config: - raise RuntimeError("move shift_ignore from sheet_configs to a file") - - if ( - (sheet_config == "skip") - or (row_idx < self._header_row) - or (row_idx in self._skip_rows) - ): - return False - - if row_idx == self._header_row: - ### - # now read the "which column" configuration - ## - - # grab the configuration - patient_id_col: str = str(sheet_config["patient_id_col"]).strip() - date_columns: list[str] = normalize_inplace(sheet_config["date_columns"]) - text_columns: list[str] = normalize_inplace(sheet_config["text_columns"]) + raise UnknownPatient(detail.page_name, detail.patient_id) + + # process each cell of this row using the subroutine + for col_idx, cell_value in enumerate(detail.row_cells): + shifted_cell = _shift_cell( + cell_value=cell_value, + col_idx=col_idx, + detail=detail, + shift_delta=shift_delta, + shift_ignore=shift_ignore, + ) - # check patient_id_col isn't re-used as other column types - if patient_id_col in date_columns: - raise ValueError( - f"{patient_id_col=} shouldn't be in date_columns of {sheet_name=}" + if str(shifted_cell) != str(cell_value): + detail.row_cells[col_idx] = shifted_cell + + logger.info(f"Output written {output_file=}") + + +def _shift_cell( + cell_value: str | object | None, + col_idx: int, + detail: PatientRow, + shift_delta: Timedelta, + shift_ignore: shift_ignore_t, +) -> str: + # skip the patient id column (it was already checked anyway) + if col_idx == detail.patient_id_idx: + return detail.patient_id + + # get the cell value. replace it with None if it's just whitespace + if isinstance(cell_value, str): + cell_value = cell_value.strip() + + if cell_value is None: + return None + + col_name = detail.col_names[col_idx] + + # with no column name; this column of the row should have been None + if not col_name: + raise BlankColumnHasData(detail.page_name, detail.row_idx, col_idx, cell_value) + + # choose what to do based on the column/cell type + match detail.column_type(shift_ignore, col_name, cell_value): + case ColumnType.IGNORE: + # "skip" is sheet-wide, but, ignore is for a single cell of data + # ... so we're checking for the ignore command here + pass + case ColumnType.TEXT: + # check for dates in non-date columns + for found in datefinder.find_dates(str(cell_value)): + logger.debug( + f"found hidden date {detail.page_name=}/{col_name=}" + + f":{cell_value}" ) - if patient_id_col in text_columns: - raise ValueError( - f"{patient_id_col=} shouldn't be in text_columns of {sheet_name=}" + raise HiddenDate( + detail.page_name, + detail.row_idx, + col_idx, + col_name, + cell_value, + found, ) - - # check that column names don't appear in both - overlapping_columns = [col for col in date_columns if col in text_columns] - if overlapping_columns: - raise ValueError( - f"{sheet_name=} has the some columns in both date and text " - + str(overlapping_columns) + case ColumnType.DATE: + # cell_value can be str | datetime | date + # ... so parsed might not always succeed + parsed = _parse._parse_date_value(cell_value) + if parsed is None: + # raise an error; we already handled ignored values + raise ShiftFoundNonDate( + detail.page_name, detail.row_idx, col_idx, col_name, cell_value ) - # normalize the column names - col_names = normalize_inplace(row_cells) - - # find patient_id_idx and check columns - patient_id_idx = None - for col_idx, col_name in enumerate(col_names): - if not col_name: - continue - - # check the config to see if we know what to do with this column - if col_name not in ([patient_id_col] + date_columns + text_columns): - raise ExtraColumn(sheet_name, col_name) - - # select the patient id - if col_name == patient_id_col: - if patient_id_idx is not None: - raise KeyError( - f"{sheet_name=} multiple columns {patient_id_col=}" - ) - patient_id_idx = col_idx - - # check to be sure we found the patient_id_col/patient_id_idx - if patient_id_idx is None: - raise PatientColumnMissing(sheet_name, patient_id_col) - - # check for columns in the config and not in the sheet - missing = [col for col in text_columns if col not in col_names] - if missing: - raise TextColumnsMissing(sheet_name, missing) - - missing = [col for col in date_columns if col not in col_names] - if missing: - raise DateColumnsMissing(sheet_name, missing) - - # we've completed the header_row; save the archetype of the page detail - self._page_detail = PatientPageDetail( - page_name=sheet_name, - col_names=col_names, - patient_id_idx=patient_id_idx, - patient_id_col=patient_id_col, - patient_id="
", - date_columns=tuple(date_columns), - text_columns=tuple(text_columns), - ) - - # we don't shift anything in the header_row - return False - - # get the patient id for this row - patient_id = row_cells[self._page_detail.patient_id_idx] - - # check for date in the patient_id - for found in datefinder.find_dates(str(patient_id)): - raise HiddenDate( - sheet_name, - row_idx, - self._page_detail.patient_id_idx, - self._page_detail.patient_id_col, - patient_id, - found, - ) - - patient_id = _parse._normalize_patient_id(patient_id) - if patient_id is None: - # if the pid is None; the rest of the row should be None as well - non_blank = [ - cell - for cell in row_cells - # keep the cells that aren't blank - if cell is not None and str(cell).strip() != "" - ] - if non_blank: - raise ValueError( - f"{row_idx=} has no pid, should be blank but has {non_blank}" - ) - - # we've completed the checks for a None pid - # ... so we need to skip the rest of this loop - return False - - # it's a valid pid - return a page detail object - # ... so that we don't need to examine this scanner - return replace( - self._page_detail, - patient_id=patient_id, - ) - - -def normalize_inplace(row: list[str]) -> list[str]: - """modifies (and returns) the passed array to drop extra whitespace and replace - fully blank entries with None. doesn't touch non-strings""" - - for idx, val in enumerate(row): - if isinstance(val, str): - val = val.strip() - - if not val: - val = None + # perform the actual shifting + shifted = parsed + shift_delta - row[idx] = val + # TODO; allow column/sheet/workbook format settings + if isinstance(cell_value, datetime): + return cast(Any, shifted.to_pydatetime()) + else: + return cast(Any, shifted.to_pydatetime().date()) - return row + return cell_value diff --git a/pyproject.toml b/pyproject.toml index 624e056..02940fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ dependencies = [ "pandas>=3.0.5", "openpyxl>=3.1.0", "datefinder>=1.0.0", + "pyyaml>=6.0.3", ] [tool.hatch.metadata] allow-direct-references = true diff --git a/tests/test_hidden_dates.py b/tests/test_hidden_dates.py index e3ed551..d4a5ea8 100644 --- a/tests/test_hidden_dates.py +++ b/tests/test_hidden_dates.py @@ -4,7 +4,7 @@ import pytest from nuh_helper import shift_excel_dates_inplace -from nuh_helper.date_shift import ( +from nuh_helper.date_shift.defs import ( HiddenDate, ) diff --git a/tests/test_missing_id.py b/tests/test_missing_id.py index 1b9427c..412ea39 100644 --- a/tests/test_missing_id.py +++ b/tests/test_missing_id.py @@ -3,7 +3,7 @@ import pytest from nuh_helper import shift_excel_dates_inplace -from nuh_helper.date_shift import UnknownPatient +from nuh_helper.date_shift.defs import UnknownPatient test_data = ( # i feel like test data should be in a folder following this pattern diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 6b9bd04..d38ca45 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -3,7 +3,11 @@ import pytest from nuh_helper import book_page, shift_excel_dates_inplace -from nuh_helper.date_shift import HiddenDate, PatientColumnMissing, ShiftFoundNonDate +from nuh_helper.date_shift.defs import ( + HiddenDate, + PatientColumnMissing, + ShiftFoundNonDate, +) test_data = Path(__file__).parent / "data/shift_ignore" diff --git a/tests/test_structural.py b/tests/test_structural.py index c627934..f7cc9ea 100644 --- a/tests/test_structural.py +++ b/tests/test_structural.py @@ -3,12 +3,12 @@ import pytest from nuh_helper import shift_excel_dates_inplace -from nuh_helper.date_shift import ( +from nuh_helper.date_shift.defs import ( BlankColumnHasData, DateColumnsMissing, - ExtraColumn, - ExtraPage, - PageMissing, + ExtraColumns, + FoundExtraPages, + FoundMissingPages, TextColumnsMissing, ) @@ -120,7 +120,7 @@ def test_extra_column(tmp_path: Path) -> None: }, "stuff": "skip", } - with pytest.raises(ExtraColumn) as info: + with pytest.raises(ExtraColumns) as info: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -134,7 +134,7 @@ def test_extra_column(tmp_path: Path) -> None: ) assert info.value._page_name == "paige" - assert info.value._column_name == "food" + assert info.value._column_names == ["food"] def test_text_column_missing(tmp_path: Path) -> None: @@ -208,7 +208,7 @@ def test_page_missing(tmp_path: Path) -> None: "skip_rows_after_header": [], }, } - with pytest.raises(PageMissing) as info: + with pytest.raises(FoundMissingPages) as info: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -221,7 +221,7 @@ def test_page_missing(tmp_path: Path) -> None: linking_table_output=str(linking_table_out), ) - assert info.value._page_name == "a-missing-page" + assert info.value._page_name == ["a-missing-page"] def test_page_extra(tmp_path: Path) -> None: @@ -245,7 +245,7 @@ def test_page_extra(tmp_path: Path) -> None: "skip_rows_after_header": [], }, } - with pytest.raises(ExtraPage) as info: + with pytest.raises(FoundExtraPages) as info: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -258,4 +258,4 @@ def test_page_extra(tmp_path: Path) -> None: linking_table_output=str(linking_table_out), ) - assert info.value._page_name == "stuff" + assert info.value._page_names == ["stuff"] diff --git a/uv.lock b/uv.lock index a0479d1..e8714f9 100644 --- a/uv.lock +++ b/uv.lock @@ -197,6 +197,7 @@ dependencies = [ { name = "datefinder" }, { name = "openpyxl" }, { name = "pandas" }, + { name = "pyyaml" }, ] [package.dev-dependencies] @@ -216,6 +217,7 @@ requires-dist = [ { name = "datefinder", specifier = ">=1.0.0" }, { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pandas", specifier = ">=3.0.5" }, + { name = "pyyaml", specifier = ">=6.0.3" }, ] [package.metadata.requires-dev] From e15b3c5364367630651652f79de7b2922aeee5b2 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Wed, 14 Oct 2026 20:29:02 +0100 Subject: [PATCH 69/82] csv and excel workbooks now both try to parse dates to be consistent during unit testing --- nuh_helper/book_page/__init__.py | 4 +- nuh_helper/book_page/book.py | 96 +++++++++++++++++- nuh_helper/book_page/csv_book.py | 38 +++---- nuh_helper/book_page/xlsx_book.py | 4 +- nuh_helper/date_shift/defs.py | 16 --- nuh_helper/date_shift/ignore.py | 4 +- nuh_helper/date_shift/mappings.py | 40 ++++---- nuh_helper/date_shift/row_machine.py | 14 +-- nuh_helper/date_shift/stream_shift.py | 4 +- .../offsets.csv | 0 .../page-data.csv | 0 .../page-desc.csv | 0 .../shift-ignore.yaml | 0 .../workbook.xlsx | Bin tests/test_missing_id.py | 11 +- tests/test_profile.py | 4 +- tests/test_shift_ignore.py | 65 +++++------- 17 files changed, 170 insertions(+), 130 deletions(-) rename tests/data/{shift_ignore => test_shift_ignore}/offsets.csv (100%) rename tests/data/{shift_ignore => test_shift_ignore}/page-data.csv (100%) rename tests/data/{shift_ignore => test_shift_ignore}/page-desc.csv (100%) rename tests/data/{shift_ignore => test_shift_ignore}/shift-ignore.yaml (100%) rename tests/data/{shift_ignore => test_shift_ignore}/workbook.xlsx (100%) diff --git a/nuh_helper/book_page/__init__.py b/nuh_helper/book_page/__init__.py index 67093f1..20233fc 100644 --- a/nuh_helper/book_page/__init__.py +++ b/nuh_helper/book_page/__init__.py @@ -1,5 +1,6 @@ +from nuh_helper.book_page.book import Book, Page, normalize_row from nuh_helper.book_page.csv_book import CSVBook -from nuh_helper.book_page.generic import Book, Page, book_open +from nuh_helper.book_page.generic import book_open from nuh_helper.book_page.xlsx_book import XLSXBook __all__ = [ # noqa: PLE0604 @@ -8,4 +9,5 @@ Book, Page, book_open, + normalize_row, ] diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index 22988c6..3b167bd 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -1,9 +1,13 @@ from abc import ABC, abstractmethod from collections.abc import Generator as gen from collections.abc import Iterable as iterator +from datetime import date, datetime from functools import cached_property from pathlib import Path +cell_val_t = None | str | date | datetime +row_val_t = list[cell_val_t] + class Page(ABC): def __init__(self, name: str, page: object) -> None: @@ -11,7 +15,18 @@ def __init__(self, name: str, page: object) -> None: self._page = page @abstractmethod - def stream_rows(self) -> gen[tuple[int, list[str]]]: ... + def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: + """performs the workbook specific yield/write of the row data""" + ... + + def stream_rows(self) -> gen[tuple[int, row_val_t]]: + """public method for reading a RO stream of rows from a specific page""" + + for row_idx, row_data in self._stream_rows_impl(): + for raw_idx, raw_val in enumerate(row_data): + row_data[raw_idx] = normalize_val(raw_val) + + yield row_idx, row_data @cached_property def name(self) -> str: @@ -29,9 +44,86 @@ def __getitem__(self, name: str) -> gen[Page]: ... def keys(self) -> iterator[str]: ... @abstractmethod - def stream_copy(self, into: Path) -> gen[tuple[str, int, list[str]]]: ... + def _stream_copy_impl(self, into: Path) -> gen[tuple[str, int, row_val_t]]: + """performs the workbook specific yield/write of the row data""" + ... + + def stream_copy( + self, + into: str | Path, + date_format: None | str = None, + time_format: None | str = None, + ) -> gen[tuple[str, int, row_val_t]]: + """public method for copying a stream of rows between two workbooks""" + + into = (into if isinstance(into, Path) else Path(into)).absolute() + + time_format = ( + time_format + if time_format + else (date_format if date_format else "%Y-%m-%d %H:%M:%S") + ) + date_format = date_format if date_format else "%Y-%m-%d" + + for page_name, row_idx, row_data in self._stream_copy_impl(into): + yield page_name, row_idx, row_data + for raw_idx, raw_val in enumerate(row_data): + if isinstance(raw_val, date): + row_data[raw_idx] = raw_val.strftime(date_format) + elif isinstance(raw_val, datetime): + row_data[raw_idx] = raw_val.strftime(time_format) def __iter__(self) -> gen[Page]: for name in self.keys(): with self[name] as page: yield page + + +def normalize_val(src: object) -> object: + """parse to a datetime, .strip(), turn "" into None, retain bool""" + + if isinstance(src, (bool | float | int | datetime | date)): + return src + + if isinstance(src, str): + src = src.strip() + + if not src: + return None + + if isinstance(src, str): + # Try a handful of common formats, including YYYY-DD-MM found in some feeds + for fmt in ("%Y-%m-%d", "%Y-%d-%m", "%d-%m-%Y", "%m-%d-%Y"): + try: + return datetime.strptime(src, fmt) + except (TypeError, ValueError): + # If parsing with this specific format fails, silently try next format. + pass + + # Fallback: try ISO 8601 parsing (handles many ambiguous/edge cases) + try: + return datetime.fromisoformat(src) + except (TypeError, ValueError): + pass + + # Last resort: try a few more relaxed formats + for fmt in ( + "%Y/%m/%d", + "%d/%m/%Y", + "%m/%d/%Y", + "%Y-%m-%dT%H:%M:%S", + "%Y-%m-%d %H:%M:%S", + ): + try: + return datetime.strptime(src, fmt) + except (TypeError, ValueError): + pass + + return src + + +def normalize_row(row: list[str]) -> list[str]: + """modifies (and returns) the passed array to drop extra whitespace and replace + fully blank entries with None. doesn't touch non-strings""" + + return [normalize_val(val) for val in row] diff --git a/nuh_helper/book_page/csv_book.py b/nuh_helper/book_page/csv_book.py index c558294..9eabc3c 100644 --- a/nuh_helper/book_page/csv_book.py +++ b/nuh_helper/book_page/csv_book.py @@ -4,12 +4,14 @@ from collections.abc import Generator as gen from pathlib import Path -from nuh_helper.book_page.book import Book, Page +from nuh_helper.book_page.book import Book, Page, normalize_row, row_val_t class CSVBook(Book): def __init__(self, path: str | Path | list[Path]) -> None: - self._page_files = {page.stem.lower(): page for page in csv_files(path)} + self._page_files = { + page.stem.lower(): page.absolute() for page in csv_files(path) + } def keys(self) -> dict_keys: return self._page_files.keys() @@ -18,7 +20,7 @@ def keys(self) -> dict_keys: def __getitem__(self, name: str) -> gen[Page]: class CSVPage(Page): - def stream_rows(self) -> gen[tuple[int, list[str]]]: + def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: assert isinstance(self._page, Path) with self._page.open() as file: for row, line in enumerate( @@ -27,15 +29,6 @@ def stream_rows(self) -> gen[tuple[int, list[str]]]: delimiter="\t" if file.name.endswith(".tsv") else ",", ) ): - # strip cell strings - line = [ - cell.strip() if isinstance(cell, str) else cell - for cell in line - ] - - # convert empty cells to None - line = [cell if cell else None for cell in line] - # strip bom if ( row == 0 @@ -44,18 +37,15 @@ def stream_rows(self) -> gen[tuple[int, list[str]]]: ): line[0] = line[0][1:] - yield ( - row, - line, - ) + # strip cell strings, convert empty cells to None + line = normalize_row(line) + + yield (row, line) assert isinstance(self._page_files[name], Path) yield CSVPage(name, self._page_files[name]) - def stream_copy(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: - if not isinstance(into, Path): - into = Path(into) - into = into.absolute() + def _stream_copy_impl(self, into: Path) -> gen[tuple[str, int, row_val_t]]: if into.is_file(): raise RuntimeError(f"can't write csvs to {into} because it's a file") @@ -65,7 +55,7 @@ def stream_copy(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: [ input for input in self._page_files.values() - if str(input.absolute()).startswith(str(into)) + if str(input).startswith(str(into)) ] ): raise RuntimeError( @@ -82,9 +72,9 @@ def stream_copy(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: data, delimiter="\t" if file.endswith(".tsv") else "," ) - for row, cell in page.stream_rows(): - yield name, row, cell - writer.writerow(cell) + for row_idx, row_cells in page.stream_rows(): + yield name, row_idx, row_cells + writer.writerow(row_cells) def csv_files(path: str | Path | list[Path]) -> list[Path]: diff --git a/nuh_helper/book_page/xlsx_book.py b/nuh_helper/book_page/xlsx_book.py index 2ed363f..e274d18 100644 --- a/nuh_helper/book_page/xlsx_book.py +++ b/nuh_helper/book_page/xlsx_book.py @@ -20,7 +20,7 @@ def __init__(self, file: str | Path, read_only: bool = True) -> None: def keys(self) -> list[str]: return [page.title for page in self._book.worksheets] - def stream_copy(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: + def _stream_copy_impl(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: if not isinstance(into, Path): into = Path(into) @@ -48,7 +48,7 @@ def __getitem__(self, name: str) -> gen[Page]: raise KeyError("excel limits you to 31 character page names") class XLSXPage(Page): - def stream_rows(self) -> gen[tuple[int, list[str]]]: + def _stream_rows_impl(self) -> gen[tuple[int, list[str]]]: for row in range(self._page.max_row): yield ( row, diff --git a/nuh_helper/date_shift/defs.py b/nuh_helper/date_shift/defs.py index b8576ea..ceb6f88 100644 --- a/nuh_helper/date_shift/defs.py +++ b/nuh_helper/date_shift/defs.py @@ -144,19 +144,3 @@ def __init__(self, page_names: list[str], expected: list[str]) -> None: self._page_names: list[str] = page_names self._config_names: list[str] = expected - - -def normalize_inplace(row: list[str]) -> list[str]: - """modifies (and returns) the passed array to drop extra whitespace and replace - fully blank entries with None. doesn't touch non-strings""" - - for idx, val in enumerate(row): - if isinstance(val, str): - val = val.strip() - - if not val: - val = None - - row[idx] = val - - return row diff --git a/nuh_helper/date_shift/ignore.py b/nuh_helper/date_shift/ignore.py index 4ddeedf..e14449b 100644 --- a/nuh_helper/date_shift/ignore.py +++ b/nuh_helper/date_shift/ignore.py @@ -3,7 +3,7 @@ import yaml -from nuh_helper.date_shift.defs import normalize_inplace +from nuh_helper.book_page import normalize_row shift_ignore_t = dict[str, dict[str, set[str]]] @@ -33,7 +33,7 @@ def shift_ignore_load(shift_ignore_yaml: None | Path | str) -> shift_ignore_t: for col_name, col_ignored in page_data.items(): shift_ignore_page[col_name.strip()] = set( - normalize_inplace(col_ignored + [None]) + normalize_row(col_ignored + [None]) ) shift_ignore_full[page_name.strip()] = shift_ignore_page return shift_ignore_full diff --git a/nuh_helper/date_shift/mappings.py b/nuh_helper/date_shift/mappings.py index 52a4f93..f7b0212 100644 --- a/nuh_helper/date_shift/mappings.py +++ b/nuh_helper/date_shift/mappings.py @@ -69,8 +69,8 @@ def load_shift_deltas( sheet_configs: sheet_configs_t, min_shift_days: int, max_shift_days: int, - load_linking_table: str | Path | None, - save_linking_table: str | Path | None, + old_linking_table: str | Path | None, + out_linking_table: str | Path | None, seed: int | None, ) -> dict[str, Timedelta]: """load the day-offsets for patients. @@ -92,15 +92,15 @@ def load_shift_deltas( """ - if not load_linking_table: - load_linking_table = None - elif not isinstance(load_linking_table, Path): - load_linking_table = Path(load_linking_table) + if not old_linking_table: + old_linking_table = None + elif not isinstance(old_linking_table, Path): + old_linking_table = Path(old_linking_table) - if not save_linking_table: - save_linking_table = None - elif not isinstance(save_linking_table, Path): - save_linking_table = Path(save_linking_table) + if not out_linking_table: + out_linking_table = None + elif not isinstance(out_linking_table, Path): + out_linking_table = Path(out_linking_table) with book[patient_sheet] as page: scanner = PatientRowStateMachine(sheet_configs) @@ -114,17 +114,14 @@ def load_shift_deltas( ] # load the old ids (should we remove IDs that are "gone"?) - if isinstance(load_linking_table, str): - load_linking_table = Path(load_linking_table) - if not load_linking_table.is_file(): + if not old_linking_table or not old_linking_table.is_file(): old_mappings: dict[str, int] = {} else: - with open(load_linking_table) as file: + with open(old_linking_table) as file: old_mappings: dict[str, int] = { _parse._normalize_patient_id(row[0]): int(row[1]) for row in csv.reader(file) if row and row[1] and row[1].strip() != "shift_days" - # if row and row[0] and row[0].strip() != "patient_id" } if seed: @@ -145,11 +142,12 @@ def load_shift_deltas( ) # store all mappings in the out file - save_linking_table.parent.mkdir(parents=True, exist_ok=True) - with open(save_linking_table, "w") as file: - csv.writer(file).writerows( - [["patient_id", "shift_days"]] - + [[key, all_mappings[key]] for key in sorted(all_mappings)] - ) + if out_linking_table: + out_linking_table.parent.mkdir(parents=True, exist_ok=True) + with open(out_linking_table, "w") as file: + csv.writer(file).writerows( + [["patient_id", "shift_days"]] + + [[key, all_mappings[key]] for key in sorted(all_mappings)] + ) return {k: pd.Timedelta(days=all_mappings[k]) for k in all_mappings} diff --git a/nuh_helper/date_shift/row_machine.py b/nuh_helper/date_shift/row_machine.py index cee85e2..c2a5cb3 100644 --- a/nuh_helper/date_shift/row_machine.py +++ b/nuh_helper/date_shift/row_machine.py @@ -1,8 +1,7 @@ """implements a Finite State Machine to handle calls from the streaming. -While the book_page API is meant to be generic, this is certainly not - - +While the book_page API is meant to be generic, this is certainly not - so it's part of +the shift module. """ import logging @@ -13,8 +12,8 @@ from typing import Literal, cast import datefinder -from book_page import Book +from nuh_helper.book_page import Book, normalize_row from nuh_helper.date_shift import _parse from nuh_helper.date_shift.defs import ( DateColumnsMissing, @@ -22,7 +21,6 @@ HiddenDate, PatientColumnMissing, TextColumnsMissing, - normalize_inplace, sheet_configs_t, ) @@ -88,8 +86,6 @@ def next_details( this row doesn't need further processing, or a page-description that includes the located patient id""" - row_cells = normalize_inplace(row_cells) - sheet_config: dict | str = self._sheet_configs[sheet_name] if row_idx == 0: @@ -143,8 +139,8 @@ def next_details( # grab the configuration patient_id_col: str = str(sheet_config["patient_id_col"]).strip() - date_columns: list[str] = normalize_inplace(sheet_config["date_columns"]) - text_columns: list[str] = normalize_inplace(sheet_config["text_columns"]) + date_columns: list[str] = normalize_row(sheet_config["date_columns"]) + text_columns: list[str] = normalize_row(sheet_config["text_columns"]) # check patient_id_col isn't re-used as other column types if patient_id_col in date_columns: diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index dfe5bad..e687020 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -101,8 +101,8 @@ def shift_excel_dates_inplace( sheet_configs, min_shift_days, max_shift_days, - load_linking_table=linking_table_path, - save_linking_table=linking_table_output, + old_linking_table=linking_table_path, + out_linking_table=linking_table_output, seed=seed, ) diff --git a/tests/data/shift_ignore/offsets.csv b/tests/data/test_shift_ignore/offsets.csv similarity index 100% rename from tests/data/shift_ignore/offsets.csv rename to tests/data/test_shift_ignore/offsets.csv diff --git a/tests/data/shift_ignore/page-data.csv b/tests/data/test_shift_ignore/page-data.csv similarity index 100% rename from tests/data/shift_ignore/page-data.csv rename to tests/data/test_shift_ignore/page-data.csv diff --git a/tests/data/shift_ignore/page-desc.csv b/tests/data/test_shift_ignore/page-desc.csv similarity index 100% rename from tests/data/shift_ignore/page-desc.csv rename to tests/data/test_shift_ignore/page-desc.csv diff --git a/tests/data/shift_ignore/shift-ignore.yaml b/tests/data/test_shift_ignore/shift-ignore.yaml similarity index 100% rename from tests/data/shift_ignore/shift-ignore.yaml rename to tests/data/test_shift_ignore/shift-ignore.yaml diff --git a/tests/data/shift_ignore/workbook.xlsx b/tests/data/test_shift_ignore/workbook.xlsx similarity index 100% rename from tests/data/shift_ignore/workbook.xlsx rename to tests/data/test_shift_ignore/workbook.xlsx diff --git a/tests/test_missing_id.py b/tests/test_missing_id.py index 412ea39..59c4c25 100644 --- a/tests/test_missing_id.py +++ b/tests/test_missing_id.py @@ -5,10 +5,7 @@ from nuh_helper import shift_excel_dates_inplace from nuh_helper.date_shift.defs import UnknownPatient -test_data = ( - # i feel like test data should be in a folder following this pattern - Path(__file__).parent / "data" / __name__ -) +test_data = Path(__file__).parent / "data" / __name__ def test_missing_id(tmp_path: Path) -> None: @@ -36,8 +33,6 @@ def test_missing_id(tmp_path: Path) -> None: source_file = str(test_data / "missing-id.xlsx") output_path = tmp_path / "target.xlsx" - linking_table_old = tmp_path / "linking_table_old.csv" - linking_table_out = tmp_path / "linking_table_out.csv" with pytest.raises(UnknownPatient) as info: shift_excel_dates_inplace( @@ -48,8 +43,8 @@ def test_missing_id(tmp_path: Path) -> None: min_shift_days=-20, max_shift_days=-1, seed=14333, - linking_table_path=str(linking_table_old), - linking_table_output=str(linking_table_out), + linking_table_path=None, + linking_table_output=None, ) assert str(info.value._message) == "Unknown id='nuh006' on page='deceased'" diff --git a/tests/test_profile.py b/tests/test_profile.py index 760e770..46a5959 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -99,7 +99,7 @@ def test_has_bom( [ str(csv_path), # this one does have a BOM which ends up in the final - str(Path(__file__).parent / "data/shift_ignore/offsets.csv"), + str(Path(__file__).parent / "data/test_shift_ignore/offsets.csv"), ], output_path=str(tmp_path / "report-1.xlsx"), ) @@ -108,7 +108,7 @@ def test_has_bom( [ str(csv_path), # this one does have a BOM which ends up in the final - str(Path(__file__).parent / "data/shift_ignore/offsets.csv"), + str(Path(__file__).parent / "data/test_shift_ignore/offsets.csv"), ], output_path=str(tmp_path / "report-1.xlsx"), remove_bom=remove_bom, diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index d38ca45..7bb1cda 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -3,20 +3,21 @@ import pytest from nuh_helper import book_page, shift_excel_dates_inplace +from nuh_helper.book_page import normalize_row from nuh_helper.date_shift.defs import ( HiddenDate, PatientColumnMissing, ShiftFoundNonDate, ) -test_data = Path(__file__).parent / "data/shift_ignore" +test_data = Path(__file__).parent / "data" / __name__ @pytest.mark.parametrize("use_csv", [False, True]) def test_shift_exceptions(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -54,7 +55,7 @@ def test_shift_exceptions(use_csv: bool, tmp_path: Path) -> None: def test_ptid_in_date_columns(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -91,7 +92,7 @@ def test_ptid_in_date_columns(use_csv: bool, tmp_path: Path) -> None: def test_ptid_in_text_columns(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -128,7 +129,7 @@ def test_ptid_in_text_columns(use_csv: bool, tmp_path: Path) -> None: def test_columns_overlap(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -165,7 +166,7 @@ def test_columns_overlap(use_csv: bool, tmp_path: Path) -> None: def test_ptid_column_missing(use_csv: bool, tmp_path: Path) -> None: """check that misconfigurations cause an error""" - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -203,7 +204,7 @@ def test_ignore_in_date_columns( use_csv: bool, shift_ignore: bool, tmp_path: Path ) -> None: - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -235,7 +236,10 @@ def body(shift_ignore_yaml: None | Path) -> None: if not shift_ignore: with pytest.raises(ShiftFoundNonDate) as info: - body(None) + body( + # instead of None - use a nonsense path to increase code coverage + str(tmp_path / "no-such-file.yaml") + ) assert info.value._page == "page-data" assert info.value._row == 5 assert info.value._col == 1 @@ -261,6 +265,7 @@ def body(shift_ignore_yaml: None | Path) -> None: ], ["nuh65", "mssing", None, "this one has spaces"], ] + expected = [normalize_row(row) for row in expected] assert obtained == expected @@ -270,7 +275,7 @@ def test_ignore_in_text_columns( use_csv: bool, shift_ignore: bool, tmp_path: Path ) -> None: - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -333,40 +338,10 @@ def body(shift_ignore_yaml: None | Path) -> None: assert expected == obtained -def load_workbook(path: str | Path) -> dict[str, list[list[str | None]]]: - - book = {} - - for page in book_page.book_open(path): - data = [] - - for row, cells in page.stream_rows(): - assert row == len(data) - for idx, val in enumerate(cells): - if val is None: - continue - val = str(val).strip() - if not val: - cells[idx] = None - continue - - import re - - pattern = r"\d{4}-\d{2}-\d{2} 00:00:00" - if re.fullmatch(pattern, val): - val = val[:10] - cells[idx] = val - data.append(cells) - - book[page.name] = data - - return book - - @pytest.mark.parametrize("use_csv", [False, True]) def test_move_to_file(use_csv: bool, tmp_path: Path) -> None: - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -412,7 +387,7 @@ def test_move_to_file(use_csv: bool, tmp_path: Path) -> None: @pytest.mark.parametrize("use_csv", [False, True]) def test_skip_hole(use_csv: bool, tmp_path: Path) -> None: - linking_table_old = Path(__file__).parent / "data/shift_ignore/offsets.csv" + linking_table_old = test_data / "offsets.csv" linking_table_out = tmp_path / "linking_table_out.csv" sheet_configs = { @@ -470,3 +445,11 @@ def src_and_out_names( source_file = test_data / "workbook.xlsx" output_path = tmp_path / "target.xlsx" return source_file, output_path + + +def load_workbook(path: str | Path) -> dict[str, list[list[str | None]]]: + + return { + page.name: [normalize_row(cells) for _, cells in page.stream_rows()] + for page in book_page.book_open(path) + } From fc8a0b8426d1c5250eb301a0b7c20a1d69b277d5 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 15 Oct 2026 11:08:11 +0100 Subject: [PATCH 70/82] (WIP) updates the test, but, it doesn't seem to ignore noted dates in the ignore list --- nuh_helper/book_page/book.py | 12 +++++++----- tests/conftest.py | 10 ++++++++++ tests/test_date_shift.py | 13 +++++++++++++ tests/test_shift_ignore.py | 11 ++--------- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index 3b167bd..914e231 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from collections.abc import Generator as gen -from collections.abc import Iterable as iterator +from collections.abc import Iterable as iterable from datetime import date, datetime from functools import cached_property from pathlib import Path @@ -17,7 +17,7 @@ def __init__(self, name: str, page: object) -> None: @abstractmethod def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: """performs the workbook specific yield/write of the row data""" - ... + raise NotImplementedError() def stream_rows(self) -> gen[tuple[int, row_val_t]]: """public method for reading a RO stream of rows from a specific page""" @@ -38,10 +38,12 @@ def __contains__(self, name: str) -> bool: return name in self.keys() @abstractmethod - def __getitem__(self, name: str) -> gen[Page]: ... + def __getitem__(self, name: str) -> gen[Page]: + raise NotImplementedError() @abstractmethod - def keys(self) -> iterator[str]: ... + def keys(self) -> iterable[str]: + raise NotImplementedError() @abstractmethod def _stream_copy_impl(self, into: Path) -> gen[tuple[str, int, row_val_t]]: @@ -122,7 +124,7 @@ def normalize_val(src: object) -> object: return src -def normalize_row(row: list[str]) -> list[str]: +def normalize_row(row: iterable[str]) -> list[str]: """modifies (and returns) the passed array to drop extra whitespace and replace fully blank entries with None. doesn't touch non-strings""" diff --git a/tests/conftest.py b/tests/conftest.py index 32f4cc6..33f113d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,9 @@ import pytest +from nuh_helper import book_page +from nuh_helper.book_page import normalize_row + OUTPUT_DIR = Path(__file__).parent / "output" @@ -30,3 +33,10 @@ def output_path(request: pytest.FixtureRequest, tmp_path: Path) -> Path: path.mkdir(parents=True) return path return tmp_path + + +def load_workbook(path: str | Path) -> dict[str, list[list[str | None]]]: + return { + page.name: [normalize_row(cells) for _, cells in page.stream_rows()] + for page in book_page.book_open(path) + } diff --git a/tests/test_date_shift.py b/tests/test_date_shift.py index dd65234..8a274ba 100644 --- a/tests/test_date_shift.py +++ b/tests/test_date_shift.py @@ -5,8 +5,10 @@ import pandas as pd import pytest +from conftest import load_workbook from openpyxl import Workbook +from nuh_helper.book_page.book import normalize_row from nuh_helper.date_shift import ( generate_shift_mappings, shift_excel_dates_inplace, @@ -276,6 +278,17 @@ def test_exception_date_unchanged_other_dates_shifted(self, tmp_path: Path) -> N shift_ignore_yaml=test_data / "shift-ignore.yaml", seed=42, ) + expected = { + "patients": [ + normalize_row(["patient_id", "last_alive"]), + normalize_row(["P001", "2023-06-06"]), + normalize_row(["P002", "2024-12-31"]), + ], + } + obtained = load_workbook(out) + + assert obtained == expected + raise Exception(out) df = pd.read_excel(str(out), sheet_name="patients") p001_date = df.loc[df["patient_id"] == "P001", "last_alive"].iloc[0] p002_date = df.loc[df["patient_id"] == "P002", "last_alive"].iloc[0] diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 7bb1cda..076af04 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -1,8 +1,9 @@ from pathlib import Path import pytest +from conftest import load_workbook -from nuh_helper import book_page, shift_excel_dates_inplace +from nuh_helper import shift_excel_dates_inplace from nuh_helper.book_page import normalize_row from nuh_helper.date_shift.defs import ( HiddenDate, @@ -445,11 +446,3 @@ def src_and_out_names( source_file = test_data / "workbook.xlsx" output_path = tmp_path / "target.xlsx" return source_file, output_path - - -def load_workbook(path: str | Path) -> dict[str, list[list[str | None]]]: - - return { - page.name: [normalize_row(cells) for _, cells in page.stream_rows()] - for page in book_page.book_open(path) - } From e195a13bad539dcdd77641e662d8b6e258ac87ed Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Thu, 15 Oct 2026 11:23:17 +0100 Subject: [PATCH 71/82] more consistent xlsx reading --- nuh_helper/book_page/book.py | 4 ++-- nuh_helper/book_page/xlsx_book.py | 26 ++++++++++++++++---------- tests/conftest.py | 6 ++++-- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index 914e231..dc79522 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -81,7 +81,7 @@ def __iter__(self) -> gen[Page]: yield page -def normalize_val(src: object) -> object: +def normalize_val(src: object) -> cell_val_t: """parse to a datetime, .strip(), turn "" into None, retain bool""" if isinstance(src, (bool | float | int | datetime | date)): @@ -124,7 +124,7 @@ def normalize_val(src: object) -> object: return src -def normalize_row(row: iterable[str]) -> list[str]: +def normalize_row(row: iterable[str]) -> row_val_t: """modifies (and returns) the passed array to drop extra whitespace and replace fully blank entries with None. doesn't touch non-strings""" diff --git a/nuh_helper/book_page/xlsx_book.py b/nuh_helper/book_page/xlsx_book.py index e274d18..1be08b2 100644 --- a/nuh_helper/book_page/xlsx_book.py +++ b/nuh_helper/book_page/xlsx_book.py @@ -4,8 +4,9 @@ from pathlib import Path import openpyxl +from openpyxl.cell.read_only import EmptyCell -from .book import Book, Page +from .book import Book, Page, normalize_row, row_val_t class XLSXBook(Book): @@ -15,12 +16,12 @@ def __init__(self, file: str | Path, read_only: bool = True) -> None: if not file.is_file(): raise RuntimeError(f"xlsx file not found {file}") self._file = file - self._book = openpyxl.load_workbook(file, read_only=read_only) + self._book = openpyxl.load_workbook(file, read_only=read_only, data_only=True) def keys(self) -> list[str]: return [page.title for page in self._book.worksheets] - def _stream_copy_impl(self, into: str | Path) -> gen[tuple[str, int, list[str]]]: + def _stream_copy_impl(self, into: str | Path) -> gen[tuple[str, int, row_val_t]]: if not isinstance(into, Path): into = Path(into) @@ -48,14 +49,19 @@ def __getitem__(self, name: str) -> gen[Page]: raise KeyError("excel limits you to 31 character page names") class XLSXPage(Page): - def _stream_rows_impl(self) -> gen[tuple[int, list[str]]]: - for row in range(self._page.max_row): + def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: + + for row_idx, row_data in enumerate(self._page.iter_rows()): yield ( - row, - [ - self._page.cell(row + 1, col + 1).value - for col in range(self._page.max_column) - ], + row_idx, + normalize_row( + [ + None + if isinstance(cell, EmptyCell) + else cell.internal_value + for cell in row_data + ] + ), ) yield XLSXPage(name, self._book[name]) diff --git a/tests/conftest.py b/tests/conftest.py index 33f113d..4a75977 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ import pytest from nuh_helper import book_page -from nuh_helper.book_page import normalize_row +from nuh_helper.book_page import normalize_row, row_val_t OUTPUT_DIR = Path(__file__).parent / "output" @@ -35,7 +35,9 @@ def output_path(request: pytest.FixtureRequest, tmp_path: Path) -> Path: return tmp_path -def load_workbook(path: str | Path) -> dict[str, list[list[str | None]]]: +def load_workbook( + path: str | Path, +) -> dict[str, list[row_val_t]]: return { page.name: [normalize_row(cells) for _, cells in page.stream_rows()] for page in book_page.book_open(path) From 54d8c3ecd30e1beb74e645f6681863f17d2dd06f Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 11:51:43 +0100 Subject: [PATCH 72/82] adds the hidden-date bypass to book_page --- nuh_helper/date_shift/defs.py | 2 ++ nuh_helper/date_shift/row_machine.py | 19 ++++++++++-------- nuh_helper/date_shift/stream_shift.py | 17 ++++++++-------- tests/test_hidden_dates.py | 29 ++++++++++++++++++++------- 4 files changed, 44 insertions(+), 23 deletions(-) diff --git a/nuh_helper/date_shift/defs.py b/nuh_helper/date_shift/defs.py index ceb6f88..0e552d5 100644 --- a/nuh_helper/date_shift/defs.py +++ b/nuh_helper/date_shift/defs.py @@ -85,6 +85,8 @@ def __init__( class HiddenDate(ValueError): + perform_checks: bool = True + def __init__( self, sheet_name: str, diff --git a/nuh_helper/date_shift/row_machine.py b/nuh_helper/date_shift/row_machine.py index c2a5cb3..d23ddc2 100644 --- a/nuh_helper/date_shift/row_machine.py +++ b/nuh_helper/date_shift/row_machine.py @@ -213,14 +213,17 @@ def next_details( # check for date in the patient_id for found in datefinder.find_dates(str(patient_id)): - raise HiddenDate( - sheet_name, - row_idx, - self._detail_archetype.patient_id_idx, - self._detail_archetype.patient_id_col, - patient_id, - found, - ) + logger.debug(f"found hidden date {sheet_name=} {patient_id=}") + + if HiddenDate.perform_checks: + raise HiddenDate( + sheet_name, + row_idx, + self._detail_archetype.patient_id_idx, + self._detail_archetype.patient_id_col, + patient_id, + found, + ) patient_id = _parse._normalize_patient_id(patient_id) if patient_id is None: diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index e687020..438aa74 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -177,14 +177,15 @@ def _shift_cell( f"found hidden date {detail.page_name=}/{col_name=}" + f":{cell_value}" ) - raise HiddenDate( - detail.page_name, - detail.row_idx, - col_idx, - col_name, - cell_value, - found, - ) + if HiddenDate.perform_checks: + raise HiddenDate( + detail.page_name, + detail.row_idx, + col_idx, + col_name, + cell_value, + found, + ) case ColumnType.DATE: # cell_value can be str | datetime | date # ... so parsed might not always succeed diff --git a/tests/test_hidden_dates.py b/tests/test_hidden_dates.py index d4a5ea8..9a02b36 100644 --- a/tests/test_hidden_dates.py +++ b/tests/test_hidden_dates.py @@ -49,7 +49,8 @@ def test_iso8601(tmp_path: Path) -> None: ) -def test_us_date(tmp_path: Path) -> None: +@pytest.mark.parametrize("bypass", [True, False]) +def test_us_date(bypass: bool, tmp_path: Path) -> None: source_file = Path(__file__).parent / "data/hidden_dates/us_date.xlsx" output_path = tmp_path / "target.xlsx" @@ -69,7 +70,8 @@ def test_us_date(tmp_path: Path) -> None: "skip_rows_after_header": [], }, } - with pytest.raises(HiddenDate) as info: + + def shift() -> None: shift_excel_dates_inplace( input_file=str(source_file), output_file=str(output_path), @@ -82,11 +84,24 @@ def test_us_date(tmp_path: Path) -> None: linking_table_output=str(linking_table_out), ) - assert info.value._message == ( - "hidden date in [sheet_name='data', 5, 2 @ diagnosis]" - + ' value="can\'t recall the date but on 12/11/2001 they had an itchy tummy"' - + " // found=datetime.datetime(2001, 12, 11, 0, 0)" - ) + assert HiddenDate.perform_checks + + if bypass: + try: + HiddenDate.perform_checks = False + shift() + finally: + HiddenDate.perform_checks = True + else: + with pytest.raises(HiddenDate) as info: + shift() + + assert info.value._message == ( + "hidden date in [sheet_name='data', 5, 2 @ diagnosis]" + + " value=\"can't recall the date but " + + 'on 12/11/2001 they had an itchy tummy"' + + " // found=datetime.datetime(2001, 12, 11, 0, 0)" + ) def test_written(tmp_path: Path) -> None: From 09e2e8af79c3bb965716108e8826f7a2388a36e6 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 12:04:17 +0100 Subject: [PATCH 73/82] always allow "NULL" in date columns --- nuh_helper/date_shift/row_machine.py | 12 +++++++----- tests/data/test_shift_ignore/page-data.csv | 2 ++ tests/data/test_shift_ignore/workbook.xlsx | Bin 10388 -> 10431 bytes tests/test_shift_ignore.py | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/nuh_helper/date_shift/row_machine.py b/nuh_helper/date_shift/row_machine.py index d23ddc2..daea97d 100644 --- a/nuh_helper/date_shift/row_machine.py +++ b/nuh_helper/date_shift/row_machine.py @@ -4,6 +4,7 @@ the shift module. """ +import datetime as dt import logging from collections.abc import Generator as gen from dataclasses import dataclass, replace @@ -55,16 +56,17 @@ def column_type( ) -> bool: """determine what we should do for a given column""" - ignored: set[None | str] = shift_ignore.get(self.page_name, {}).get( - col_name, {None, ""} - ) + ignored: set[None | str | dt.date | dt.datetime] = shift_ignore.get( + self.page_name, {} + ).get(col_name, {None, "", "NULL", "None"}) return ( ColumnType.IGNORE if cell_value in ignored else ColumnType.TEXT if col_name in self.text_columns - else ColumnType.DATE + # always allow NULL in date columns - even when it's not in the list + else (ColumnType.DATE if cell_value != "NULL" else ColumnType.IGNORE) ) @@ -226,7 +228,7 @@ def next_details( ) patient_id = _parse._normalize_patient_id(patient_id) - if patient_id is None: + if patient_id is None or patient_id == "NULL": # if the pid is None; the rest of the row should be None as well if non_blank := [ cell diff --git a/tests/data/test_shift_ignore/page-data.csv b/tests/data/test_shift_ignore/page-data.csv index 3dc5175..5011f19 100644 --- a/tests/data/test_shift_ignore/page-data.csv +++ b/tests/data/test_shift_ignore/page-data.csv @@ -6,3 +6,5 @@ nuh23,,,mushrooms nuh67,mssing,,this can't be a date - sorry 2016 nuh27,1999-12-07,,idk - this can't be a date anymore nuh65, mssing ,,this one has spaces + nuh56, NULL ,,oo +nuh57 , ,,oo diff --git a/tests/data/test_shift_ignore/workbook.xlsx b/tests/data/test_shift_ignore/workbook.xlsx index a3c84279dc41231ad19a5aa2c7dfc63c6b5b131c..ef2efa11249bc2a3c086c935152ebcd212c1af9d 100644 GIT binary patch delta 2879 zcmY+GX*d)L7stmkV#+oq%UBxw*cA;~GWIpu8N$p+DQn2s#zck4)`*%Y6CvA`HM_B8 zU*d8ZA$tbd!brT``##TmpZ9z^&-tHk|MNTNq+7RGmk+ZL409PUR3-r60~-Lq2><{> zeUxzif!F3$HXx;MyCEPcKY-tbTU#*)X6-tUH{zFN2oQ0KavS z=?&ppZ~2~Dvo4_sQgdmSl5yi0>dz9OANS>#Jw4BdpG4RavoND`*T$DUcU3R9DA_<> z7%8EMm4qVS!?DP(P4uPtgW17}($FX$>TMkAMH7rKr#)5 zTz8md8{BE0yD|RU(_8JUOh0J{2}g~oz3W| zvaJ;i_oi{cMEZ)-m7rs)Yx0eecbzFBa>^0b2Lf{+G~AV%1NWXNKs{Ef5YZA zLshjT{-zFvSM>Xh@+AZ8ww#vyb0*N0o}%zclI$f4TT>kv_Kf^@Hm!jt9GXOCLp%rM$joLBlGo2x1lq zkv*|yipXy9O?K!AG3g_(45W)SMrCFx=ah$25=5#ArR1hsG zYVuyi@fl#HMO`Oji%*^Zrwe#ysrUeb-M1tN%TuKVx%e@)nwXE#8auAe?nSNYoY$oE zcDA5bIJS9k$3 z!qoeJQThrP{{lPu-+U?W7)cSq7>(4P|MZ>?1*QpG`^UmVWI#e1y4=8myeGsX%SB+1 z{P1*%+F~#T_^e(Hnk^w5F%MtCvvj)t>9KKdl#5(%bYodVGMV#1>Wt6>&_^A#H``N3 z&yrqPnq5H}Q_RMbDT>-oQD!~i40|RTOjh67ek7KquK1SHVZ?CXN44NX_Qxo&w2Gsl zW7EyH)7-H&JJ6@vOe{)SlrP3}Xlt-xsZW5Wn>@wDyM;FoS!PjL2N;r`L98!Pyb!za zz=g4Uc<>O9ns%!FvXTyJmC^jovIU0~v)u`8#j5(7Ux-fKwBj-9W`8`GH>tPQGOI7GfTud1I3(HI=VN{haL3;x9 z<@zpqo|>VJl6NHpE=0`ST%nrJ2J%KJxBQH&p~dSy{!w;LcY%Fp!`{oKd%Db=;n&SQ zdCeB~#|NCM?+mP2vEur)pdos0NMoGjS(xXy9MOx`>jq4ue5kB^t^`=L~CdCv9J}&o*q0ay#xCdLyGL0lJ}W67Mi38xDFe0rNzw z-w*a_vt?zTHR{TdbhQ#|PCp&8&#Jrp1NP*#iZRO-v(8Eq(zGeOWf~cOex?klBbcMm z)Wb-#tabz(VMF7KUGZB$J+Mv4)Zq9sTLl66+P9%>fJSXA^Bwui0zEtoa81f*Xl7@} zA#=T7ae4*&sws^E43qzqMMsUy5PfeyW;VzLV@@6{AR!tFSh|;-jVscWy}4XD@=r9QE@( zmV`Bdjp)4K^oqEt19$83dG^TG+D<_PiREs>%0MZ@yPO)eP8zvMOaY&jlFc|rEf1Rd zl5uI~eo`f-CcFzVq%EK@+sYhC4O8v69a3L?P=4DdqV|&k=jL`Ucb1EyBUAZp;jZH3 z=XYiK4v{VBd+0h<8))Y41<(z65mOs!4ZEZF26T2cM%z(&g;u~^W(~RZdqost?094*LeJ|}BLx{Ze- za7`No05MerQrXKh1!7w=t9@6?004v#OGShTc71-+%I6ySR&}`Q_tFp3ZmF?04w8|U zy4MHJ1Spi1mPppU%^IU0ef~jyaC25Ql07c(Re?^(BOJ)5QS<#;jUBRVck1Iwo*O}X zRtr8CZ@I84oIR#j@_OnIcjz)Jn0A;K%U|&6?VKMhF_I=7h^;$BU+h)c1744|+C@SL z8%|@MVQ_h9LC$VvqV$hnIp~n(0}CC*+-Bia>@{k~{yg5cqefpou2nGoQZ+W1VpsJc6FnkK1&4q{0x>WfH)&uJ7bP zuOu0{cZwXe4J(YtS)^TkVWBA!xs0QrRg$(zbYRNl;?I!|lSwn*B;Zugg`4x#=`4 z=zE^?Uk<<;f1F!jS6jwccOQfeCiqR##tN;^oFv!Ki`VmR7C+`xTx=OR0sd_zP@Q9b z)2uQACdEK-_il$lI%m)P+^r@tC&(fY3P*j z!Czs6ojy@^_Q3#Juk%X06tiKDp@Ml+-$>sr?jfs1t%+xj)v-d^t>mzKgiN+}17D1< zH`93L+*{=5WHw@M(?Kj|y+Z_oGYwdU`FR(AnGN*&489jm| z->TRo$bF8SDq=E;CNl!3G%a`%TAD;k0tp43*rz7(2QpT^p4c zC`{zuu|2MFzKXM7u&HeTKS;0uNy!JeQ?cw{m%Ms78;fA=Oz*y6JG0#%KlV3I0B`se%;gy~sca)dUeDoe`uMlp+dJCDa5Gr5UqI zuc1g01w&{e3Je4gfpvDzSPWDlOh5pU`u9%11m$26#G_0oeX%!C_{wz3ciTHap{0LZ4ba z=JB(0uDly5;q(&@3Got}@BBcHSO$@AZrb+rNsseD;k^7J@bM@s)4a zJ2*~6^g2iE<2;Lx5y6*f?+RpyPd4GU6iE;znKf>J1SPx-VaS#}2M(b-ek#nTq-p4qooeAxVnNR& zQhbGL^CUC4^yE3m`?&A5;3walH9Nz?;n_N28j*ki-|4@n@g<;-qe}4}fpQ>}e!S>F zsK|*4FPFAbZEgc79&=wBOF2}kv~5)zPe0~CCi5w2n&xP+$7?*Px);xB<*!>ZpS;!k zTyy6woSE~0KZ_GlB^q75%0(AcY5knA6|IN-s8-#QSFk>x&z&i3WC7PO0xrT^nJ(H` z)O6d3=k3FcXfoKfJLz+nY&#SOB0mx+|AXfW!1NkEn2{^MBP@eUlgc7%8~bgj zP)+~Z9=X+?^OX9iQ3G{4sI&gr@8vQkQ>bDJwQ1;g+I?VToDqcMV-1sB^ufAVJ3d7Ov)&RHDmn`4sn&*dITXm$`#t z{QH5pZKz53+R0pvQu?0n4Jm z-5=Z)h6%pftu;)e&yC2j+EZ$<3k9Z+s*9>OhzJ?8Fdie0Witxh|uAS{5DE z`d0&2zvjSlT+MrPLo&hVM@hP+0o7`Qw7xL6@GtkaA*N#GOxItBjyLYz$z z;o?K$3e$zVSe2151i?Z%OykH13FL~bJVB4 zNgTlo(9{b{=|_3r=dp*OLJ!{OZs!EE!Szd95>f4Ly4m>cvG?SC+7<812oaY#r*z2= znaFL1T@jClpwH#yWnNBOLZP^x5SFS^U}kCocHJclghql)G$5EuBnS`YxWr7s@~Vh2 z;fGhGsU29Gpjke4_if;e6KZ?wK8+AbyZINv6njmQcw{G*EPQ(+cY9El++@6r6#Xfs zSbot^M7{dQ1pi8d@1dZo8EFC%`&B4Hjt7$BQy4v0EP7PnnSE|OsEY45XrDyX36jy1 zmzo%OCx?&CfXd`Rd?6*}jbH6tS@G!?2EY$F{hiWjb!rNP@u_YLO3HeR-K%08Ypc74 z2#C_vXb9k!c@cE4i)r}YqIo88vezqdzsv)K9qd^FJ&fXqEyo0^{Wi z`j!rSEdnj6v<&=+N#ceqpaZY-vYyboTl+`Ze%B0(Mxr9jNwj%&z7S1+a+R7K!U_%d zDNC`>P{^}P2)E+t&=w9^PH&osTyREMAZvVEtG+6S*BkVJJ3Esz1Q!!(uWJukj#2ed zDbmMu65`rr_+DL;tQc{;q-Q{E+O9M(S#qo+KCoCoVl*@n{*2`yVf>3&+t7vYtSG&2 zHLogug$>LA0Py}-wK2E+qJsh*W1~YOLSp{$ZMfrb#G(>=x4?!WxO2bo2)kX&9KLQ} z;WLp0n5DQ6xfjY0g+6U*@$_L0jho6Z49*BV-0L6RF*j~p_g2rt(^1G_!;URUXGdo{ z&dH{M586V$%$QetXXJ^YbPAa3AJ^QJ>Rau=1gcT(MNhAmUAieI(ih9&2(^Yx`)5@? zX&^L@KQb9+H77=Yomn?04bf(;mN=JVZli{-j6yyBS_Xfi(+cXwr2@N$-X9t3q9~~J3Y$>-BjQ2cf+Nsq-2bTvkm z<7%5qgUhMGTrgjEdwP{Qtx^uk7f)qzOG{ED{BuzmZ3{N`{GAfQ&*ycBTcnM33Bg*X zyFh4JkWXvuV^WQ}T{z>}%-QUu9Bh9_;CD~*0+;~IzYvN81fOz}tlR1!Qb3f0^;;$0 z^d-h~lfi^1wJ`x~Fpylrg0HXH?qx}Y2iDSrXLft(38BqJ`lz@phFBJ3*{L@ur zGQ+BZ6gO$f8)Uv<2+|3n7S`y7Ep753M)>OwahY(su8Z7c#P5^-5FATp&4~(`43!x*V z<$&Gj5@{n~4SGQuBK-f+`+e{j%_k$x_#cXBV;Mf+E%Z$p1G+Dl&_yzrm|j8uS^PI^ C*G!%O diff --git a/tests/test_shift_ignore.py b/tests/test_shift_ignore.py index 7bb1cda..41db13d 100644 --- a/tests/test_shift_ignore.py +++ b/tests/test_shift_ignore.py @@ -264,6 +264,8 @@ def body(shift_ignore_yaml: None | Path) -> None: "idk - this can't be a date anymore", ], ["nuh65", "mssing", None, "this one has spaces"], + ["nuh56", "NULL", None, "oo"], + ["nuh57", None, None, "oo"], ] expected = [normalize_row(row) for row in expected] assert obtained == expected @@ -448,7 +450,6 @@ def src_and_out_names( def load_workbook(path: str | Path) -> dict[str, list[list[str | None]]]: - return { page.name: [normalize_row(cells) for _, cells in page.stream_rows()] for page in book_page.book_open(path) From a4eb73bb7e41475e4de804da838567f13a2ee161 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 13:40:51 +0100 Subject: [PATCH 74/82] option to skip dating date --- nuh_helper/book_page/book.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index 3b167bd..6906e00 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -23,8 +23,9 @@ def stream_rows(self) -> gen[tuple[int, row_val_t]]: """public method for reading a RO stream of rows from a specific page""" for row_idx, row_data in self._stream_rows_impl(): - for raw_idx, raw_val in enumerate(row_data): - row_data[raw_idx] = normalize_val(raw_val) + if Book.normalize: + for raw_idx, raw_val in enumerate(row_data): + row_data[raw_idx] = normalize_val(raw_val) yield row_idx, row_data @@ -34,6 +35,8 @@ def name(self) -> str: class Book(ABC): + normalize: bool = True + def __contains__(self, name: str) -> bool: return name in self.keys() @@ -67,11 +70,12 @@ def stream_copy( for page_name, row_idx, row_data in self._stream_copy_impl(into): yield page_name, row_idx, row_data - for raw_idx, raw_val in enumerate(row_data): - if isinstance(raw_val, date): - row_data[raw_idx] = raw_val.strftime(date_format) - elif isinstance(raw_val, datetime): - row_data[raw_idx] = raw_val.strftime(time_format) + if Book.normalize: + for raw_idx, raw_val in enumerate(row_data): + if isinstance(raw_val, date): + row_data[raw_idx] = raw_val.strftime(date_format) + elif isinstance(raw_val, datetime): + row_data[raw_idx] = raw_val.strftime(time_format) def __iter__(self) -> gen[Page]: for name in self.keys(): From c7d8776fd4a8bb25cc1ac335a98cf01fe882e7ff Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 13:46:48 +0100 Subject: [PATCH 75/82] add "skip" to sheet/page types --- nuh_helper/date_shift/defs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nuh_helper/date_shift/defs.py b/nuh_helper/date_shift/defs.py index 0e552d5..e5379db 100644 --- a/nuh_helper/date_shift/defs.py +++ b/nuh_helper/date_shift/defs.py @@ -4,12 +4,13 @@ """ from datetime import datetime +from typing import Literal # each entry in a sheet config will match this _sheet_config_row_t = str | int | list[str] | dict[str, set[str] | list[str]] # a single sheet will consist of this config -_sheet_config_single_t = dict[str, _sheet_config_row_t] +_sheet_config_single_t = dict[str, _sheet_config_row_t] | Literal["skip"] # the whole configuration for a shift workbook will have this type sheet_configs_t = dict[str, _sheet_config_single_t] From c17e023df2a6d54de2d71e98926563a5e2dac5b3 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 13:59:08 +0100 Subject: [PATCH 76/82] cut out normalise in a different way --- nuh_helper/book_page/book.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index 6906e00..e082f16 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -86,7 +86,7 @@ def __iter__(self) -> gen[Page]: def normalize_val(src: object) -> object: """parse to a datetime, .strip(), turn "" into None, retain bool""" - if isinstance(src, (bool | float | int | datetime | date)): + if isinstance(src, (bool | float | int | datetime | date)) or not Book.normalize: return src if isinstance(src, str): From a17deb20d5b3c24ec8c5b3e7c89a8709d72e2d70 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 15:43:12 +0100 Subject: [PATCH 77/82] renamed and documented the bypasses --- README.md | 21 +++++++++++++++ nuh_helper/book_page/__init__.py | 14 +++++----- nuh_helper/book_page/book.py | 19 ++++++++------ nuh_helper/book_page/csv_book.py | 11 +++----- nuh_helper/book_page/generic.py | 14 +++++----- nuh_helper/book_page/xlsx_book.py | 8 +++--- nuh_helper/date_shift/ignore.py | 9 ++++--- nuh_helper/date_shift/mappings.py | 4 +-- nuh_helper/date_shift/row_machine.py | 4 +-- nuh_helper/date_shift/stream_shift.py | 4 +-- tests/test_book_page.py | 38 +++++++++++++-------------- 11 files changed, 84 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index d5703c5..02b3f7e 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,27 @@ In most cases, the process for using `shift_excel_dates_inplace()` with `.csv` o There is technically no requirement that all of the `.csv` files be in the same folder. If you're date-shifting carrot-transform output, you will want to use the list of files and remove the `person_ids.tsv` and `summary_mapstream.tsv` files. +### Bypassing Data Normalization and Hidden Date Checks + +In one instance we've needed to disable data normalization and hidden date checks. +If you need to do this, at the top of your pipeline ... + +```python +# this didn't work until we did it at the top of the python file + +# import the HiddenDate exception class +from nuh_helper.date_shift.defs import HiddenDate +from nuh_helper.book_page.book import BaseWorkBook + +# disable raising this when dates are detected in places they shouldn't be +HiddenDate.perform_checks = True + +# disable changing values (mostly CSV) into date or datetime objects +# ... also stops turning "" and " " into None +BaseWorkBook.normalize_values = False + +``` + ## Usage - profile Profile a dataset and generate a Scan Report. diff --git a/nuh_helper/book_page/__init__.py b/nuh_helper/book_page/__init__.py index 20233fc..bf02025 100644 --- a/nuh_helper/book_page/__init__.py +++ b/nuh_helper/book_page/__init__.py @@ -1,13 +1,13 @@ -from nuh_helper.book_page.book import Book, Page, normalize_row -from nuh_helper.book_page.csv_book import CSVBook +from nuh_helper.book_page.book import BasePage, BaseWorkBook, normalize_row +from nuh_helper.book_page.csv_book import CSVWorkBook from nuh_helper.book_page.generic import book_open -from nuh_helper.book_page.xlsx_book import XLSXBook +from nuh_helper.book_page.xlsx_book import XLSXWorkBook __all__ = [ # noqa: PLE0604 - CSVBook, - XLSXBook, - Book, - Page, + CSVWorkBook, + XLSXWorkBook, + BaseWorkBook, + BasePage, book_open, normalize_row, ] diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index e082f16..bc1113a 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -9,7 +9,7 @@ row_val_t = list[cell_val_t] -class Page(ABC): +class BasePage(ABC): def __init__(self, name: str, page: object) -> None: self._name = name self._page = page @@ -23,7 +23,7 @@ def stream_rows(self) -> gen[tuple[int, row_val_t]]: """public method for reading a RO stream of rows from a specific page""" for row_idx, row_data in self._stream_rows_impl(): - if Book.normalize: + if BaseWorkBook.normalize_values: for raw_idx, raw_val in enumerate(row_data): row_data[raw_idx] = normalize_val(raw_val) @@ -34,14 +34,14 @@ def name(self) -> str: return self._name -class Book(ABC): - normalize: bool = True +class BaseWorkBook(ABC): + normalize_values: bool = True def __contains__(self, name: str) -> bool: return name in self.keys() @abstractmethod - def __getitem__(self, name: str) -> gen[Page]: ... + def __getitem__(self, name: str) -> gen[BasePage]: ... @abstractmethod def keys(self) -> iterator[str]: ... @@ -70,14 +70,14 @@ def stream_copy( for page_name, row_idx, row_data in self._stream_copy_impl(into): yield page_name, row_idx, row_data - if Book.normalize: + if BaseWorkBook.normalize_values: for raw_idx, raw_val in enumerate(row_data): if isinstance(raw_val, date): row_data[raw_idx] = raw_val.strftime(date_format) elif isinstance(raw_val, datetime): row_data[raw_idx] = raw_val.strftime(time_format) - def __iter__(self) -> gen[Page]: + def __iter__(self) -> gen[BasePage]: for name in self.keys(): with self[name] as page: yield page @@ -86,7 +86,10 @@ def __iter__(self) -> gen[Page]: def normalize_val(src: object) -> object: """parse to a datetime, .strip(), turn "" into None, retain bool""" - if isinstance(src, (bool | float | int | datetime | date)) or not Book.normalize: + if ( + isinstance(src, (bool | float | int | datetime | date)) + or not BaseWorkBook.normalize_values + ): return src if isinstance(src, str): diff --git a/nuh_helper/book_page/csv_book.py b/nuh_helper/book_page/csv_book.py index 9eabc3c..deb0f40 100644 --- a/nuh_helper/book_page/csv_book.py +++ b/nuh_helper/book_page/csv_book.py @@ -4,10 +4,10 @@ from collections.abc import Generator as gen from pathlib import Path -from nuh_helper.book_page.book import Book, Page, normalize_row, row_val_t +from nuh_helper.book_page.book import BasePage, BaseWorkBook, row_val_t -class CSVBook(Book): +class CSVWorkBook(BaseWorkBook): def __init__(self, path: str | Path | list[Path]) -> None: self._page_files = { page.stem.lower(): page.absolute() for page in csv_files(path) @@ -17,9 +17,9 @@ def keys(self) -> dict_keys: return self._page_files.keys() @contextlib.contextmanager - def __getitem__(self, name: str) -> gen[Page]: + def __getitem__(self, name: str) -> gen[BasePage]: - class CSVPage(Page): + class CSVPage(BasePage): def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: assert isinstance(self._page, Path) with self._page.open() as file: @@ -37,9 +37,6 @@ def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: ): line[0] = line[0][1:] - # strip cell strings, convert empty cells to None - line = normalize_row(line) - yield (row, line) assert isinstance(self._page_files[name], Path) diff --git a/nuh_helper/book_page/generic.py b/nuh_helper/book_page/generic.py index 8ccbac7..0b252ae 100644 --- a/nuh_helper/book_page/generic.py +++ b/nuh_helper/book_page/generic.py @@ -1,11 +1,11 @@ from pathlib import Path -from .book import Page -from .csv_book import CSVBook -from .xlsx_book import XLSXBook +from .book import BasePage +from .csv_book import CSVWorkBook +from .xlsx_book import XLSXWorkBook # use this or-type instead of the "real" base class since we only have/support these two -Book = CSVBook | XLSXBook +Book = CSVWorkBook | XLSXWorkBook def book_open(src: str | Path | list[Path]) -> Book: @@ -13,13 +13,13 @@ def book_open(src: str | Path | list[Path]) -> Book: src = Path(src) if isinstance(src, Path) and src.name.endswith(".xlsx"): - return XLSXBook(src) + return XLSXWorkBook(src) else: - return CSVBook(src) + return CSVWorkBook(src) __all__ = [ # noqa: PLE0604 Book, - Page, + BasePage, book_open, ] diff --git a/nuh_helper/book_page/xlsx_book.py b/nuh_helper/book_page/xlsx_book.py index e274d18..4d04ea0 100644 --- a/nuh_helper/book_page/xlsx_book.py +++ b/nuh_helper/book_page/xlsx_book.py @@ -5,10 +5,10 @@ import openpyxl -from .book import Book, Page +from .book import BasePage, BaseWorkBook -class XLSXBook(Book): +class XLSXWorkBook(BaseWorkBook): def __init__(self, file: str | Path, read_only: bool = True) -> None: if isinstance(file, str): file = Path(file) @@ -42,12 +42,12 @@ def _stream_copy_impl(self, into: str | Path) -> gen[tuple[str, int, list[str]]] the_copy.save(into) @contextlib.contextmanager - def __getitem__(self, name: str) -> gen[Page]: + def __getitem__(self, name: str) -> gen[BasePage]: if len(name) > 31: raise KeyError("excel limits you to 31 character page names") - class XLSXPage(Page): + class XLSXPage(BasePage): def _stream_rows_impl(self) -> gen[tuple[int, list[str]]]: for row in range(self._page.max_row): yield ( diff --git a/nuh_helper/date_shift/ignore.py b/nuh_helper/date_shift/ignore.py index e14449b..c104567 100644 --- a/nuh_helper/date_shift/ignore.py +++ b/nuh_helper/date_shift/ignore.py @@ -3,7 +3,7 @@ import yaml -from nuh_helper.book_page import normalize_row +from nuh_helper.book_page import BaseWorkBook, normalize_row shift_ignore_t = dict[str, dict[str, set[str]]] @@ -32,8 +32,9 @@ def shift_ignore_load(shift_ignore_yaml: None | Path | str) -> shift_ignore_t: shift_ignore_page = {} for col_name, col_ignored in page_data.items(): - shift_ignore_page[col_name.strip()] = set( - normalize_row(col_ignored + [None]) - ) + col_ignored = col_ignored + [None] + if BaseWorkBook.normalize_values: + col_ignored = normalize_row(col_ignored) + shift_ignore_page[col_name.strip()] = set(col_ignored) shift_ignore_full[page_name.strip()] = shift_ignore_page return shift_ignore_full diff --git a/nuh_helper/date_shift/mappings.py b/nuh_helper/date_shift/mappings.py index f7b0212..2090c63 100644 --- a/nuh_helper/date_shift/mappings.py +++ b/nuh_helper/date_shift/mappings.py @@ -8,7 +8,7 @@ import pandas as pd from pandas import Timedelta -from nuh_helper.book_page import Book +from nuh_helper.book_page import BaseWorkBook from nuh_helper.date_shift import _parse from nuh_helper.date_shift.defs import ( sheet_configs_t, @@ -64,7 +64,7 @@ def load_shift_mappings(csv_path: str) -> pd.DataFrame: def load_shift_deltas( - book: Book, + book: BaseWorkBook, patient_sheet: str, sheet_configs: sheet_configs_t, min_shift_days: int, diff --git a/nuh_helper/date_shift/row_machine.py b/nuh_helper/date_shift/row_machine.py index daea97d..940e4fc 100644 --- a/nuh_helper/date_shift/row_machine.py +++ b/nuh_helper/date_shift/row_machine.py @@ -14,7 +14,7 @@ import datefinder -from nuh_helper.book_page import Book, normalize_row +from nuh_helper.book_page import BaseWorkBook, normalize_row from nuh_helper.date_shift import _parse from nuh_helper.date_shift.defs import ( DateColumnsMissing, @@ -253,7 +253,7 @@ def next_details( row_cells=row_cells, ) - def book_copy(scanner, book: Book, output_file: Path) -> gen[PatientRow]: + def book_copy(scanner, book: BaseWorkBook, output_file: Path) -> gen[PatientRow]: for sheet_name, row_idx, row_cells in book.stream_copy(output_file): # check if we're in a patient row, and, yield the row details if so detail = scanner.next_details(sheet_name, row_idx, row_cells) diff --git a/nuh_helper/date_shift/stream_shift.py b/nuh_helper/date_shift/stream_shift.py index 438aa74..79a52b6 100644 --- a/nuh_helper/date_shift/stream_shift.py +++ b/nuh_helper/date_shift/stream_shift.py @@ -6,7 +6,7 @@ import datefinder from pandas import Timedelta -from nuh_helper.book_page import Book, book_open +from nuh_helper.book_page import BaseWorkBook, book_open from nuh_helper.date_shift import _parse, mappings from nuh_helper.date_shift.defs import ( BlankColumnHasData, @@ -81,7 +81,7 @@ def shift_excel_dates_inplace( + f"{min_shift_days=} {max_shift_days=} {seed=}" ) - book: Book = book_open(input_file) + book: BaseWorkBook = book_open(input_file) logger.info(f"Book has {type(book)=}") # check for missing sheets diff --git a/tests/test_book_page.py b/tests/test_book_page.py index 519dba7..624ad8a 100644 --- a/tests/test_book_page.py +++ b/tests/test_book_page.py @@ -4,7 +4,7 @@ import pytest -from nuh_helper.book_page import Book, CSVBook, XLSXBook, book_open +from nuh_helper.book_page import BaseWorkBook, CSVWorkBook, XLSXWorkBook, book_open test_data = ( # i feel like test data should be in a folder following this pattern @@ -31,7 +31,7 @@ def test_csv_name_collision() -> None: with pytest.raises(ValueError) as error: - CSVBook([test_book / "data_3x4.csv", test_book / "data_3x4.tsv"]) + CSVWorkBook([test_book / "data_3x4.csv", test_book / "data_3x4.tsv"]) assert ( textwrap.dedent( f""" @@ -49,8 +49,8 @@ def test_csv_name_collision() -> None: @pytest.mark.parametrize( "kind, path", [ - (CSVBook, test_book), - (XLSXBook, test_book / "data.xlsx"), + (CSVWorkBook, test_book), + (XLSXWorkBook, test_book / "data.xlsx"), ], ) @pytest.mark.parametrize( @@ -69,7 +69,7 @@ def test_csv_name_collision() -> None: ], ) def test_read_data( - kind: CSVBook | XLSXBook, + kind: CSVWorkBook | XLSXWorkBook, path: Path, to_str: bool, generic: bool, @@ -78,7 +78,7 @@ def test_read_data( ) -> None: # arrange - book: Book + book: BaseWorkBook # act if generic: @@ -100,8 +100,8 @@ def test_read_data( @pytest.mark.parametrize( "kind, path", [ - (CSVBook, test_book), - (XLSXBook, test_book / "data.xlsx"), + (CSVWorkBook, test_book), + (XLSXWorkBook, test_book / "data.xlsx"), ], ) @pytest.mark.parametrize( @@ -139,7 +139,7 @@ def test_copy_book( data: list[list[str]], ) -> None: # arrange - source: Book + source: BaseWorkBook changed_data = [ [f"edit[{r},{c}]{data[r][c]}" for c in range(len(data[0]))] for r in range(len(data)) @@ -159,7 +159,7 @@ def test_copy_book( cells[col] = f"edit[{row},{col}]{val}" # assert - target: Book + target: BaseWorkBook target = book_open(end) if generic_open else kind(end) # assert @@ -177,7 +177,7 @@ def test_copy_book( def test_stream_csv_rows() -> None: - book = CSVBook([test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]) + book = CSVWorkBook([test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]) copy = [] with book["data_3x4"] as page: @@ -188,7 +188,7 @@ def test_stream_csv_rows() -> None: def test_csv_stream_wont_overwrite_file(tmp_path: Path) -> None: - book = CSVBook([test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]) + book = CSVWorkBook([test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]) into = tmp_path / "foo" into.write_text("foobar") @@ -202,8 +202,8 @@ def test_csv_stream_wont_overwrite_file(tmp_path: Path) -> None: @pytest.mark.parametrize( "kind, path", [ - (CSVBook, [test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]), - (XLSXBook, test_book / "data.xlsx"), + (CSVWorkBook, [test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]), + (XLSXWorkBook, test_book / "data.xlsx"), ], ) def test_stream_rows(kind: type, path: Path) -> None: @@ -226,10 +226,10 @@ def test_stream_rows(kind: type, path: Path) -> None: def test_non_file(tmp_path: Path) -> None: with pytest.raises(RuntimeError) as error: - XLSXBook(tmp_path / "foo.xlsx") + XLSXWorkBook(tmp_path / "foo.xlsx") assert f"xlsx file not found {tmp_path}/foo.xlsx" == str(error.value) with pytest.raises(RuntimeError) as error: - for _ in XLSXBook(tmp_path / "bar.xlsx").stream_copy(tmp_path / "foo.xlsx"): + for _ in XLSXWorkBook(tmp_path / "bar.xlsx").stream_copy(tmp_path / "foo.xlsx"): pytest.fail("this should have raised an exception") assert f"xlsx file not found {tmp_path}/bar.xlsx" == str(error.value) @@ -238,12 +238,12 @@ def test_csv_file() -> None: path = test_book / "data.xlsx" with pytest.raises(RuntimeError) as error: - CSVBook(path) + CSVWorkBook(path) assert f"need a dir or list of files, but, got a file {path=}" == str(error.value) def test_xlsx_name_cap() -> Never: - book = XLSXBook(test_book / "data.xlsx") + book = XLSXWorkBook(test_book / "data.xlsx") long_name = "name which exceeds 31 characters" with ( pytest.raises(KeyError) as error, @@ -254,7 +254,7 @@ def test_xlsx_name_cap() -> Never: def test_bom_is_removed() -> None: - book = CSVBook([test_data / "page-desc.csv"]) + book = CSVWorkBook([test_data / "page-desc.csv"]) obtained = [] with book["page-desc"] as page: From c623097110a4c6a9a9dc372dd74e489dc136f317 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 16:05:01 +0100 Subject: [PATCH 78/82] raise exceptions on abstract invocation --- nuh_helper/book_page/book.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index bc1113a..427a4e1 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -17,7 +17,7 @@ def __init__(self, name: str, page: object) -> None: @abstractmethod def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: """performs the workbook specific yield/write of the row data""" - ... + raise NotImplementedError() def stream_rows(self) -> gen[tuple[int, row_val_t]]: """public method for reading a RO stream of rows from a specific page""" @@ -41,15 +41,17 @@ def __contains__(self, name: str) -> bool: return name in self.keys() @abstractmethod - def __getitem__(self, name: str) -> gen[BasePage]: ... + def __getitem__(self, name: str) -> gen[BasePage]: + raise NotImplementedError() @abstractmethod - def keys(self) -> iterator[str]: ... + def keys(self) -> iterator[str]: + raise NotImplementedError() @abstractmethod def _stream_copy_impl(self, into: Path) -> gen[tuple[str, int, row_val_t]]: """performs the workbook specific yield/write of the row data""" - ... + raise NotImplementedError() def stream_copy( self, From 24bd638b0cc76a8d3338d7aa68a3a917cf7bd06d Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 16:06:16 +0100 Subject: [PATCH 79/82] Potential fix for pull request finding 'CodeQL / Clear-text logging of sensitive information' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- nuh_helper/date_shift/row_machine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuh_helper/date_shift/row_machine.py b/nuh_helper/date_shift/row_machine.py index 940e4fc..8748d5a 100644 --- a/nuh_helper/date_shift/row_machine.py +++ b/nuh_helper/date_shift/row_machine.py @@ -215,7 +215,7 @@ def next_details( # check for date in the patient_id for found in datefinder.find_dates(str(patient_id)): - logger.debug(f"found hidden date {sheet_name=} {patient_id=}") + logger.debug("found hidden date in patient_id field %s", sheet_name) if HiddenDate.perform_checks: raise HiddenDate( From 780682aec25948c6c436c9f236486d252dd375a3 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 16:06:44 +0100 Subject: [PATCH 80/82] renamed self to self --- nuh_helper/date_shift/row_machine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nuh_helper/date_shift/row_machine.py b/nuh_helper/date_shift/row_machine.py index 940e4fc..2267408 100644 --- a/nuh_helper/date_shift/row_machine.py +++ b/nuh_helper/date_shift/row_machine.py @@ -253,9 +253,9 @@ def next_details( row_cells=row_cells, ) - def book_copy(scanner, book: BaseWorkBook, output_file: Path) -> gen[PatientRow]: + def book_copy(self, book: BaseWorkBook, output_file: Path) -> gen[PatientRow]: for sheet_name, row_idx, row_cells in book.stream_copy(output_file): # check if we're in a patient row, and, yield the row details if so - detail = scanner.next_details(sheet_name, row_idx, row_cells) + detail = self.next_details(sheet_name, row_idx, row_cells) if detail: yield detail From 58f7c1eb4633d7f52a2e379362ba3e9d79a38039 Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 17:29:25 +0100 Subject: [PATCH 81/82] restores the ... in abstract methods --- nuh_helper/book_page/book.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index 33dc765..ea15b8b 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -17,7 +17,7 @@ def __init__(self, name: str, page: object) -> None: @abstractmethod def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: """performs the workbook specific yield/write of the row data""" - pass + ... def stream_rows(self) -> gen[tuple[int, row_val_t]]: """public method for reading a RO stream of rows from a specific page""" @@ -41,17 +41,15 @@ def __contains__(self, name: str) -> bool: return name in self.keys() @abstractmethod - def __getitem__(self, name: str) -> gen[BasePage]: - pass + def __getitem__(self, name: str) -> gen[BasePage]: ... @abstractmethod - def keys(self) -> iterable[str]: - pass + def keys(self) -> iterable[str]: ... @abstractmethod def _stream_copy_impl(self, into: Path) -> gen[tuple[str, int, row_val_t]]: """performs the workbook specific yield/write of the row data""" - pass + ... def stream_copy( self, From 89b59a23f336ecd65f4f5785633a7be62bef8fec Mon Sep 17 00:00:00 2001 From: Peter LaValle Date: Fri, 16 Oct 2026 18:20:43 +0100 Subject: [PATCH 82/82] removed the redundant context manager --- nuh_helper/book_page/book.py | 5 ++--- nuh_helper/book_page/csv_book.py | 6 ++---- nuh_helper/book_page/xlsx_book.py | 4 +--- nuh_helper/date_shift/mappings.py | 18 ++++++++-------- tests/test_book_page.py | 36 +++++++++++++++---------------- 5 files changed, 32 insertions(+), 37 deletions(-) diff --git a/nuh_helper/book_page/book.py b/nuh_helper/book_page/book.py index ea15b8b..c8ae838 100644 --- a/nuh_helper/book_page/book.py +++ b/nuh_helper/book_page/book.py @@ -41,7 +41,7 @@ def __contains__(self, name: str) -> bool: return name in self.keys() @abstractmethod - def __getitem__(self, name: str) -> gen[BasePage]: ... + def __getitem__(self, name: str) -> BasePage: ... @abstractmethod def keys(self) -> iterable[str]: ... @@ -82,8 +82,7 @@ def stream_copy( def __iter__(self) -> gen[BasePage]: for name in self.keys(): - with self[name] as page: - yield page + yield self[name] def normalize_val(src: object) -> cell_val_t: diff --git a/nuh_helper/book_page/csv_book.py b/nuh_helper/book_page/csv_book.py index 4347249..fbcf1e2 100644 --- a/nuh_helper/book_page/csv_book.py +++ b/nuh_helper/book_page/csv_book.py @@ -1,4 +1,3 @@ -import contextlib import csv from _collections_abc import dict_keys from collections.abc import Generator as gen @@ -16,8 +15,7 @@ def __init__(self, path: str | Path | list[Path]) -> None: def keys(self) -> dict_keys: return self._page_files.keys() - @contextlib.contextmanager - def __getitem__(self, name: str) -> gen[BasePage]: + def __getitem__(self, name: str) -> BasePage: class CSVPage(BasePage): def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: @@ -41,7 +39,7 @@ def _stream_rows_impl(self) -> gen[tuple[int, row_val_t]]: yield (row, line) assert isinstance(self._page_files[name], Path) - yield CSVPage(name, self._page_files[name]) + return CSVPage(name, self._page_files[name]) def _stream_copy_impl(self, into: Path) -> gen[tuple[str, int, row_val_t]]: diff --git a/nuh_helper/book_page/xlsx_book.py b/nuh_helper/book_page/xlsx_book.py index ce379ec..16bb358 100644 --- a/nuh_helper/book_page/xlsx_book.py +++ b/nuh_helper/book_page/xlsx_book.py @@ -1,4 +1,3 @@ -import contextlib import shutil from collections.abc import Generator as gen from pathlib import Path @@ -42,7 +41,6 @@ def _stream_copy_impl(self, into: str | Path) -> gen[tuple[str, int, row_val_t]] page.cell(r + 1, c + 1).value = val the_copy.save(into) - @contextlib.contextmanager def __getitem__(self, name: str) -> gen[BasePage]: if len(name) > 31: @@ -59,4 +57,4 @@ def _stream_rows_impl(self) -> gen[tuple[int, list[str]]]: ], ) - yield XLSXPage(name, self._book[name]) + return XLSXPage(name, self._book[name]) diff --git a/nuh_helper/date_shift/mappings.py b/nuh_helper/date_shift/mappings.py index aa9ba4e..7828a97 100644 --- a/nuh_helper/date_shift/mappings.py +++ b/nuh_helper/date_shift/mappings.py @@ -59,16 +59,16 @@ def load_shift_deltas( elif not isinstance(out_linking_table, Path): out_linking_table = Path(out_linking_table) - with book[patient_sheet] as page: - scanner = PatientRowStateMachine(sheet_configs) - patient_ids: list[str] = [ - pid.patient_id - for pid in [ - scanner.next_details(page.name, row, data) - for row, data in page.stream_rows() - ] - if pid + page = book[patient_sheet] + scanner = PatientRowStateMachine(sheet_configs) + patient_ids: list[str] = [ + pid.patient_id + for pid in [ + scanner.next_details(page.name, row, data) + for row, data in page.stream_rows() ] + if pid + ] # load the old ids (should we remove IDs that are "gone"?) if not old_linking_table or not old_linking_table.is_file(): diff --git a/tests/test_book_page.py b/tests/test_book_page.py index 8dedb34..d757d65 100644 --- a/tests/test_book_page.py +++ b/tests/test_book_page.py @@ -94,12 +94,12 @@ def test_read_data( # assert assert isinstance(book, kind) assert name in book - with book[name] as page: - copy = [] - for _, row in page.stream_rows(): - copy.append([str(c) for c in row]) + page = book[name] + copy = [] + for _, row in page.stream_rows(): + copy.append([str(c) for c in row]) - assert data == copy + assert data == copy @pytest.mark.parametrize( @@ -173,21 +173,21 @@ def test_copy_book( assert isinstance(source, kind) assert isinstance(target, kind) - with target[name] as page: - copy = [] - for row, cell in page.stream_rows(): - assert len(copy) == row - copy.append(cell) - assert copy == changed_data + page = target[name] + copy = [] + for row, cell in page.stream_rows(): + assert len(copy) == row + copy.append(cell) + assert copy == changed_data def test_stream_csv_rows() -> None: book = CSVWorkBook([test_book / "data_3x4.csv", test_book / "data_3x6.tsv"]) copy = [] - with book["data_3x4"] as page: - for _, row in page.stream_rows(): - copy.append(row.copy()) + page = book["data_3x4"] + for _, row in page.stream_rows(): + copy.append(row.copy()) assert copy == data_3x4 @@ -262,10 +262,10 @@ def test_bom_is_removed() -> None: book = CSVWorkBook([test_data / "page-desc.csv"]) obtained = [] - with book["page-desc"] as page: - for idx, row in page.stream_rows(): - assert idx == len(obtained) - obtained.append(row.copy()) + page = book["page-desc"] + for idx, row in page.stream_rows(): + assert idx == len(obtained) + obtained.append(row.copy()) expected = [ ["foo", "pid", "glitter"],