diff --git a/devtools/profile_pdp_course.py b/devtools/profile_pdp_course.py new file mode 100644 index 00000000..69849274 --- /dev/null +++ b/devtools/profile_pdp_course.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Local harness to time/profile PDP course validation on a CSV. + +Usage (from edvise-api root, venv active): + + PDP_COURSE_CSV=/path/to/course.csv python devtools/profile_pdp_course.py + +Do not commit institution CSVs. +""" + +import os +import sys +import time + +from src.webapp.validation import _read_pdp_course_edvise + +path = os.environ.get("PDP_COURSE_CSV") or (sys.argv[1] if len(sys.argv) > 1 else None) +if not path: + raise SystemExit( + "Usage: PDP_COURSE_CSV=/path/to.csv python devtools/profile_pdp_course.py" + ) + +t0 = time.perf_counter() +df = _read_pdp_course_edvise(path) +print(f"rows={len(df)} cols={len(df.columns)} elapsed_s={time.perf_counter() - t0:.1f}") diff --git a/src/webapp/validation.py b/src/webapp/validation.py index 86cde8c3..9eeff678 100644 --- a/src/webapp/validation.py +++ b/src/webapp/validation.py @@ -57,15 +57,8 @@ def _default_pdp_course_duplicate_converter(df: pd.DataFrame) -> pd.DataFrame: - """ - PDP course duplicate cleanup for read_raw_pdp_course_data. - - Passes the schema selector as the second *positional* argument so this works - with current edvise (``schema_type``) and older builds that used the same slot - for ``school_type``. Do not pass bare ``handling_duplicates`` as a converter: - read_raw_pdp_course_data calls ``converter_func(df)`` with a single argument. - """ - return handling_duplicates(df, "pdp") + """PDP course duplicate cleanup for ``read_raw_pdp_course_data``.""" + return handling_duplicates(df) # --------------------------------------------------------------------------- # @@ -240,9 +233,6 @@ def _model_list_from_models(models: Union[str, List[str], None]) -> List[str]: # converters. # --------------------------------------------------------------------------- # -# Datetime formats to try for PDP course (same order as pdp_data_audit) -PDP_COURSE_DTTM_FORMATS = ("ISO8601", "%Y%m%d.0", "%Y%m%d") - # Datetime formats for ES cohort/course (same order as es_data_audit) ES_DTTM_FORMATS = ("ISO8601", "%Y%m%d.0") @@ -617,8 +607,9 @@ def _read_pdp_course_edvise( """ Read and validate a PDP course CSV using edvise helpers. - Tries each value in ``PDP_COURSE_DTTM_FORMATS`` with each converter: optional - ``course_converter_func`` first, then :func:`_default_pdp_course_duplicate_converter`. + Tries each converter (optional ``course_converter_func``, then the default + duplicate handler). Datetime formats are handled by + :func:`read_raw_pdp_course_data`. Batch PDP jobs may also try school-specific converters from ``dataio``; this path only runs converters passed in here, so results may differ from those jobs. @@ -630,33 +621,31 @@ def _read_pdp_course_edvise( Returns: Validated DataFrame from ``read_raw_pdp_course_data`` for the first successful - converter and datetime format. + converter. Raises: - HardValidationError: If every converter and format combination fails. + HardValidationError: If every converter attempt fails. """ - default_converters = (_default_pdp_course_duplicate_converter,) converters = ( (course_converter_func,) if course_converter_func is not None else () - ) + default_converters + ) + (_default_pdp_course_duplicate_converter,) + schema = pdp_edvise.get_edvise_schema_for_models(["COURSE"]) last_error: Optional[Exception] = None for converter in converters: - for fmt in PDP_COURSE_DTTM_FORMATS: - try: - return read_raw_pdp_course_data( - file_path=path, - schema=pdp_edvise.get_edvise_schema_for_models(["COURSE"]), - dttm_format=fmt, - converter_func=converter, - spark_session=None, - ) - except ValueError as e: - last_error = e - except TypeError as e: - if "school_type" in str(e) or "schema_type" in str(e): - last_error = None - break - raise + try: + return read_raw_pdp_course_data( + file_path=path, + schema=schema, + converter_func=converter, + spark_session=None, + ) + except ValueError as e: + last_error = e + except TypeError as e: + if "school_type" in str(e) or "schema_type" in str(e): + last_error = None + continue + raise error_message = ( "Course data did not parse with any known datetime format." if last_error is not None diff --git a/src/webapp/validation_pdp_read_path_test.py b/src/webapp/validation_pdp_read_path_test.py index e5a32577..afa5b8fa 100644 --- a/src/webapp/validation_pdp_read_path_test.py +++ b/src/webapp/validation_pdp_read_path_test.py @@ -431,7 +431,7 @@ def test_read_pdp_course_edvise_success_returns_dataframe() -> None: def test_read_pdp_course_edvise_all_attempts_fail_raises_hard_validation_error() -> ( None ): - """When all converter/format attempts raise ValueError, HardValidationError is raised.""" + """When all converter attempts raise ValueError, HardValidationError is raised.""" with patch( "src.webapp.validation.read_raw_pdp_course_data", side_effect=ValueError("bad datetime"), @@ -445,23 +445,18 @@ def test_read_pdp_course_edvise_all_attempts_fail_raises_hard_validation_error() def test_read_pdp_course_edvise_falls_back_after_custom_converter_fails() -> None: - """When custom converter fails all datetime formats, default PDP converter is used.""" + """When custom converter fails, default PDP converter is used.""" expected = pd.DataFrame({"course_id": ["c1"]}) with patch( "src.webapp.validation.read_raw_pdp_course_data", - side_effect=[ - ValueError("bad datetime"), - ValueError("bad datetime"), - ValueError("bad datetime"), - expected, - ], + side_effect=[ValueError("bad converter"), expected], ) as mock_read: result = _read_pdp_course_edvise( "/path.csv", course_converter_func=lambda df: df, # noqa: ARG005 ) pd.testing.assert_frame_equal(result, expected) - assert mock_read.call_count == 4 + assert mock_read.call_count == 2 def test_read_pdp_course_edvise_custom_converter_tried_first() -> None: