perf(validation): stop reloading PDP course CSV for each datetime format

devtools/profile_pdp_course.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+"""Local harness to time/profile PDP course validation on a CSV.
+
+Usage (from edvise-api root, venv active):
+
+  PDP_COURSE_CSV=/path/to/course.csv python devtools/profile_pdp_course.py
+
+Do not commit institution CSVs.
+"""
+
+import os
+import sys
+import time
+
+from src.webapp.validation import _read_pdp_course_edvise
+
+path = os.environ.get("PDP_COURSE_CSV") or (sys.argv[1] if len(sys.argv) > 1 else None)
+if not path:
+    raise SystemExit(
+        "Usage: PDP_COURSE_CSV=/path/to.csv python devtools/profile_pdp_course.py"
+    )
+
+t0 = time.perf_counter()
+df = _read_pdp_course_edvise(path)
+print(f"rows={len(df)} cols={len(df.columns)} elapsed_s={time.perf_counter() - t0:.1f}")

src/webapp/validation.py

@@ -57,15 +57,8 @@
 
 
 def _default_pdp_course_duplicate_converter(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    PDP course duplicate cleanup for read_raw_pdp_course_data.
-
-    Passes the schema selector as the second *positional* argument so this works
-    with current edvise (``schema_type``) and older builds that used the same slot
-    for ``school_type``. Do not pass bare ``handling_duplicates`` as a converter:
-    read_raw_pdp_course_data calls ``converter_func(df)`` with a single argument.
-    """
-    return handling_duplicates(df, "pdp")
+    """PDP course duplicate cleanup for ``read_raw_pdp_course_data``."""
+    return handling_duplicates(df)
 
 
 # --------------------------------------------------------------------------- #
@@ -240,9 +233,6 @@ def _model_list_from_models(models: Union[str, List[str], None]) -> List[str]:
 # converters.
 # --------------------------------------------------------------------------- #
 
-# Datetime formats to try for PDP course (same order as pdp_data_audit)
-PDP_COURSE_DTTM_FORMATS = ("ISO8601", "%Y%m%d.0", "%Y%m%d")
-
 # Datetime formats for ES cohort/course (same order as es_data_audit)
 ES_DTTM_FORMATS = ("ISO8601", "%Y%m%d.0")
 
@@ -617,8 +607,9 @@ def _read_pdp_course_edvise(
     """
     Read and validate a PDP course CSV using edvise helpers.
 
-    Tries each value in ``PDP_COURSE_DTTM_FORMATS`` with each converter: optional
-    ``course_converter_func`` first, then :func:`_default_pdp_course_duplicate_converter`.
+    Tries each converter (optional ``course_converter_func``, then the default
+    duplicate handler). Datetime formats are handled by
+    :func:`read_raw_pdp_course_data`.
 
     Batch PDP jobs may also try school-specific converters from ``dataio``; this
     path only runs converters passed in here, so results may differ from those jobs.
@@ -630,33 +621,31 @@ def _read_pdp_course_edvise(
 
     Returns:
         Validated DataFrame from ``read_raw_pdp_course_data`` for the first successful
-        converter and datetime format.
+        converter.
 
     Raises:
-        HardValidationError: If every converter and format combination fails.
+        HardValidationError: If every converter attempt fails.
     """
-    default_converters = (_default_pdp_course_duplicate_converter,)
     converters = (
         (course_converter_func,) if course_converter_func is not None else ()
-    ) + default_converters
+    ) + (_default_pdp_course_duplicate_converter,)
+    schema = pdp_edvise.get_edvise_schema_for_models(["COURSE"])
     last_error: Optional[Exception] = None
     for converter in converters:
-        for fmt in PDP_COURSE_DTTM_FORMATS:
-            try:
-                return read_raw_pdp_course_data(
-                    file_path=path,
-                    schema=pdp_edvise.get_edvise_schema_for_models(["COURSE"]),
-                    dttm_format=fmt,
-                    converter_func=converter,
-                    spark_session=None,
-                )
-            except ValueError as e:
-                last_error = e
-            except TypeError as e:
-                if "school_type" in str(e) or "schema_type" in str(e):
-                    last_error = None
-                    break
-                raise
+        try:
+            return read_raw_pdp_course_data(
+                file_path=path,
+                schema=schema,
+                converter_func=converter,
+                spark_session=None,
+            )
+        except ValueError as e:
+            last_error = e
+        except TypeError as e:
+            if "school_type" in str(e) or "schema_type" in str(e):
+                last_error = None
+                continue
+            raise
     error_message = (
         "Course data did not parse with any known datetime format."
         if last_error is not None

src/webapp/validation_pdp_read_path_test.py

@@ -431,7 +431,7 @@ def test_read_pdp_course_edvise_success_returns_dataframe() -> None:
 def test_read_pdp_course_edvise_all_attempts_fail_raises_hard_validation_error() -> (
     None
 ):
-    """When all converter/format attempts raise ValueError, HardValidationError is raised."""
+    """When all converter attempts raise ValueError, HardValidationError is raised."""
     with patch(
         "src.webapp.validation.read_raw_pdp_course_data",
         side_effect=ValueError("bad datetime"),
@@ -445,23 +445,18 @@ def test_read_pdp_course_edvise_all_attempts_fail_raises_hard_validation_error()
 
 
 def test_read_pdp_course_edvise_falls_back_after_custom_converter_fails() -> None:
-    """When custom converter fails all datetime formats, default PDP converter is used."""
+    """When custom converter fails, default PDP converter is used."""
     expected = pd.DataFrame({"course_id": ["c1"]})
     with patch(
         "src.webapp.validation.read_raw_pdp_course_data",
-        side_effect=[
-            ValueError("bad datetime"),
-            ValueError("bad datetime"),
-            ValueError("bad datetime"),
-            expected,
-        ],
+        side_effect=[ValueError("bad converter"), expected],
     ) as mock_read:
         result = _read_pdp_course_edvise(
             "/path.csv",
             course_converter_func=lambda df: df,  # noqa: ARG005
         )
     pd.testing.assert_frame_equal(result, expected)
-    assert mock_read.call_count == 4
+    assert mock_read.call_count == 2
 
 
 def test_read_pdp_course_edvise_custom_converter_tried_first() -> None: