diff --git a/backend/app/features/catalog/api.py b/backend/app/features/catalog/api.py index edf4ce43..ee4e459b 100644 --- a/backend/app/features/catalog/api.py +++ b/backend/app/features/catalog/api.py @@ -20,6 +20,7 @@ from app.features.catalog.models import ( Artifact, Case, + DiagnosticProvenanceState, Execution, ExternalLink, MetadataChange, @@ -32,7 +33,9 @@ CaseSummaryOut, CaseUpdate, CatalogOverviewOut, + DiagnosticProvenanceStateOut, DiagnosticsLinkRequest, + DiagnosticsScannerLinkRequest, ExecutionCreate, ExecutionExternalLinkOut, ExecutionFilterOptionsOut, @@ -577,6 +580,115 @@ def link_case_diagnostics( ) +@diagnostics_router.get( + "/scanner-state", response_model=DiagnosticProvenanceStateOut | None +) +def get_diagnostics_scanner_state( + machine: str, + archive_relative_case_path: str, + db: Session = Depends(get_database_session), + user: User = Depends(current_active_user), +) -> DiagnosticProvenanceStateOut | None: + """Return successful scanner state for one machine/archive case path.""" + _require_diagnostics_scanner_role(user) + resolved_machine = resolve_machine_by_name(db, machine) + + if resolved_machine is None: + raise HTTPException(status_code=404, detail="Unknown machine.") + + state = ( + db.query(DiagnosticProvenanceState) + .filter(DiagnosticProvenanceState.machine_name == resolved_machine.name) + .filter( + DiagnosticProvenanceState.archive_relative_case_path + == archive_relative_case_path + ) + .one_or_none() + ) + + return DiagnosticProvenanceStateOut.model_validate(state) if state else None + + +@diagnostics_router.post("/scanner/link", status_code=status.HTTP_204_NO_CONTENT) +def link_scanner_diagnostics( + payload: DiagnosticsScannerLinkRequest, + db: Session = Depends(get_database_session), + user: User = Depends(current_active_user), +) -> None: + """Atomically upsert one scanner-managed case diagnostic link and state.""" + _require_diagnostics_scanner_role(user) + + if len(payload.diagnostics) != 1: + raise HTTPException( + status_code=422, detail="Scanner payload requires one diagnostic." + ) + + if _unsafe_archive_relative_path(payload.provenance.archive_relative_case_path): + raise HTTPException( + status_code=422, detail="Invalid archive-relative case path." + ) + + machine = resolve_machine_by_name(db, payload.machine) + if machine is None: + raise HTTPException(status_code=404, detail="No matching case found.") + + case_id = _resolve_case_id_for_diagnostics_link( + db=db, + case_name=payload.case_name, + machine_name=payload.machine, + hpc_username=payload.hpc_username, + ) + diagnostic = payload.diagnostics[0] + now = datetime.now(timezone.utc) + + with transaction(db): + link_id = db.execute( + pg_insert(ExternalLink) + .values( + case_id=case_id, + kind=ExternalLinkKind.DIAGNOSTIC, + url=str(diagnostic.url), + label=diagnostic.name, + created_at=now, + updated_at=now, + ) + .on_conflict_do_update( + index_elements=[ + ExternalLink.case_id, + ExternalLink.kind, + ExternalLink.url, + ], + index_where=ExternalLink.case_id.is_not(None), + set_={"label": diagnostic.name, "updated_at": now}, + ) + .returning(ExternalLink.id) + ).scalar_one() + db.execute( + pg_insert(DiagnosticProvenanceState) + .values( + link_id=link_id, + machine_name=machine.name, + archive_relative_case_path=payload.provenance.archive_relative_case_path, + settings_filename=payload.provenance.settings_filename, + provenance_timestamp=payload.provenance.provenance_timestamp, + fingerprint=payload.provenance.fingerprint, + linked_url=str(diagnostic.url), + submitted_at=now, + ) + .on_conflict_do_update( + constraint="uq_diagnostic_provenance_states_machine_path", + set_={ + "link_id": link_id, + "settings_filename": payload.provenance.settings_filename, + "provenance_timestamp": payload.provenance.provenance_timestamp, + "fingerprint": payload.provenance.fingerprint, + "linked_url": str(diagnostic.url), + "submitted_at": now, + }, + ) + ) + + @execution_router.get( "", response_model=ExecutionPageOut, @@ -1048,6 +1160,20 @@ def _resolve_case_id_for_diagnostics_link( return match[0] +def _require_diagnostics_scanner_role(user: User) -> None: + if user.role not in (UserRole.ADMIN, UserRole.SERVICE_ACCOUNT): + raise HTTPException( + status_code=403, + detail="Scanner access requires an administrator or service account.", + ) + + +def _unsafe_archive_relative_path(value: str) -> bool: + return value.startswith("/") or any( + part in {"", ".", ".."} for part in value.split("/") + ) + + def _upsert_case_diagnostic_links( *, db: Session, diff --git a/backend/app/features/catalog/models.py b/backend/app/features/catalog/models.py index 77c3a6cb..47f8d2e4 100644 --- a/backend/app/features/catalog/models.py +++ b/backend/app/features/catalog/models.py @@ -322,3 +322,48 @@ class ExternalLink(Base, IDMixin, TimestampMixin): foreign_keys=[case_id], passive_deletes=True, ) + diagnostic_provenance_state: Mapped[DiagnosticProvenanceState | None] = ( + relationship( + back_populates="link", + cascade="all, delete-orphan", + passive_deletes=True, + uselist=False, + ) + ) + + +class DiagnosticProvenanceState(Base, IDMixin): + """Successful scanner submission state for one published diagnostics link.""" + + __tablename__ = "diagnostic_provenance_states" + __table_args__ = ( + UniqueConstraint( + "machine_name", + "archive_relative_case_path", + name="uq_diagnostic_provenance_states_machine_path", + ), + UniqueConstraint("link_id", name="uq_diagnostic_provenance_states_link_id"), + ) + + link_id: Mapped[UUID] = mapped_column( + PG_UUID(as_uuid=True), + ForeignKey("external_links.id", ondelete="CASCADE"), + nullable=False, + ) + machine_name: Mapped[str] = mapped_column(String(200), nullable=False) + archive_relative_case_path: Mapped[str] = mapped_column(Text, nullable=False) + settings_filename: Mapped[str] = mapped_column(String(255), nullable=False) + provenance_timestamp: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + fingerprint: Mapped[str] = mapped_column(String(128), nullable=False) + linked_url: Mapped[str] = mapped_column(String(1000), nullable=False) + submitted_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + + link: Mapped[ExternalLink] = relationship( + back_populates="diagnostic_provenance_state", + foreign_keys=[link_id], + passive_deletes=True, + ) diff --git a/backend/app/features/catalog/schemas.py b/backend/app/features/catalog/schemas.py index fb2fc91e..0501d81f 100644 --- a/backend/app/features/catalog/schemas.py +++ b/backend/app/features/catalog/schemas.py @@ -152,6 +152,33 @@ class DiagnosticsLinkRequest(CamelInBaseModel): ] +class DiagnosticProvenanceMetadata(CamelInBaseModel): + """Immutable provenance identity supplied by the diagnostics scanner.""" + + archive_relative_case_path: Annotated[ + str, Field(..., min_length=1, max_length=1000) + ] + settings_filename: Annotated[str, Field(..., min_length=1, max_length=255)] + provenance_timestamp: datetime + fingerprint: Annotated[str, Field(..., min_length=1, max_length=128)] + + +class DiagnosticsScannerLinkRequest(DiagnosticsLinkRequest): + """Scanner-only diagnostics link request with successful provenance state.""" + + provenance: DiagnosticProvenanceMetadata + + +class DiagnosticProvenanceStateOut(CamelOutBaseModel): + machine_name: str + archive_relative_case_path: str + settings_filename: str + provenance_timestamp: datetime + fingerprint: str + linked_url: str + submitted_at: datetime + + class ArtifactCreate(CamelInBaseModel): """Schema for creating a new Artifact.""" diff --git a/backend/app/scripts/README.md b/backend/app/scripts/README.md index 58839f68..a2a2c5c4 100644 --- a/backend/app/scripts/README.md +++ b/backend/app/scripts/README.md @@ -18,9 +18,13 @@ scripts/ │ ├── archive_ingestor_core.py │ ├── archive_layout.py │ ├── archive_workflow.py +│ ├── diagnostics_archives.py +│ ├── diagnostics_link_scanner.py │ ├── hpc_upload_archive_ingestor.py │ ├── nersc_archive_ingestor.py │ └── sites/ +│ ├── lcrc-diagnostics-scanner.sh +│ ├── nersc-diagnostics-scanner.sh │ └── nersc.sh ├── db/ │ ├── seed.py @@ -157,6 +161,36 @@ Archive notes: - `ARCHIVE_YEAR_START` / `ARCHIVE_YEAR_END` are intended for scoped backfills so operators can avoid scanning the full historical tree when unnecessary. - `YYYY` values expand to full-year bounds (`START=2020` means `2020-01`; `END=2020` means `2020-12`), while `YYYY-MM` values target exact archive month buckets. +## Diagnostics Provenance Scanner + +Scans newest paired zppy provenance from the reviewed static registry and creates +case-scoped diagnostic links. It never reads Mache configuration at runtime. + +Run through the NERSC wrapper: + +```bash +SIMBOARD_API_TOKEN= \ +MACHINE_NAME=perlmutter \ +DRY_RUN=true \ +backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh +``` + +Use `sites/lcrc-diagnostics-scanner.sh` at LCRC with +`MACHINE_NAME=chrysalis`. `MACHINE_NAME` is required for every diagnostics +scanner invocation; wrappers do not assign a machine default. A non-dry run +also requires an API base URL and service-account token. Roots and public URLs +come only from `diagnostics_archives.py`. + +Start with `DRY_RUN=true`; it needs no API URL or token. Inspect logs, then +schedule with `DRY_RUN=false`, which requires both API URL and service token. +The scanner emits structured events for startup configuration, discovery, +candidate selection, state lookups, retry outcomes, and completion; credentials +are never logged. Dry runs also emit one candidate event per discovered link. +Scanner account needs read/traverse access to `production/` and `development/`, +provenance settings, and published output. Failed or not-ready candidates retry +next run. Refresh registry entries from Mache `[web_portal]` cfg data only in a +reviewed change; never add archive-path environment overrides. + ## HPC Upload Archive Ingestor The HPC upload archive ingestor uses the same scan, state, dry-run, retry, and diff --git a/backend/app/scripts/ingestion/diagnostics_archives.py b/backend/app/scripts/ingestion/diagnostics_archives.py new file mode 100644 index 00000000..33caf3c4 --- /dev/null +++ b/backend/app/scripts/ingestion/diagnostics_archives.py @@ -0,0 +1,35 @@ +"""Reviewed diagnostics archive locations; never populated at scanner runtime.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class DiagnosticsArchive: + root: str + public_base_url: str + + +# Refresh from Mache [web_portal] configuration in a reviewed change when sites move. +# Source: https://github.com/E3SM-Project/mache/tree/main/mache/machines +DIAGNOSTICS_ARCHIVES_BY_MACHINE: dict[str, DiagnosticsArchive] = { + "perlmutter": DiagnosticsArchive( + root="/global/cfs/cdirs/e3sm/www/diagnostics_archive", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostics_archive", + ), + "pm": DiagnosticsArchive( + root="/global/cfs/cdirs/e3sm/www/diagnostics_archive", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostics_archive", + ), + "pm-cpu": DiagnosticsArchive( + root="/global/cfs/cdirs/e3sm/www/diagnostics_archive", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostics_archive", + ), + "pm-gpu": DiagnosticsArchive( + root="/global/cfs/cdirs/e3sm/www/diagnostics_archive", + public_base_url="https://portal.nersc.gov/cfs/e3sm/diagnostics_archive", + ), + "chrysalis": DiagnosticsArchive( + root="/lcrc/group/e3sm/public_html/diagnostic_output/diagnostics_archive", + public_base_url="https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/diagnostics_archive", + ), +} diff --git a/backend/app/scripts/ingestion/diagnostics_link_scanner.py b/backend/app/scripts/ingestion/diagnostics_link_scanner.py new file mode 100644 index 00000000..6d355cb5 --- /dev/null +++ b/backend/app/scripts/ingestion/diagnostics_link_scanner.py @@ -0,0 +1,402 @@ +"""Discover published zppy provenance and submit case diagnostics links.""" + +from __future__ import annotations + +import hashlib +import os +import re +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlparse, urlunparse + +import httpx + +from app.scripts.ingestion.archive_ingestor_core import _log_event +from app.scripts.ingestion.diagnostics_archives import ( + DIAGNOSTICS_ARCHIVES_BY_MACHINE, + DiagnosticsArchive, +) + +TIMESTAMP_RE = re.compile(r"^provenance\.(\d{8}_\d{6}_\d{6})\.cfg$") +REQUIRED_SETTINGS = {"case_name", "machine", "hpc_username", "diagnostics_url"} +MAX_SETTINGS_BYTES = 64 * 1024 +MAX_SETTINGS_LINES = 200 + + +@dataclass(frozen=True) +class Candidate: + path: Path + settings: Path + timestamp: datetime + values: dict[str, str] + fingerprint: str + + +def run() -> int: + machine = os.environ.get("MACHINE_NAME", "").strip() + if not machine: + raise ValueError("MACHINE_NAME is required") + + archive = _resolve_archive(machine) + root = Path(archive.root) + dry_run = os.environ.get("DRY_RUN", "true").lower() in {"1", "true", "yes"} + + summary = { + "discovered_candidates": 0, + "dry_run_candidates": 0, + "unchanged_candidates": 0, + "deferred_state_lookups": 0, + "submitted_links": 0, + "failed_link_submissions": 0, + } + _log_event( + "diagnostics_scanner_startup_configuration", + { + "machine_name": machine, + "archive_root": str(root), + "public_base_url": _sanitize_url(archive.public_base_url), + "dry_run": dry_run, + "has_api_base_url": bool(os.environ.get("SIMBOARD_API_BASE_URL")), + "has_api_token": bool(os.environ.get("SIMBOARD_API_TOKEN")), + }, + ) + + candidates = _discover(root, archive.public_base_url) + summary["discovered_candidates"] = len(candidates) + _log_event("diagnostics_scanner_discovery_completed", summary.copy()) + + if dry_run: + for candidate in candidates: + relative = candidate.path.parent.relative_to(root).as_posix() + summary["dry_run_candidates"] += 1 + _log_event( + "diagnostics_scanner_dry_run_candidate", + { + "archive_relative_case_path": relative, + "settings_filename": candidate.settings.name, + "fingerprint": candidate.fingerprint, + }, + ) + _log_event("diagnostics_scanner_dry_run_completed", summary.copy()) + _log_event("diagnostics_scanner_completed", summary.copy()) + + return 0 + + api_base = os.environ["SIMBOARD_API_BASE_URL"].rstrip("/") + token = os.environ["SIMBOARD_API_TOKEN"] + headers = {"Authorization": f"Bearer {token}"} + + with httpx.Client(timeout=30) as client: + for candidate in candidates: + relative = candidate.path.parent.relative_to(root).as_posix() + + state = _request_with_retry( + client.get, + f"{api_base}/api/v1/diagnostics/scanner-state", + params={"machine": machine, "archive_relative_case_path": relative}, + headers=headers, + ) + _log_event( + "diagnostics_scanner_state_lookup_result", + { + "archive_relative_case_path": relative, + "status_code": None if state is None else state.status_code, + }, + ) + + if state is None or state.status_code != 200: + summary["deferred_state_lookups"] += 1 + _log_event( + "diagnostics_scanner_state_lookup_deferred", + { + "archive_relative_case_path": relative, + "status_code": None if state is None else state.status_code, + }, + ) + continue + + state_payload = state.json() if state.content else None + if state_payload and ( + state_payload.get("settingsFilename") == candidate.settings.name + and state_payload.get("fingerprint") == candidate.fingerprint + ): + summary["unchanged_candidates"] += 1 + _log_event( + "diagnostics_scanner_skipped_unchanged", + { + "archive_relative_case_path": relative, + "settings_filename": candidate.settings.name, + "fingerprint": candidate.fingerprint, + }, + ) + continue + + payload = { + "caseName": candidate.values["case_name"], + "machine": candidate.values["machine"], + "hpcUsername": candidate.values["hpc_username"], + "diagnostics": [ + { + "name": "zppy diagnostics", + "url": candidate.values["diagnostics_url"], + "kind": "diagnostic", + } + ], + "provenance": { + "archiveRelativeCasePath": relative, + "settingsFilename": candidate.settings.name, + "provenanceTimestamp": candidate.timestamp.isoformat(), + "fingerprint": candidate.fingerprint, + }, + } + + response = _request_with_retry( + client.post, + f"{api_base}/api/v1/diagnostics/scanner/link", + json=payload, + headers=headers, + ) + + if response is None or response.status_code != 204: + summary["failed_link_submissions"] += 1 + _log_event( + "diagnostics_scanner_link_submission_failed", + { + "archive_relative_case_path": relative, + "status_code": None + if response is None + else response.status_code, + }, + ) + else: + summary["submitted_links"] += 1 + _log_event( + "diagnostics_scanner_link_submitted", + { + "archive_relative_case_path": relative, + "status_code": response.status_code, + }, + ) + + _log_event("diagnostics_scanner_completed", summary) + + return 0 + + +def _resolve_archive(machine_name: str) -> DiagnosticsArchive: + archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE.get(machine_name.lower()) + if archive is None: + raise ValueError(f"Unsupported diagnostics scanner machine: {machine_name}") + + root = Path(archive.root) + parsed = urlparse(archive.public_base_url) + if not root.is_absolute() or not root.is_dir() or not os.access(root, os.R_OK): + raise ValueError(f"Diagnostics archive is not readable: {root}") + + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("Diagnostics archive public URL must be absolute HTTP(S)") + + return archive + + +def _sanitize_url(url: str) -> str: + """Return a URL safe to include in structured logs.""" + parsed = urlparse(url) + hostname = parsed.hostname or "" + netloc = hostname + + if parsed.port is not None: + netloc = f"{netloc}:{parsed.port}" + + return urlunparse((parsed.scheme, netloc, parsed.path, "", "", "")) + + +def _discover(root: Path, public_base_url: str) -> list[Candidate]: # noqa: C901 + base = urlparse(public_base_url) + candidates: list[Candidate] = [] + + for tier in ("production", "development"): + tier_root = root / tier + + if not tier_root.is_dir(): + continue + + newest_by_case: dict[Path, tuple[Path, datetime]] = {} + + for cfg in tier_root.rglob("provenance.*.cfg"): + try: + if cfg.is_symlink() or root not in cfg.resolve().parents: + continue + except OSError as exc: + _log_invalid_provenance(root, cfg, exc) + continue + + timestamp = _timestamp(cfg) + if timestamp is None: + continue + + case_dir = cfg.parent + prior = newest_by_case.get(case_dir) + + if prior is None or timestamp > prior[1]: + newest_by_case[case_dir] = (cfg, timestamp) + + for case_dir, (cfg, timestamp) in newest_by_case.items(): + settings = cfg.with_suffix(".settings") + try: + if ( + settings.is_symlink() + or root not in settings.resolve().parents + or not settings.is_file() + or not _published_output(case_dir, root) + ): + continue + + settings_bytes = _read_settings_bytes(settings) + values = _parse_settings_bytes(settings_bytes) + url = urlparse(values["diagnostics_url"]) + + if (url.scheme, url.netloc) != ( + base.scheme, + base.netloc, + ) or not url.path.startswith(base.path.rstrip("/") + "/"): + raise ValueError( + "Diagnostics URL outside configured public archive" + ) + + _validate_layout(case_dir, root, values) + + digest = hashlib.sha256(settings_bytes).hexdigest() + candidates.append(Candidate(cfg, settings, timestamp, values, digest)) + except (OSError, UnicodeError, ValueError) as exc: + _log_invalid_provenance(root, cfg, exc) + + return candidates + + +def _log_invalid_provenance(root: Path, cfg: Path, exc: Exception) -> None: + """Log a malformed or inaccessible provenance file without halting discovery.""" + _log_event( + "diagnostics_scanner_invalid_provenance", + { + "provenance_path": cfg.relative_to(root).as_posix(), + "reason": str(exc), + }, + ) + + +def _request_with_retry(method, url: str, **kwargs) -> httpx.Response | None: + for attempt in range(3): + try: + response = method(url, **kwargs) + except httpx.RequestError: + response = None + if ( + response is not None + and response.status_code not in {408, 429} + and response.status_code < 500 + ): + return response + + if attempt < 2: + _log_event( + "diagnostics_scanner_request_retry_scheduled", + { + "attempt": attempt + 1, + "max_attempts": 3, + "status_code": None if response is None else response.status_code, + "request_error": response is None, + }, + ) + time.sleep(2**attempt) + + _log_event( + "diagnostics_scanner_request_retry_exhausted", + { + "attempts": 3, + "status_code": None if response is None else response.status_code, + "request_error": response is None, + }, + ) + return response + + +def _read_settings_bytes(path: Path) -> bytes: + with path.open("rb") as settings_file: + content = settings_file.read(MAX_SETTINGS_BYTES + 1) + + if len(content) > MAX_SETTINGS_BYTES: + raise ValueError("Provenance settings file is too large") + + return content + + +def _parse_settings_bytes(content: bytes) -> dict[str, str]: + values: dict[str, str] = {} + + for line_number, line in enumerate(content.decode("utf-8").splitlines(), start=1): + if line_number > MAX_SETTINGS_LINES: + raise ValueError("Provenance settings file has too many lines") + if not line.strip() or line.lstrip().startswith("#"): + continue + + if "=" not in line: + raise ValueError("Malformed provenance settings line") + + key, value = (part.strip() for part in line.split("=", 1)) + + if not key or not value or key in values: + raise ValueError("Malformed or duplicate provenance setting") + + values[key] = value + + if REQUIRED_SETTINGS - values.keys(): + raise ValueError("Missing required provenance settings") + + return values + + +def _published_output(case_dir: Path, root: Path) -> bool: + for entry in case_dir.iterdir(): + if entry.name.startswith("provenance."): + continue + + if entry.is_symlink() or root not in entry.resolve().parents: + continue + + if entry.is_file() or (entry.is_dir() and any(entry.iterdir())): + return True + + return False + + +def _validate_layout(case_dir: Path, root: Path, values: dict[str, str]) -> None: + parts = case_dir.relative_to(root).parts + + if len(parts) not in {3, 4} or parts[0] not in {"production", "development"}: + raise ValueError("Invalid diagnostics archive case layout") + + if values["case_name"] != parts[-1]: + raise ValueError("Provenance case_name does not match archive layout") + + expected_group = parts[-2] if len(parts) == 3 else None + + if values.get("case_group") != expected_group: + raise ValueError("Provenance case_group does not match archive layout") + + +def _timestamp(cfg: Path) -> datetime | None: + match = TIMESTAMP_RE.match(cfg.name) + + if match is None: + return None + + return datetime.strptime(match.group(1), "%Y%m%d_%H%M%S_%f").replace( + tzinfo=timezone.utc + ) + + +if __name__ == "__main__": + raise SystemExit(run()) diff --git a/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh new file mode 100755 index 00000000..ae629ab9 --- /dev/null +++ b/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +BACKEND_DIR="$(cd -- "${SCRIPT_DIR}/../../../../" && pwd)" +PYTHON_BIN="${PYTHON_BIN:-${BACKEND_DIR}/.venv/bin/python}" + +[[ -x "${PYTHON_BIN}" ]] || { echo "Missing backend Python environment" >&2; exit 1; } + +export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api.e3sm.org}" +: "${MACHINE_NAME:?MACHINE_NAME must be set}" +export DRY_RUN="${DRY_RUN:-true}" + +DRY_RUN_NORMALIZED="$(printf '%s' "${DRY_RUN}" | tr '[:upper:]' '[:lower:]')" +if [[ "${DRY_RUN_NORMALIZED}" != "true" && "${DRY_RUN}" != "1" && "${DRY_RUN_NORMALIZED}" != "yes" ]]; then + : "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set when DRY_RUN is false}" +fi + +cd "${BACKEND_DIR}" +exec "${PYTHON_BIN}" -m app.scripts.ingestion.diagnostics_link_scanner "$@" diff --git a/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh b/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh new file mode 100755 index 00000000..8d49feb5 --- /dev/null +++ b/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +BACKEND_DIR="$(cd -- "${SCRIPT_DIR}/../../../../" && pwd)" +PYTHON_BIN="${PYTHON_BIN:-${BACKEND_DIR}/.venv/bin/python}" + +[[ -x "${PYTHON_BIN}" ]] || { echo "Missing backend Python environment" >&2; exit 1; } + +export SIMBOARD_API_BASE_URL="${SIMBOARD_API_BASE_URL:-https://simboard-dev-api.e3sm.org}" +: "${MACHINE_NAME:?MACHINE_NAME must be set}" +export DRY_RUN="${DRY_RUN:-true}" + +if [[ "${DRY_RUN,,}" != "true" && "${DRY_RUN}" != "1" && "${DRY_RUN,,}" != "yes" ]]; then + : "${SIMBOARD_API_TOKEN:?SIMBOARD_API_TOKEN must be set when DRY_RUN is false}" +fi + +cd "${BACKEND_DIR}" +exec "${PYTHON_BIN}" -m app.scripts.ingestion.diagnostics_link_scanner "$@" diff --git a/backend/app/scripts/ingestion/sites/nersc.crontab.example b/backend/app/scripts/ingestion/sites/nersc.crontab.example index f154738c..974a92be 100644 --- a/backend/app/scripts/ingestion/sites/nersc.crontab.example +++ b/backend/app/scripts/ingestion/sites/nersc.crontab.example @@ -24,3 +24,9 @@ OLD_PERF_ARCHIVE_ROOT=/global/cfs/projectdirs/e3sm/OLD_PERF # Archive scan: run daily at 03:15 UTC. Add ARCHIVE_YEAR_START / ARCHIVE_YEAR_END # here only when you want a scoped archive backfill. Values may use YYYY or YYYY-MM. 15 3 * * * cd ${REPO_DIR} && SCAN_MODE=archive ARCHIVE_YEAR_START=2025-01 ARCHIVE_YEAR_END=2025-03 ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc.sh.log 2>&1 + +# Diagnostics provenance scan: start dry-run, inspect logs, then set DRY_RUN=false. +20 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/nersc-diagnostics-scanner.log 2>&1 + +# Chrysalis diagnostics provenance scan; use its local checkout path for REPO_DIR. +25 * * * * cd ${REPO_DIR} && ${REPO_DIR}/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.sh >> ${REPO_DIR}/backend/app/scripts/ingestion/sites/lcrc-diagnostics-scanner.log 2>&1 diff --git a/backend/migrations/versions/20260811_000000_add_diagnostic_provenance_state.py b/backend/migrations/versions/20260811_000000_add_diagnostic_provenance_state.py new file mode 100644 index 00000000..27ea92ca --- /dev/null +++ b/backend/migrations/versions/20260811_000000_add_diagnostic_provenance_state.py @@ -0,0 +1,44 @@ +"""Add successful diagnostics scanner provenance state. + +Revision ID: 20260811_000000 +Revises: 20260728_010000 +Create Date: 2026-08-11 00:00:00.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "20260811_000000" +down_revision: Union[str, Sequence[str], None] = "20260728_010000" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "diagnostic_provenance_states", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("link_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("machine_name", sa.String(length=200), nullable=False), + sa.Column("archive_relative_case_path", sa.Text(), nullable=False), + sa.Column("settings_filename", sa.String(length=255), nullable=False), + sa.Column("provenance_timestamp", sa.DateTime(timezone=True), nullable=False), + sa.Column("fingerprint", sa.String(length=128), nullable=False), + sa.Column("linked_url", sa.String(length=1000), nullable=False), + sa.Column("submitted_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["link_id"], ["external_links.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("link_id", name="uq_diagnostic_provenance_states_link_id"), + sa.UniqueConstraint( + "machine_name", + "archive_relative_case_path", + name="uq_diagnostic_provenance_states_machine_path", + ), + ) + + +def downgrade() -> None: + op.drop_table("diagnostic_provenance_states") diff --git a/backend/tests/features/catalog/test_diagnostic_provenance_state.py b/backend/tests/features/catalog/test_diagnostic_provenance_state.py new file mode 100644 index 00000000..ec5ee30f --- /dev/null +++ b/backend/tests/features/catalog/test_diagnostic_provenance_state.py @@ -0,0 +1,236 @@ +from unittest.mock import patch +from uuid import uuid4 + +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + +from app.api.version import API_BASE +from app.features.catalog.models import DiagnosticProvenanceState, ExternalLink +from app.features.machine.models import Machine +from app.features.user.manager import current_active_user +from app.features.user.models import User, UserRole +from app.main import app +from tests.features.catalog.test_api import ( + _create_matching_execution, + _create_service_account_token, + use_real_auth, +) + + +def _payload(*, case_name: str, machine: str, path: str) -> dict: + return { + "caseName": case_name, + "machine": machine, + "hpcUsername": "scanner-user", + "diagnostics": [ + { + "name": "zppy diagnostics", + "url": "https://diagnostics.example.org/archive/case", + "kind": "diagnostic", + } + ], + "provenance": { + "archiveRelativeCasePath": path, + "settingsFilename": "provenance.20260811_120000_000000.settings", + "provenanceTimestamp": "2026-08-11T12:00:00Z", + "fingerprint": "a" * 64, + }, + } + + +def _matching_case(db: Session): + machine = db.query(Machine).first() + assert machine is not None + user, token = _create_service_account_token(db) + case, _ = _create_matching_execution( + db, + case_name=f"scanner-state-{uuid4()}", + machine_id=machine.id, + machine_name=machine.name, + user_id=user.id, + execution_id=f"scanner-{uuid4()}", + hpc_username="scanner-user", + source_reference=f"scanner-state-{uuid4()}", + ) + return machine, user, token, case + + +@use_real_auth +def test_scanner_link_is_idempotent_and_state_is_readable(client, db: Session) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="production/e3sm/case" + ) + headers = {"Authorization": f"Bearer {token}"} + + assert ( + client.post( + f"{API_BASE}/diagnostics/scanner/link", json=payload, headers=headers + ).status_code + == 204 + ) + assert ( + client.post( + f"{API_BASE}/diagnostics/scanner/link", json=payload, headers=headers + ).status_code + == 204 + ) + + state = db.query(DiagnosticProvenanceState).one() + assert state.machine_name == machine.name + assert state.settings_filename == payload["provenance"]["settingsFilename"] + assert db.query(ExternalLink).filter(ExternalLink.case_id == case.id).count() == 1 + + response = client.get( + f"{API_BASE}/diagnostics/scanner-state", + params={ + "machine": machine.name, + "archive_relative_case_path": "production/e3sm/case", + }, + headers=headers, + ) + assert response.status_code == 200 + assert response.json()["fingerprint"] == "a" * 64 + + +@use_real_auth +def test_scanner_state_returns_404_for_unknown_machine(client, db: Session) -> None: + _, _, token, _ = _matching_case(db) + + response = client.get( + f"{API_BASE}/diagnostics/scanner-state", + params={ + "machine": "unknown-machine", + "archive_relative_case_path": "production/e3sm/case", + }, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "Unknown machine." + + +@use_real_auth +def test_scanner_link_rejects_multiple_diagnostics(client, db: Session) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="production/e3sm/case" + ) + payload["diagnostics"].append(payload["diagnostics"][0].copy()) + + response = client.post( + f"{API_BASE}/diagnostics/scanner/link", + json=payload, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "Scanner payload requires one diagnostic." + + +@use_real_auth +def test_scanner_link_rejects_unsafe_archive_path(client, db: Session) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload(case_name=case.name, machine=machine.name, path="../outside") + + response = client.post( + f"{API_BASE}/diagnostics/scanner/link", + json=payload, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 422 + assert response.json()["detail"] == "Invalid archive-relative case path." + + +@use_real_auth +def test_scanner_link_returns_404_for_unknown_machine(client, db: Session) -> None: + _, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine="unknown-machine", path="production/e3sm/case" + ) + + response = client.post( + f"{API_BASE}/diagnostics/scanner/link", + json=payload, + headers={"Authorization": f"Bearer {token}"}, + ) + + assert response.status_code == 404 + assert response.json()["detail"] == "No matching case found." + + +@use_real_auth +def test_scanner_endpoints_reject_regular_user(client, db: Session) -> None: + machine, _, _, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="production/e3sm/case" + ) + regular_user = User( + id=uuid4(), + email="regular-scanner-user@example.com", + is_active=True, + is_verified=True, + role=UserRole.USER, + ) + app.dependency_overrides[current_active_user] = lambda: regular_user + try: + response = client.post(f"{API_BASE}/diagnostics/scanner/link", json=payload) + finally: + app.dependency_overrides.pop(current_active_user, None) + + assert response.status_code == 403 + assert response.json()["detail"] == ( + "Scanner access requires an administrator or service account." + ) + + +@use_real_auth +def test_scanner_link_rolls_back_link_when_state_write_fails( + client, db: Session +) -> None: + machine, service_user, _, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="production/e3sm/fail" + ) + original_execute = db.execute + + def fail_only_state_insert(statement, *args, **kwargs): + table = getattr(statement, "table", None) + if table is not None and table.name == "diagnostic_provenance_states": + raise RuntimeError("state failure") + return original_execute(statement, *args, **kwargs) + + app.dependency_overrides[current_active_user] = lambda: service_user + try: + with patch.object(db, "execute", side_effect=fail_only_state_insert): + with TestClient(app, raise_server_exceptions=False) as error_client: + response = error_client.post( + f"{API_BASE}/diagnostics/scanner/link", json=payload + ) + finally: + app.dependency_overrides.pop(current_active_user, None) + + assert response.status_code == 500 + assert db.query(ExternalLink).filter(ExternalLink.case_id == case.id).count() == 0 + + +@use_real_auth +def test_deleting_scanner_link_cascades_provenance_state(client, db: Session) -> None: + machine, _, token, case = _matching_case(db) + payload = _payload( + case_name=case.name, machine=machine.name, path="development/e3sm/case" + ) + assert ( + client.post( + f"{API_BASE}/diagnostics/scanner/link", + json=payload, + headers={"Authorization": f"Bearer {token}"}, + ).status_code + == 204 + ) + + link = db.query(ExternalLink).filter(ExternalLink.case_id == case.id).one() + db.delete(link) + db.commit() + assert db.query(DiagnosticProvenanceState).count() == 0 diff --git a/backend/tests/features/ingestion/test_diagnostics_link_scanner.py b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py new file mode 100644 index 00000000..fdfb4236 --- /dev/null +++ b/backend/tests/features/ingestion/test_diagnostics_link_scanner.py @@ -0,0 +1,335 @@ +from pathlib import Path + +import httpx +import pytest + +from app.scripts.ingestion.diagnostics_archives import ( + DIAGNOSTICS_ARCHIVES_BY_MACHINE, + DiagnosticsArchive, +) +from app.scripts.ingestion.diagnostics_link_scanner import ( + _discover, + _parse_settings_bytes, + _read_settings_bytes, + _request_with_retry, + run, +) + +BASE_URL = "https://diagnostics.example.org/archive" + + +def test_chrysalis_diagnostics_archive_settings() -> None: + archive = DIAGNOSTICS_ARCHIVES_BY_MACHINE["chrysalis"] + assert ( + archive.root + == "/lcrc/group/e3sm/public_html/diagnostic_output/diagnostics_archive" + ) + assert ( + archive.public_base_url + == "https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/diagnostics_archive" + ) + + +@pytest.mark.parametrize("machine_name", [None, " "]) +def test_run_requires_machine_name_before_archive_resolution( + monkeypatch: pytest.MonkeyPatch, machine_name: str | None +) -> None: + def resolve_archive(_machine: str) -> DiagnosticsArchive: + pytest.fail("archive resolution must not be called without MACHINE_NAME") + raise AssertionError("unreachable") + + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", + resolve_archive, + ) + if machine_name is None: + monkeypatch.delenv("MACHINE_NAME", raising=False) + else: + monkeypatch.setenv("MACHINE_NAME", machine_name) + + with pytest.raises(ValueError, match="MACHINE_NAME is required"): + run() + + +def _case(root: Path, path: str, *, timestamp: str = "20260811_120000_000000") -> Path: + directory = root / path + directory.mkdir(parents=True) + cfg = directory / f"provenance.{timestamp}.cfg" + cfg.write_text("cfg", encoding="utf-8") + case_group = ( + f"case_group = {directory.parent.name}\n" + if len(directory.relative_to(root).parts) == 3 + else "" + ) + cfg.with_suffix(".settings").write_text( + f"case_name = {directory.name}\nmachine = perlmutter\nhpc_username = user\n" + f"{case_group}" + "diagnostics_url = https://diagnostics.example.org/archive/case\n", + encoding="utf-8", + ) + (directory / "index.html").write_text("ready", encoding="utf-8") + return directory + + +def test_newest_missing_settings_defers_without_stale_fallback(tmp_path: Path) -> None: + directory = _case(tmp_path, "production/type/case") + (directory / "provenance.20260812_120000_000000.cfg").write_text("cfg") + assert _discover(tmp_path, BASE_URL) == [] + + +def test_discovery_rejects_case_and_group_mismatches(tmp_path: Path) -> None: + directory = _case(tmp_path, "development/type/group/case") + settings = next(directory.glob("*.settings")) + settings.write_text( + settings.read_text().replace("case_name = case", "case_name = wrong") + ) + assert _discover(tmp_path, BASE_URL) == [] + + +def test_parse_settings_rejects_duplicate_required_key(tmp_path: Path) -> None: + settings = tmp_path / "provenance.settings" + settings.write_text("case_name = one\ncase_name = two\n", encoding="utf-8") + with pytest.raises(ValueError): + _parse_settings_bytes(settings.read_bytes()) + + +def test_settings_reader_rejects_oversized_file(tmp_path: Path) -> None: + settings = tmp_path / "provenance.settings" + settings.write_bytes(b"x" * (64 * 1024 + 1)) + with pytest.raises(ValueError): + _read_settings_bytes(settings) + + +def test_discovery_rejects_settings_symlink_outside_root(tmp_path: Path) -> None: + directory = _case(tmp_path, "production/type/case") + settings = next(directory.glob("*.settings")) + outside = tmp_path.parent / "outside.settings" + outside.write_text(settings.read_text(), encoding="utf-8") + settings.unlink() + settings.symlink_to(outside) + assert _discover(tmp_path, BASE_URL) == [] + + +def test_discovery_rejects_external_output_directory_symlink(tmp_path: Path) -> None: + directory = _case(tmp_path, "production/type/case") + (directory / "index.html").unlink() + outside = tmp_path.parent / "outside-output" + outside.mkdir(exist_ok=True) + (outside / "index.html").write_text("ready") + (directory / "output").symlink_to(outside, target_is_directory=True) + assert _discover(tmp_path, BASE_URL) == [] + + +def test_discovery_continues_after_published_output_oserror( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + failing_case = _case(tmp_path, "production/type/failing") + valid_case = _case(tmp_path, "production/type/valid") + for case_dir, case_name in ((failing_case, "failing"), (valid_case, "valid")): + settings = next(case_dir.glob("*.settings")) + settings.write_text( + settings.read_text().replace("case_name = case", f"case_name = {case_name}") + ) + events: list[tuple[str, dict | None]] = [] + + def published_output(case_dir: Path, _root: Path) -> bool: + if case_dir == failing_case: + raise OSError("output unavailable") + return True + + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._published_output", + published_output, + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) + + candidates = _discover(tmp_path, BASE_URL) + + assert [candidate.path.parent for candidate in candidates] == [valid_case] + assert ( + "diagnostics_scanner_invalid_provenance", + { + "provenance_path": "production/type/failing/" + "provenance.20260811_120000_000000.cfg", + "reason": "output unavailable", + }, + ) in events + + +def test_retry_helper_retries_transient_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + responses = [httpx.Response(503), httpx.Response(204)] + events: list[tuple[str, dict | None]] = [] + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.time.sleep", lambda _: None + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) + response = _request_with_retry( + lambda *_args, **_kwargs: responses.pop(0), "https://x" + ) + assert response is not None + assert response.status_code == 204 + retry_fields = next( + fields + for event, fields in events + if event == "diagnostics_scanner_request_retry_scheduled" + ) + assert retry_fields == { + "attempt": 1, + "max_attempts": 3, + "status_code": 503, + "request_error": False, + } + + +class _Client: + def __init__(self, get_response: httpx.Response) -> None: + self.get_response = get_response + self.get_calls: list[dict] = [] + self.post_calls: list[dict] = [] + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def get(self, *_args, **kwargs): + self.get_calls.append(kwargs) + return self.get_response + + def post(self, *_args, **kwargs): + self.post_calls.append(kwargs) + return httpx.Response(204) + + +def test_run_submits_exact_payload_and_bearer_auth( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _case(tmp_path, "production/type/case") + client = _Client(httpx.Response(200, json=None)) + events: list[tuple[str, dict | None]] = [] + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", + lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.httpx.Client", + lambda **_kwargs: client, + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) + monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") + monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") + monkeypatch.setenv("MACHINE_NAME", "perlmutter") + monkeypatch.setenv("DRY_RUN", "false") + assert run() == 0 + assert client.get_calls[0]["params"] == { + "machine": "perlmutter", + "archive_relative_case_path": "production/type/case", + } + assert client.post_calls[0]["headers"] == {"Authorization": "Bearer token"} + assert client.post_calls[0]["json"]["diagnostics"][0]["name"] == "zppy diagnostics" + startup_fields = next( + fields + for event, fields in events + if event == "diagnostics_scanner_startup_configuration" + ) + assert startup_fields is not None + assert startup_fields["has_api_token"] is True + assert "token" not in startup_fields + assert ( + "diagnostics_scanner_link_submitted", + {"archive_relative_case_path": "production/type/case", "status_code": 204}, + ) in events + + +def test_dry_run_requires_no_api_configuration( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _case(tmp_path, "production/type/case") + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", + lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), + ) + monkeypatch.delenv("SIMBOARD_API_BASE_URL", raising=False) + monkeypatch.delenv("SIMBOARD_API_TOKEN", raising=False) + monkeypatch.setenv("MACHINE_NAME", "perlmutter") + monkeypatch.setenv("DRY_RUN", "true") + events: list[tuple[str, dict | None]] = [] + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) + assert run() == 0 + candidate_events = [ + fields + for event, fields in events + if event == "diagnostics_scanner_dry_run_candidate" + ] + assert len(candidate_events) == 1 + candidate_event = candidate_events[0] + assert candidate_event is not None + assert candidate_event["archive_relative_case_path"] == "production/type/case" + assert ( + candidate_event["settings_filename"] + == "provenance.20260811_120000_000000.settings" + ) + assert isinstance(candidate_event["fingerprint"], str) + assert events[-1] == ( + "diagnostics_scanner_completed", + { + "discovered_candidates": 1, + "dry_run_candidates": 1, + "unchanged_candidates": 0, + "deferred_state_lookups": 0, + "submitted_links": 0, + "failed_link_submissions": 0, + }, + ) + + +def test_run_defers_after_exhausted_state_lookup( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _case(tmp_path, "production/type/case") + client = _Client(httpx.Response(503)) + events: list[tuple[str, dict | None]] = [] + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.time.sleep", lambda _: None + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._resolve_archive", + lambda _machine: DiagnosticsArchive(str(tmp_path), BASE_URL), + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner.httpx.Client", + lambda **_kwargs: client, + ) + monkeypatch.setattr( + "app.scripts.ingestion.diagnostics_link_scanner._log_event", + lambda event, fields=None: events.append((event, fields)), + ) + monkeypatch.setenv("SIMBOARD_API_BASE_URL", "https://api.example.org") + monkeypatch.setenv("SIMBOARD_API_TOKEN", "token") + monkeypatch.setenv("MACHINE_NAME", "perlmutter") + monkeypatch.setenv("DRY_RUN", "false") + run() + assert client.post_calls == [] + assert ( + "diagnostics_scanner_request_retry_exhausted", + {"attempts": 3, "status_code": 503, "request_error": False}, + ) in events + assert ( + "diagnostics_scanner_state_lookup_deferred", + {"archive_relative_case_path": "production/type/case", "status_code": 503}, + ) in events diff --git a/docs/architecture/diagnostics-linkage.md b/docs/architecture/diagnostics-linkage.md new file mode 100644 index 00000000..27d4003c --- /dev/null +++ b/docs/architecture/diagnostics-linkage.md @@ -0,0 +1,32 @@ +# Diagnostics Linkage Architecture + +The diagnostics scanner is separate from performance-metadata collection. It discovers published zppy diagnostics and attaches a case-scoped diagnostic link to an already ingested SimBoard Case. + +## Terminology + +| Term | Definition | +| --- | --- | +| Diagnostics archive | A reviewed, machine-specific readable filesystem root and its corresponding public HTTP(S) base URL. It contains published diagnostics output; it is not a performance archive directory. | +| Diagnostics case | One published diagnostics case directory below a diagnostics archive. This filesystem identity is distinct from, and is resolved to, a SimBoard Case using the provenance case name, machine, and HPC username. | +| Provenance configuration | A timestamped `provenance.*.cfg` file in a diagnostics case directory. Its timestamp identifies which configuration is newest for discovery purposes. | +| Provenance settings | The non-symlink `.settings` file paired with the selected provenance configuration. It supplies the required case-resolution and diagnostics URL values and is the content used to produce the fingerprint. | +| Scanner candidate | A diagnostics case whose newest timestamped provenance configuration has valid paired settings, a published output, a diagnostics URL under the archive's public base URL, and a layout consistent with its settings. If that selected configuration or its settings are invalid or missing, the case is skipped; the scanner does not fall back to an older configuration. At most one candidate is discovered per diagnostics case directory in each archive tier. | +| Fingerprint | The SHA-256 digest of the selected provenance settings file bytes. It lets scanner state distinguish unchanged settings from changed settings without treating the provenance timestamp alone as sufficient. | +| Scanner state | The successful scanner submission record for a machine and archive-relative diagnostics case path. It records the selected settings filename, provenance timestamp, fingerprint, linked URL, submission time, and linked diagnostic link. | +| Linked candidate | A candidate for which scanner state has the same settings filename and fingerprint. It is already represented by a successful scanner submission and is not submitted again. | +| Unchanged candidate | A linked candidate: its selected settings filename and fingerprint match scanner state. “Unchanged” describes scanner submission state, not whether files in the diagnostics directory changed. | +| Deferred candidate | A candidate whose scanner-state lookup cannot complete successfully. It is left for a later scanner run rather than submitted without state. A candidate whose link submission fails is likewise not recorded as successfully linked and remains eligible later. | + +Invalid, unreadable, unsafe, or malformed provenance and settings inputs are skipped during discovery rather than becoming scanner candidates. The scanner only considers the `production` and `development` archive tiers, and selects the newest timestamped provenance configuration in each diagnostics case directory. + +## Scanner State Flow + +1. `MACHINE_NAME` is required and must name the machine whose diagnostics archive the scanner resolves. The scanner rejects unset or blank values before archive resolution, then verifies that the configured root is readable and its public base URL is absolute HTTP(S). +2. It discovers scanner candidates by selecting the newest timestamped provenance configuration for each diagnostics case directory, then validating its paired settings and computing the settings fingerprint. If the selected configuration or settings are invalid or missing, the case is skipped without falling back to an older configuration. +3. For every candidate, it calls `GET /api/v1/diagnostics/scanner-state` with the configured machine and archive-relative diagnostics case path. A missing state is an unlinked candidate; matching settings filename and fingerprint make it unchanged; a failed or non-successful lookup defers it. +4. For each unlinked or changed candidate, it calls `POST /api/v1/diagnostics/scanner/link` with one diagnostic link and provenance metadata. The API resolves the target SimBoard Case from case name, machine, and HPC username, then atomically upserts the case diagnostic link and scanner state. A successful request returns no content. +5. On a later run, the persisted state makes a matching candidate unchanged. A changed filename or fingerprint is submitted again and updates the state for that machine/path identity. + +Scanner API access requires the diagnostics-scanner role. A state lookup can return no state, and it returns an error when the supplied machine is unknown. The scanner-link endpoint requires exactly one diagnostic and rejects unsafe archive-relative paths; case-resolution failures also prevent a successful state update. + +With `DRY_RUN` enabled, the scanner performs archive resolution and candidate discovery, logs the diagnostics case paths it would link, and exits without reading scanner state or submitting links. It therefore creates or updates no diagnostic links or scanner state; every discovered candidate is reported as a proposed link rather than classified as linked, unchanged, or deferred. diff --git a/docs/architecture/metadata-ingestion.md b/docs/architecture/metadata-ingestion.md index 21c569d6..6a77b72b 100644 --- a/docs/architecture/metadata-ingestion.md +++ b/docs/architecture/metadata-ingestion.md @@ -34,6 +34,8 @@ that case by an execution ID derived from a CIME LID. | Staging directory | The active `PERF_ARCHIVE_DIR` tree where new performance output from E3SM runs appears before PACE moves it elsewhere. | | Archive directory | The long-term `OLD_PERF_ARCHIVE_DIR` tree managed by PACE after staging output is moved. | +Published diagnostics linking is documented in [Diagnostics Linkage](diagnostics-linkage.md). + ### Case and execution state terms Case-level state is derived from execution-level state. diff --git a/docs/developer/README.md b/docs/developer/README.md index d6946b0c..53104d2a 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -65,6 +65,8 @@ SimBoard supports local path ingestion from NERSC / Perlmutter and remote automa See [Metadata Ingestion Architecture](../architecture/metadata-ingestion.md) for terminology, ingestion modes, submission-state flow, runner configuration, site mapping, and PACE reference scripts. +See [Diagnostics Linkage Architecture](../architecture/diagnostics-linkage.md) for published diagnostics scanner terminology and state flow. + ## Local Environment Setup Prerequisites: diff --git a/docs/github-issues/174-zppy-links/phase-4-provenance-scanner.md b/docs/github-issues/174-zppy-links/phase-4-provenance-scanner.md index e62d50cb..b66c2cde 100644 --- a/docs/github-issues/174-zppy-links/phase-4-provenance-scanner.md +++ b/docs/github-issues/174-zppy-links/phase-4-provenance-scanner.md @@ -1,73 +1,174 @@ -# Phase 4 Plan: Provenance Scanner and Ops Docs +# Phase 4 Plan: Provenance Scanner and Operations ## Task -Add standalone scanner that discovers zppy provenance cfg files from configured NERSC roots, verifies completion markers, and calls internal diagnostics-link API with service-account auth. +Add a standalone scanner that discovers zppy provenance under each configured +site diagnostics archive, verifies that diagnostics have published output, and +links them to cases with service-account authentication. + +The scanner follows the zppy SimBoard publishing contract: diagnostics live +under `diagnostics_archive//`, optionally grouped by +`/`, and each case directory may contain multiple timestamped +provenance files. The newest timestamped provenance is authoritative. ## Scope ### In scope - New script `backend/app/scripts/ingestion/diagnostics_link_scanner.py` -- State-file persistence and retry behavior -- Provenance cfg parsing and completion checks -- Script test coverage -- Script documentation and env example updates +- Static diagnostics-archive site registry under `backend/app/scripts/ingestion/` +- Provenance discovery, settings parsing, validation, and published-output checks +- Database-backed provenance state, dry-run behavior, and transient-failure retries +- Scanner-specific diagnostics state API, schema, and migration +- Site-wrapper configuration, scanner tests, and operational documentation ### Out of scope -- zppy repo changes -- Historical backfill tooling beyond normal scanner behavior -- New backend endpoint behavior outside Phase 2 contract +- Mache runtime/config retrieval +- zppy publishing or provenance-contract changes +- Historical backfill beyond normal scanner operation +- Changes to the existing `POST /api/v1/diagnostics/link` contract or frontend +- Diagnostics content ingestion or interpretation ## Approach -1. Mirror existing operational script structure. - - Base new script on patterns from `backend/app/scripts/ingestion/nersc_upload_archive_ingestor.py`. - - Reuse same style for config parsing, structured logs, dry-run handling, retry/backoff, and state persistence. - -2. Discover provenance cfg files. - - Recursively search configured roots from required env var `ZPPY_PROVENANCE_ROOTS`. - - Accept files matching `provenance*.cfg`. - -3. Parse required fields from each cfg. - - Require `case_name`, `machine`, `hpc_username`, `diagnostic_url`, and `output`. - - Also extract `www` from the provenance cfg for preserved diagnostics provenance context. - - Do not derive `diagnostic_url` from `www` in MVP; continue treating explicit `diagnostic_url` as authoritative. - - Treat missing required fields as terminal skip with structured log. - -4. Verify diagnostics completion before linking. - - Require `/index.html` to exist. - - Require every filename listed in env var `DIAGNOSTICS_REQUIRED_STATUS_FILES` to exist under ``. - - Skip incomplete diagnostics without calling API. - -5. Call internal diagnostics-link API. - - Send bearer token from `SIMBOARD_API_TOKEN`. - - POST one diagnostics-link request per eligible cfg to `POST /api/v1/diagnostics/link`. - - Use one diagnostics item for MVP: `name="zppy diagnostics"`, `url=diagnostic_url`, `kind="diagnostic"`. - -6. Persist scanner state. - - Store state in `DIAGNOSTICS_STATE_PATH`. - - Key by provenance file path. - - Persist cfg fingerprint, last outcome, and timestamp. - - Reprocess only when cfg fingerprint changes. - -7. Document operational config. - - Update `backend/app/scripts/README.md` with purpose, env vars, and example invocation. - - Add placeholders to `.envs/example/backend.env.example` only for operator-provided values required by this script. +1. Resolve archive locations from a static internal site registry. + - Add a module under `backend/app/scripts/ingestion/` containing a + `DIAGNOSTICS_ARCHIVES_BY_MACHINE` dictionary. Each entry contains the + complete diagnostics archive root and matching public archive base URL. + - Seed the checked-in dictionary by parsing Mache's + `mache/machines/*.cfg` files during development, retaining machines with + non-empty `[web_portal] base_path` and `base_url`, then appending the + `diagnostics_archive` path component. This is a deliberate development or + maintenance operation, never a scanner runtime dependency. + - Key the registry by SimBoard's accepted machine names and aliases, not + solely Mache cfg filenames. Map machine aliases with the same published + archive to one registry entry. + - Select the registry entry from `MACHINE_NAME`. Fail before scanning for an + unsupported machine; do not accept archive locations from environment + variables or fetch Mache configuration at runtime. + - Validate that the selected filesystem root is absolute and readable and + that the public base URL uses HTTP or HTTPS before scanning. + - Treat registry values as deliberate source-controlled site configuration: + refresh the dictionary through a reviewed SimBoard change when a site moves + its published archive. + +2. Discover case provenance within one bounded archive. + - Scan only the configured root's `production/` and `development/` + subdirectories. + - Support both `//` and + `///` layouts. + - Traverse only inside the configured archive root; do not follow symlinks + outside it. Use a normalized archive-relative case-directory path for + scanner-state identity. + - Find only `provenance..cfg` files with a valid zppy timestamp + in case directories; order candidates by that parsed timestamp, not file + modification time. + - Select the newest timestamp for each case and require a matching + `provenance..settings` file. + - If the newest pair is incomplete, defer that case instead of falling back + to older provenance. + +3. Parse and validate the selected provenance pair. + - Read `case_name`, `machine`, `hpc_username`, optional `case_group`, and + authoritative `diagnostics_url` only from the settings file. The cfg + exists only to establish a matching timestamped provenance pair. + - Parse settings as bounded UTF-8 `key = value` lines without evaluation; + reject malformed input and duplicate required keys. + - Require all case-identity fields needed by `POST /api/v1/diagnostics/link`. + - Verify the provenance case and optional case group agree with the archive + layout. + - Parse `diagnostics_url` and require an exact scheme, authority, and path + boundary under the configured `DIAGNOSTICS_ARCHIVE_BASE_URL`; never + derive or accept an unrelated URL. + - Log and skip malformed or unsafe provenance without terminating the full + scan. + +4. Verify published diagnostic output before linking. + - Require the published case directory to contain at least one + non-provenance diagnostic artifact or a non-empty diagnostic subdirectory. + - Do not inspect zppy status files: they are not published archive artifacts + and are not a reliable completion signal for one timestamped provenance + pair. + - Treat published-output presence as a readiness check, not proof that every + zppy task has completed. + +5. Read and update scanner state through a scanner-specific diagnostics API. + - Read the database-backed state for the configured machine before submitting + a candidate. Compare the selected settings filename and fingerprint with + the state for its archive-relative case-directory path. + - Skip a candidate whose selected settings filename and fingerprint already + match successful state. + - Build the scanner endpoint from `SIMBOARD_API_BASE_URL`. Keep the existing + `POST /api/v1/diagnostics/link` contract unchanged; add a separate internal + scanner endpoint that accepts the diagnostics-link payload plus provenance + source metadata. + - Authenticate with bearer token from `SIMBOARD_API_TOKEN`. + - Submit the provenance identity, one diagnostics item with + `name="zppy diagnostics"`, the authoritative `diagnostics_url`, and + `kind="diagnostic"`, plus the archive-relative case path, selected settings + filename, timestamp, and fingerprint. + - Treat HTTP 204 as success. + - Retry network failures, HTTP 408/429, and 5xx responses with bounded + backoff. Do not retry permanent 4xx responses within the same run. + - The scanner endpoint must atomically upsert the case-scoped link and its + successful provenance state. Leave failed and output-not-ready candidates + without successful state so a later scan retries them. + +6. Persist central successful provenance state. + - Add a `DiagnosticProvenanceState` record for each scanner-managed + diagnostic link. Key the record by canonical machine and normalized + archive-relative case-directory path, which includes the simulation type + and optional case group. + - Store the selected settings filename, parsed timestamp, content fingerprint, + linked URL, and successful submission timestamp. + - Link state to its scanner-managed `ExternalLink` with a unique foreign key + using `ON DELETE CASCADE`. Deleting that link removes or invalidates its + state; a later scan can recreate the still-published link. + - Use a database uniqueness constraint and one transaction for the link + upsert and state upsert so concurrent scanners are safe. + - Development and production directory paths may create distinct diagnostic + links for the same SimBoard case. The scanner never removes obsolete links; + operators remove them manually. + - In dry-run mode, state reads are allowed, but make no link or state writes. + +7. Document and expose site operation. + - Update `backend/app/scripts/README.md` with configuration, dry-run rollout, + retry behavior, database-state handling, and example scheduled invocation. + - Add scanner execution to supported site wrappers without moving scanning + logic into shell. + - Document required shared-archive permissions, including scanner read access + to provenance settings. + - Explain how maintainers add or refresh a machine entry in the static site + registry from Mache cfg data. ## Tests -- Add `backend/tests/features/ingestion/test_diagnostics_link_scanner.py` covering: - - provenance discovery - - cfg parsing success and failure - - `www` extraction when present - - missing required identity or URL - - completion-marker checks - - dry-run behavior - - retry behavior for transient API failures - - state dedup and retry-on-fingerprint-change - - API payload formatting +- Add `backend/tests/features/ingestion/test_diagnostics_link_scanner.py` + covering: + - static site-registry selection by canonical machine name and alias + - rejection of unsupported machines + - valid and invalid registry filesystem roots and public URLs + - registry generation from representative Mache cfg files, including skipped + files with missing `[web_portal]` values + - production and development discovery + - grouped and ungrouped case layouts + - newest-timestamp selection + - missing newest settings file without stale fallback + - settings-only identity and URL parsing, including malformed and duplicate + required settings keys + - case-directory and case-group mismatch rejection + - diagnostics URL scheme, authority, and path-boundary validation + - published diagnostic output, empty output, and provenance-only directories + - exact scanner API payload and bearer authentication + - transient retries and permanent response handling + - database-state lookup, successful-state deduplication, and changed-settings + reprocessing + - atomic link-and-state persistence, concurrent submissions, and cascade state + removal when a scanner-managed link is deleted + - retry after output-not-ready or failed submissions + - dry-run behavior with no link or state writes + - Run: - `make backend-test` - `make pre-commit-run` @@ -76,8 +177,15 @@ Add standalone scanner that discovers zppy provenance cfg files from configured - Risk score: 5 - Main failure modes: - - Completion-marker policy is too strict or too loose. - - State logic suppresses needed retries or replays unchanged cfgs. + - Static registry becomes stale after a site moves its published archive + location. + - zppy provenance settings format changes before its publishing contract is + finalized. + - Published output appears before every zppy task finishes; this MVP links + published diagnostics rather than proving complete zppy execution. + - Archive permissions prevent the scanner from reading provenance settings. + - State identity or transaction logic suppresses needed retries or records a + link without matching successful provenance state. ## Open Questions diff --git a/docs/github-issues/174-zppy-links/plan.md b/docs/github-issues/174-zppy-links/plan.md index 0c034abf..d45eae06 100644 --- a/docs/github-issues/174-zppy-links/plan.md +++ b/docs/github-issues/174-zppy-links/plan.md @@ -4,7 +4,7 @@ Replace manual diagnostics URL entry with automated linking from zppy diagnostics outputs to existing SimBoard simulation records. -MVP is NERSC-only. +MVP uses SimBoard's static diagnostics-archive registry for supported machines. ## Scope @@ -12,12 +12,12 @@ MVP is NERSC-only. - Add required zppy provenance fields: `case_name`, `machine`, `hpc_username` - Add required diagnostics URLs in zppy provenance -- Require standardized zppy diagnostics output locations for NERSC production runs -- Discover zppy diagnostics provenance files from configured NERSC production filesystem roots -- Confirm diagnostics completion from index page plus status files +- Require standardized zppy diagnostics archive locations for supported machines +- Discover newest paired zppy provenance from the static archive registry +- Require published diagnostic output before linking - Match diagnostics to SimBoard records using `(case_name, machine, hpc_username)` - Create idempotent case-scoped diagnostic links -- Maintain scanner state to avoid repeated processing +- Maintain database-backed scanner provenance state ### Out @@ -27,7 +27,7 @@ MVP is NERSC-only. - Diagnostics content ingestion or indexing - Public HTML directory scraping - Historical backfill beyond configured provenance roots -- Non-NERSC deployments +- Mache runtime/config retrieval ## Core Decisions @@ -45,9 +45,12 @@ All three fields are required. `case_name` alone is not globally safe, and `CASE Avoid public directory scraping. It is fragile, web-server-coupled, slow, and expands the SSRF/content-injection attack surface. -### Use zppy provenance cfg as the primary input +### Use paired zppy provenance files -SimBoard discovers zppy provenance files from configured NERSC filesystem roots. Newer zppy runs already emit provenance cfg files under diagnostics output paths, for example: +SimBoard discovers timestamped paired provenance files from the filesystem roots +selected by its static site registry. The cfg establishes that a matching +provenance pair exists; the paired settings file is the authoritative source of +case identity and diagnostics URL. For example: ```text post/scripts/provenance.20260303_230804_991619.cfg @@ -58,7 +61,7 @@ Reference example: - https://github.com/E3SM-Project/zppy/blob/main/examples/post.v3.LR.historical.zppy_v3.cfg - https://web.lcrc.anl.gov/public/e3sm/diagnostic_output/zppy_example/v3.2.0/v3.LR.historical_0051/provenance.20260303_230804_991619.cfg -Current cfg examples expose useful fields: +Current cfg examples expose useful contextual fields: - `case`: case name - `input`: case run directory @@ -66,7 +69,7 @@ Current cfg examples expose useful fields: - `www`: public diagnostics root - `campaign`: optional campaign metadata -But current cfg is not yet an authoritative join source because it may lack: +The cfg is not an authoritative join source because it may lack: - `machine` - canonical simulation owner @@ -79,7 +82,8 @@ input path owner: ac.wlin output path owner: ac.zhang40 ``` -Therefore, zppy must enrich provenance cfg with required case identity copied from `/case_scripts/env_case.xml`: +Therefore, zppy must write required case identity to provenance settings, copied +from `/case_scripts/env_case.xml`: | XML field | Provenance field | | ---------- | ---------------- | @@ -87,39 +91,57 @@ Therefore, zppy must enrich provenance cfg with required case identity copied fr | `MACH` | `machine` | | `REALUSER` | `hpc_username` | -If any required field is missing, SimBoard skips the provenance file and logs it as invalid for linking. +If any required field is missing, SimBoard skips the provenance pair and logs it +as invalid for linking. Settings also provide the explicit `diagnostics_url`. -For MVP, zppy should reuse existing top-level cfg fields rather than emit a new versioned normalized block. +SimBoard selects the newest complete cfg/settings pair by parsed filename +timestamp and does not fall back to older provenance when the newest pair is +incomplete. -### Require standardized output locations for production runs +### Require standardized archive locations -For MVP, NERSC production runs must use standardized zppy diagnostics output locations. SimBoard relies on those known production roots for provenance discovery. +SimBoard keeps a checked-in `DIAGNOSTICS_ARCHIVES_BY_MACHINE` registry under +`backend/app/scripts/ingestion/`. Each entry supplies the complete filesystem +archive root and matching public archive URL for a supported SimBoard machine. +The registry is seeded or refreshed during development from Mache machine cfg +files, but the scanner never fetches or parses Mache at runtime. -Custom or ad hoc layouts do not block the overall design, but they are not the required path for MVP. +Custom or ad hoc layouts are not part of this MVP. ### Require explicit diagnostics URLs in provenance -For MVP, SimBoard should not derive diagnostics URLs from path conventions. zppy should emit explicit diagnostics URLs in provenance cfg. +For MVP, SimBoard should not derive diagnostics URLs from path conventions. zppy +should emit an explicit `diagnostics_url` in provenance settings. SimBoard +validates that URL against the configured public archive prefix. -### Use index page plus status files as completion signal +### Use published output as readiness signal -Treat diagnostics as complete only when the expected index page and zppy status files are present. +Treat a candidate as ready when its published case directory contains a +non-provenance diagnostic artifact or non-empty diagnostic subdirectory. Do not +inspect zppy status files: they are not published archive artifacts and do not +reliably identify one provenance run. This is a published-output readiness +check, not proof every zppy task has completed. ### Persist links, do not resolve at query time Create database rows when diagnostics are discovered. Frontend queries should not crawl filesystems or remote URLs. -Diagnostic links are case-scoped. For MVP, store them on `Case` by adding `case_id` to `ExternalLink`. Keep the existing manual-link rendering path where possible by surfacing case-scoped diagnostic links alongside current links. +Diagnostic links are case-scoped. Store scanner state centrally with each +scanner-managed link, keyed by canonical machine and archive-relative case path. +State is removed when its linked `ExternalLink` is deleted, allowing a later +scan to recreate a still-published link. ## Implementation -Implement in order: provenance contract -> scanner -> storage target -> resolver/API -> frontend verification. +Implement in order: provenance contract -> storage/API state -> scanner -> +frontend verification. ### zppy #### 1. Emit required provenance fields -For MVP, production runs must write diagnostics outputs and provenance cfg files to the standardized NERSC zppy output locations. +For MVP, runs must write diagnostics outputs and paired provenance +cfg/settings files to the standardized site diagnostics archive. | Field | Source | | -------------- | ------------------------- | @@ -135,9 +157,9 @@ Implementation note: Tests: -- uses standardized NERSC production output locations +- uses standardized diagnostics archive locations - emits `case_name`, `machine`, `hpc_username` -- emits explicit diagnostics URLs (`diagnostic_url`) +- emits explicit diagnostics URLs (`diagnostics_url`) - can construct explicit diagnostics URLs from cfg `www` plus `mache` machine metadata - parses values from `env_case.xml` - parses values from `env_build.xml` @@ -146,69 +168,95 @@ Tests: ### SimBoard -#### 1. Add diagnostics scanner +#### 1. Add diagnostics scanner and static site registry Add `diagnostics_link_scanner.py`. Responsibilities: -- scan configured NERSC production diagnostics roots for `provenance*.cfg` -- dedup with state file -- verify diagnostics completion from index page plus status files -- parse `case_name`, `machine`, `hpc_username` -- parse explicit diagnostics URLs (`diagnostic_url`) -- call internal API with service-account auth -- skip and log if full join key is unavailable +- add a checked-in machine-to-archive registry under `scripts/ingestion`, seeded + or refreshed from Mache cfg files during development only +- select a registry entry by accepted SimBoard machine name or alias; reject an + unsupported machine before scanning +- scan bounded production and development archive trees, including optional + case-group directories, without following symlinks outside the archive root +- select newest paired cfg/settings provenance by parsed filename timestamp +- parse identity and `diagnostics_url` only from settings; validate settings + syntax, archive layout, and URL scheme/authority/path boundary +- require published diagnostic output beyond provenance files; do not inspect + zppy status files +- read database-backed provenance state and skip an unchanged successful + settings filename/fingerprint +- call the scanner-specific internal diagnostics endpoint with service-account + auth, leaving `POST /api/v1/diagnostics/link` unchanged +- skip and log malformed, output-not-ready, or non-matching candidates Tests: -- discovers cfgs -- parses required cfg identity -- handles malformed cfgs +- selects static archive registry entries by canonical machine name and alias +- rejects unsupported machines and invalid registry roots or public URLs +- generates registry candidates from representative Mache cfg files while + skipping files without usable `[web_portal]` values +- discovers grouped and ungrouped cases in both archive classifications +- selects the newest paired provenance without stale fallback +- parses required settings identity and URL without parsing cfg identity +- handles malformed or unsafe provenance - skips missing identity -- checks index-plus-status completion marker -- dedups state +- checks published output and provenance-only directories without status files +- retries transient submissions and dedups central successful state - handles duplicate links idempotently -#### 2. Resolve link storage +#### 2. Resolve link storage and scanner state -Add `DiagnosticsLinkRequest` in `backend/app/features/simulation/schemas.py`. +Use the existing diagnostics-link request schema in +`backend/app/features/catalog/schemas.py`. -For MVP, add `case_id` to `ExternalLink` and store diagnostic links at case scope. -Add a partial unique index on `(case_id, kind, url)` where `case_id IS NOT NULL` so case-owned diagnostic links remain idempotent under repeated or concurrent writes. +Use the existing case-owned `ExternalLink` storage and partial uniqueness on +`(case_id, kind, url)` so repeated or concurrent submissions remain idempotent. + +Add `DiagnosticProvenanceState` for each scanner-managed diagnostic link. Store +the canonical machine, normalized archive-relative case path, settings filename, +timestamp, fingerprint, URL, and successful submission time. Give the state row +a unique foreign key to `ExternalLink` with cascade deletion, and add a unique +machine/path constraint. #### 3. Add matching resolver | Input | Match | | -------------- | ----------------------- | | `case_name` | `Case.name` | -| `machine` | joined case simulations | -| `hpc_username` | joined case simulations | +| `machine` | resolved `Case.machine_id` | +| `hpc_username` | `Case.hpc_username` | Outcomes: - 1 case match: create/update case-scoped links - 0 matches: `404` -- multiple matches: `409` +- case uniqueness makes multiple matches invalid Tests: - matching triple creates links - same case/machine under different user does not cross-link - no match returns `404` -- ambiguous match returns `409` +- case uniqueness prevents ambiguous matches -#### 4. Add internal API endpoint +#### 4. Add internal diagnostics APIs Endpoint: `POST /api/v1/diagnostics/link` Implementation note: -- Define the endpoint in `backend/app/features/simulation/api.py` using a dedicated `diagnostics_router` with prefix `/diagnostics`. +- Define the endpoint in `backend/app/features/catalog/api.py` using a dedicated `diagnostics_router` with prefix `/diagnostics`. - Register that router in `backend/app/main.py` with `API_BASE` so the public path remains exactly `/api/v1/diagnostics/link` instead of inheriting the `/simulations` prefix. Roles: `ADMIN`, `SERVICE_ACCOUNT` +Keep this endpoint's contract unchanged. Add scanner-specific state read and +link endpoints for service accounts. The scanner link endpoint accepts existing +diagnostics-link identity plus provenance metadata and atomically upserts the +case link and `DiagnosticProvenanceState` in one transaction. + Request: | Field | Required | @@ -232,6 +280,10 @@ Tests: - concurrent duplicate request is idempotent - invalid payload returns `422` - auth required +- scanner state lookup skips unchanged successful provenance +- scanner link submission atomically persists link and state +- concurrent scanner submissions remain safe +- deleting a scanner-managed link cascades state deletion #### 5. Keep frontend unchanged @@ -255,13 +307,12 @@ make backend-test && make pre-commit-run Mitigation: add `case_id` for MVP and keep migration/API behavior narrow. - **Missing identity**: SimBoard cannot link a provenance file without `case_name`, `machine`, and `hpc_username`. Mitigation: require zppy provenance enrichment; skip and log invalid files. -- **NERSC deployment variability**: zppy roots and public URL prefixes may still vary by campaign or user layout within NERSC. - Mitigation: use env-configured NERSC scanner roots and NERSC public-prefix mappings. +- **Static registry drift**: a site may move its published archive. + Mitigation: refresh the checked-in registry through a reviewed SimBoard change. - **Provenance drift**: cfg layout and required-field coverage may vary across zppy versions. Mitigation: add parser tests, schema/version detection, and a documented support window. ## Remaining Open Questions -1. **NERSC deployment scope:** Which NERSC scanner roots and public URL prefixes are supported in MVP? -2. **Retroactive linking:** Does MVP include historical backfill, or only provenance files with the required join key? -3. **Case identity hardening:** Is `(case_name, machine, hpc_username)` sufficient until issue #136 is resolved? +1. **Retroactive linking:** Does MVP include historical backfill, or only provenance files with the required join key? +2. **Case identity hardening:** Is `(case_name, machine, hpc_username)` sufficient until issue #136 is resolved?