chore: sync OpenAPI contract from logion-private

.deps.lock.json

@@ -38,6 +38,7 @@
     "dependencies": [
       "cryptography>=50.0.0,<51.0.0",
       "logion-client",
+      "logion-runner",
       "logion-skillmap",
       "pydantic>=2.7,<3.0.0",
       "pyyaml>=6.0,<7.0",

packages/cli/cli/_parser.py

@@ -32,6 +32,7 @@
 from cli.commands import (
     credits as credits_mod,
 )
+from cli.commands import eval as eval_mod
 from cli.commands import (
     referrals as referrals_mod,
 )
@@ -58,6 +59,7 @@ def build_parser() -> argparse.ArgumentParser:
 
     health.register(subparsers)
     doctor.register(subparsers)
+    eval_mod.register(subparsers)
     identity.register(subparsers)
     indexed.register(subparsers)
     listings.register(subparsers)

packages/cli/cli/commands/eval/__init__.py

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: MIT
+"""Portable eval contract commands."""
+
+from cli.commands.eval.parser import register
+
+__all__ = ["register"]

packages/cli/cli/commands/eval/_bundle.py

@@ -0,0 +1,153 @@
+# SPDX-License-Identifier: MIT
+"""Handlers for portable eval creation and reproduction."""
+
+from __future__ import annotations
+
+import argparse
+import zipfile
+from pathlib import Path
+
+from logion_eval_contract import (
+    EvalContract,
+    EvalContractError,
+    contract_digest,
+    contract_to_json,
+    load_document,
+    pair_key,
+    parse_result_document,
+    result_digest,
+)
+
+from cli._json import JsonObject
+from cli._output import emit_json
+from cli._version import __version__ as cli_version
+from cli.commands.eval._scaffold import (
+    _BUNDLE_MEDIA_TYPE,
+    _digest,
+    _fail,
+    _json_bytes,
+    _load_contract,
+    _safe_member,
+)
+
+_HARNESS_ID = "logion-cli"
+_MODEL_ID = "reference-subject"
+_MODEL_VERSION = "1.0.0"
+_EXIT_INVALID = 2
+_EXIT_ERROR = 1
+_EXIT_REFUSED = 3
+
+
+def _bundle_manifest(
+    contract: EvalContract,
+    subject: bytes,
+    fixture_entries: dict[str, JsonObject],
+    result_entries: list[JsonObject],
+) -> JsonObject:
+    contract_bytes = _json_bytes(contract_to_json(contract))
+    return {
+        "contract": {
+            "contract_digest": contract_digest(contract),
+            "digest": _digest(contract_bytes),
+            "path": "contract.json",
+        },
+        "fixtures": fixture_entries,
+        "harness": {"id": _HARNESS_ID, "version": cli_version},
+        "media_type": _BUNDLE_MEDIA_TYPE,
+        "results": result_entries,
+        "schema_version": 1,
+        "subject": {"digest": _digest(subject), "path": "subject.bin"},
+    }
+
+
+def _checked_results(
+    paths: list[str], contract: EvalContract, subject: bytes
+) -> tuple[list[tuple[str, bytes]], list[JsonObject]]:
+    if len(paths) < 2:
+        raise ValueError("export requires at least two --result files")
+    files: list[tuple[str, bytes]] = []
+    entries: list[JsonObject] = []
+    parsed = []
+    for index, path in enumerate(paths, start=1):
+        document, _ = load_document(path)
+        result = parse_result_document(document)
+        if result.contract_digest != contract_digest(contract):
+            raise ValueError(f"result {path!r} belongs to another contract")
+        if result.subject_digest != _digest(subject):
+            raise ValueError(f"result {path!r} belongs to another subject")
+        member = f"results/result-{index}.json"
+        raw = _json_bytes(result.to_json())
+        files.append((member, raw))
+        entries.append({
+            "digest": _digest(raw),
+            "path": member,
+            "result_digest": result_digest(result),
+        })
+        parsed.append(result)
+    if any(pair_key(item) != pair_key(parsed[0]) for item in parsed[1:]):
+        raise ValueError("results belong to different execution environments")
+    if (
+        contract.determinism_class == "deterministic"
+        and len({result_digest(item) for item in parsed}) != 1
+    ):
+        raise ValueError("deterministic run results do not match")
+    return files, entries
+
+
+def _require_fixture_digest(raw: bytes, name: str, expected: str) -> None:
+    if _digest(raw) != expected:
+        raise ValueError(f"fixture {name!r} digest mismatch")
+
+
+def handle_eval_export(args: argparse.Namespace) -> int:
+    """Package the contract, fixtures, subject, and run evidence."""
+    output = Path(args.output)
+    try:
+        if output.exists() and not args.force:
+            raise FileExistsError(
+                f"{output} already exists; pass --force to replace it"
+            )
+        contract = _load_contract(args.contract)
+        subject = Path(args.subject).read_bytes()
+        result_files, result_entries = _checked_results(
+            args.result, contract, subject
+        )
+        base = Path(args.contract).resolve().parent
+        fixture_files: list[tuple[str, bytes]] = []
+        fixture_entries: dict[str, JsonObject] = {}
+        for fixture in contract.fixtures:
+            name = _safe_member(fixture.name)
+            raw = (base / fixture.name).read_bytes()
+            _require_fixture_digest(raw, fixture.name, fixture.digest)
+            member = f"fixtures/{name}"
+            fixture_files.append((member, raw))
+            fixture_entries[fixture.name] = {
+                "digest": fixture.digest,
+                "path": member,
+            }
+        manifest = _bundle_manifest(
+            contract, subject, fixture_entries, result_entries
+        )
+        output.parent.mkdir(parents=True, exist_ok=True)
+        with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED) as archive:
+            archive.writestr("manifest.json", _json_bytes(manifest))
+            archive.writestr(
+                "contract.json", _json_bytes(contract_to_json(contract))
+            )
+            archive.writestr("subject.bin", subject)
+            for member, raw in [*fixture_files, *result_files]:
+                archive.writestr(member, raw)
+    except EvalContractError as exc:
+        return _fail(exc.code, str(exc))
+    except (OSError, ValueError, zipfile.BadZipFile) as exc:
+        return _fail("eval_bundle_invalid", str(exc))
+    emit_json(
+        "logion.eval.export",
+        {
+            "bundle": str(output),
+            "bundle_digest": _digest(output.read_bytes()),
+            "contract_digest": contract_digest(contract),
+            "result_count": len(result_entries),
+        },
+    )
+    return 0

packages/cli/cli/commands/eval/_execution.py

@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: MIT
+"""Handlers for portable eval creation and reproduction."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from importlib.metadata import version
+from pathlib import Path
+
+import logion_eval_contract as eval_contract_package
+import yaml
+from logion_eval_contract import (
+    EvalContractError,
+    contract_digest,
+    parse_result_document,
+    result_digest,
+)
+from logion_runner.evals import (
+    EvalExecutionError,
+    resolve_eval_job,
+)
+
+from cli._output import emit_json
+from cli.commands.eval._scaffold import (
+    _digest,
+    _execute,
+    _fail,
+    _json_bytes,
+    _load_contract,
+    _scaffold_documents,
+    _write,
+)
+
+
+def _publish_result(args, contract, job, result):
+    """Route publishing through the public module for testable clients."""
+    from cli.commands.eval import handlers
+
+    return handlers._publish_result(args, contract, job, result)
+
+
+_BUNDLE_MEDIA_TYPE = "application/vnd.logion.eval-reproduction.v1+zip"
+_HARNESS_ID = "logion-cli"
+_MODEL_ID = "reference-subject"
+_MODEL_VERSION = "1.0.0"
+_EXIT_INVALID = 2
+_EXIT_ERROR = 1
+_EXIT_REFUSED = 3
+
+
+def handle_eval_scaffold(args: argparse.Namespace) -> int:
+    """Create a complete starter contract and subject."""
+    directory = Path(args.directory)
+    contract, subject = _scaffold_documents()
+    try:
+        _write(
+            directory / "eval-contract.yaml",
+            yaml.safe_dump(contract, sort_keys=False).encode(),
+            force=args.force,
+        )
+        _write(
+            directory / "subject.json",
+            _json_bytes(subject),
+            force=args.force,
+        )
+        parsed = _load_contract(str(directory / "eval-contract.yaml"))
+    except (EvalContractError, OSError) as exc:
+        return _fail("eval_scaffold_failed", str(exc))
+    emit_json(
+        "logion.eval.scaffold",
+        {
+            "contract": str(directory / "eval-contract.yaml"),
+            "contract_digest": contract_digest(parsed),
+            "subject": str(directory / "subject.json"),
+        },
+    )
+    return 0
+
+
+def handle_eval_validate(args: argparse.Namespace) -> int:
+    """Validate a contract and resolve all declared fixtures."""
+    try:
+        contract = _load_contract(args.contract)
+        base = Path(args.contract).resolve().parent
+        fixture_digests: dict[str, str] = {}
+        for fixture in contract.fixtures:
+            raw = (base / fixture.name).read_bytes()
+            actual = _digest(raw)
+            if actual != fixture.digest:
+                from logion_eval_contract import EvalFixtureDigestMismatch
+
+                raise EvalFixtureDigestMismatch(
+                    f"fixture {fixture.name!r} hashes to {actual},"
+                    f" not {fixture.digest}"
+                )
+            fixture_digests[fixture.name] = actual
+    except EvalContractError as exc:
+        return _fail(exc.code, str(exc))
+    except OSError as exc:
+        return _fail("eval_fixture_unreadable", str(exc))
+    emit_json(
+        "logion.eval.validate",
+        {
+            "contract_digest": contract_digest(contract),
+            "determinism_class": contract.determinism_class,
+            "fixture_digests": fixture_digests,
+            "valid": True,
+            "validator_import_root": (
+                "site-packages"
+                if "site-packages" in str(eval_contract_package.__file__)
+                else "source-tree"
+            ),
+            "validator_package_version": version("logion-eval-contract"),
+        },
+    )
+    return 0
+
+
+def handle_eval_run(args: argparse.Namespace) -> int:
+    """Resolve inputs, execute the subject, and emit a normalized result."""
+    try:
+        contract = _load_contract(args.contract)
+        subject = Path(args.subject).read_bytes()
+        job = resolve_eval_job(
+            contract,
+            subject,
+            contract_dir=Path(args.contract).resolve().parent,
+        )
+        outcome = _execute(contract, subject, args.contract)
+        published = (
+            _publish_result(args, contract, job, outcome.result_document)
+            if args.publish
+            else None
+        )
+        if args.output:
+            _write(
+                Path(args.output),
+                _json_bytes(outcome.result_document),
+                force=args.force,
+            )
+    except EvalContractError as exc:
+        return _fail(exc.code, str(exc))
+    except (OSError, FileExistsError, ValueError, json.JSONDecodeError) as exc:
+        return _fail("eval_input_unreadable", str(exc))
+    except EvalExecutionError as exc:
+        return _fail("eval_execution_failed", str(exc), _EXIT_ERROR)
+    emit_json(
+        "logion.eval.run",
+        {
+            "assertion_outcomes": outcome.assertion_outcomes,
+            "executed": True,
+            "output": args.output,
+            "resolved": {
+                "contract_digest": job.contract_digest,
+                "evaluator_digest": contract.evaluator_requirement.digest,
+                "image": job.sandbox_profile["image"],
+                "sandbox_profile_digest": job.sandbox_profile_digest,
+                "subject_digest": job.subject_digest,
+            },
+            "result": outcome.result_document,
+            "result_digest": result_digest(
+                parse_result_document(outcome.result_document)
+            ),
+            "published": published,
+        },
+    )
+    return 0