From 426301340e2515c7f21bb7fe2d46d1536834ccd1 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Wed, 4 Feb 2026 10:53:30 -0500 Subject: [PATCH 01/10] Add unimplemented functions for archive extraction --- alchemiscale/interface/api.py | 29 +++++++++++++++++++++++++++++ alchemiscale/interface/client.py | 19 +++++++++++++++++++ alchemiscale/storage/statestore.py | 16 ++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index 341069e9..0a32e75e 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -258,6 +258,35 @@ def get_network_transformations( return [str(sk) for sk in n4js.get_network_transformations(network=sk)] +@router.get("/bulk/networks/archive") +def get_networks_archives( + *, + networks: list[str] = Body(embed=True), + n4js: Neo4jStore = Depends(get_n4js_depends), + token: TokenData = Depends(get_token_data_depends), +): + + network_sks = [ScopedKey.from_str(sk) for sk in networks] + + # check for repeated entries + for idx in range(len(networks_sks) - 1): + if networks_sks[idx] in networks[idx + 1 :]: + msg = "Provided network ScopedKey list contains repeats" + raise HTTPException( + status_code=http_status.HTTP_400_BAD_REQUEST, detail=msg + ) + + for sk in network_sks: + validate_scopes(sk.scope, token) + + archives = [] + for sk in network_sks: + archive = n4js.get_network_archive(sk) + archives.append(archive) + + return archives + + @router.get("/transformations/{transformation_scoped_key}/networks") def get_transformation_networks( transformation_scoped_key, diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index 9a321a39..c5602ac0 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -13,6 +13,7 @@ from async_lru import alru_cache import networkx as nx from gufe import AlchemicalNetwork, Transformation, ChemicalSystem +from gufe.archival import AlchemicalArchive from gufe.tokenization import GufeTokenizable, KeyedChain from gufe.protocols import ProtocolResult, ProtocolDAGResult import zstandard as zstd @@ -1748,6 +1749,24 @@ def get_network_failures( network=network, ok=False, compress=compress, visualize=visualize ) + def get_network_archives( + self, networks: list[ScopedKey] + ) -> list[AlchemicalArchive | None]: + """Get the archives for the given ``AlchemicalNetwork`` objects. + + Parameters + ---------- + networks + A list of ``AlchemicalNetwork`` ``ScopedKey`` values. The list must contain unique values. + + Returns + ------- + A list of ``AlchemicalArchive`` instances matching the order + of ``networks``. If a network was not found, ``None`` is + returned in its place. + """ + raise NotImplementedError + def get_transformation_results( self, transformation: ScopedKey, diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 343bdcde..78dd90b0 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -23,6 +23,7 @@ NonTransformation, Protocol, ) +from gufe.archival import AlchemicalArchive from gufe.settings import SettingsBaseModel from gufe.tokenization import GufeTokenizable, GufeKey, JSON_HANDLER, KeyedChain from gufe.protocols import ProtocolUnitFailure @@ -1206,6 +1207,21 @@ def get_transformation_failures(self, transformation: ScopedKey) -> list[ScopedK """ return self._get_protocoldagresultrefs(q, transformation) + def get_network_archive(self, network: ScopedKey) -> AlchemicalArchive | None: + """Get an ``AlchemicalArchive`` for a given ``AlchemicalNetwork`` + + Parameters + ---------- + network + ScopedKey of the AlchemicalNetwork. + + Returns + ------- + The AlchemicalArchive for the AlchemicalNetwork. If the + network count not be found, None is returned. + """ + raise NotImplementedError + ## compute def set_network_strategy( From 0060e91969f0867b9236f0b44dfa0f5da7fa33cc Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Mon, 9 Feb 2026 17:47:17 -0500 Subject: [PATCH 02/10] Move archive construction to API layer --- alchemiscale/interface/api.py | 48 ++++++++++++++++++++++++++++++ alchemiscale/interface/client.py | 16 ++++++++++ alchemiscale/storage/statestore.py | 16 ---------- 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index 0a32e75e..916d996a 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -27,6 +27,7 @@ _check_store_connectivity, GzipRoute, ) +from ..compression import decompress_gufe_zstd, json_to_gufe from ..settings import get_api_settings from ..settings import get_base_api_settings from ..storage.statestore import Neo4jStore @@ -258,6 +259,53 @@ def get_network_transformations( return [str(sk) for sk in n4js.get_network_transformations(network=sk)] +def _get_network_archive(n4js: Neo4JStore, network: ScopedKey): + try: + an = n4js.get_gufe(network) + except KeyError: + return None + + transformation_sks = n4js.get_network_transformations(network) + + transformation_results = [] + for transformation_sk in transformation_sks: + pdrrs_sks = n4js.get_transformation_results(transformation_sk) + if not pdrs: + continue + transformation = n4js.get_gufe(transformation_sk) + transform_results.append([transformation, []]) + for pdrr_sk in pdrr_sks: + try: + pdrr = n4js.get_gufe(pdrr_sk) + except KeyError: + raise HTTPException( + status_code=http_status.HTTP_400_BAD_REQUEST, + detail=str(e), + ) + pdr_sk = ScopedKey( + gufe_key=protocoldagresultref.obj_key, **sk.scope.to_dict() + ) + try: + pdr_bytes: str = s3os.pull_protocoldagresult( + pdr_sk, transformation_sk, ok=ok + ) + except Exception: + pdr_bytes: str = s3os.pull_protocoldagresult( + location=protocoldagresultref.location, + ok=True, + ) + try: + pdr = decompress_gufe_zstd(pdr_bytes) + except zstd.ZstdError: + pdr = json_to_gufe(pdr_bytes.decode("utf-8")) + transformation_results[-1][1].append(pdr) + + archive = AlchemicalArchive( + network=an, transform_results=transform_results, metadata={} + ) + return archive + + @router.get("/bulk/networks/archive") def get_networks_archives( *, diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index c5602ac0..f0f3c829 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -1767,6 +1767,22 @@ def get_network_archives( """ raise NotImplementedError + def get_network_archive(self, network: ScopedKey) -> AlchemicalArchive | None: + """Get the archive for a given ``AlchemicalNetwork``. + + Parameters + ---------- + network + The ``AlchemicalNetwork`` to archive. + + Returns + ------- + An ``AlchemicalArchive`` for the provided + ``AlchemicalNetwork``. If the ntwork was not found, ``None`` + is returned. + """ + return self.get_network_archives([network])[0] + def get_transformation_results( self, transformation: ScopedKey, diff --git a/alchemiscale/storage/statestore.py b/alchemiscale/storage/statestore.py index 78dd90b0..343bdcde 100644 --- a/alchemiscale/storage/statestore.py +++ b/alchemiscale/storage/statestore.py @@ -23,7 +23,6 @@ NonTransformation, Protocol, ) -from gufe.archival import AlchemicalArchive from gufe.settings import SettingsBaseModel from gufe.tokenization import GufeTokenizable, GufeKey, JSON_HANDLER, KeyedChain from gufe.protocols import ProtocolUnitFailure @@ -1207,21 +1206,6 @@ def get_transformation_failures(self, transformation: ScopedKey) -> list[ScopedK """ return self._get_protocoldagresultrefs(q, transformation) - def get_network_archive(self, network: ScopedKey) -> AlchemicalArchive | None: - """Get an ``AlchemicalArchive`` for a given ``AlchemicalNetwork`` - - Parameters - ---------- - network - ScopedKey of the AlchemicalNetwork. - - Returns - ------- - The AlchemicalArchive for the AlchemicalNetwork. If the - network count not be found, None is returned. - """ - raise NotImplementedError - ## compute def set_network_strategy( From e44caaa0acff964789396572e2633714b78cf8e4 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Tue, 17 Feb 2026 11:34:21 -0500 Subject: [PATCH 03/10] Allow user to specify metadata --- alchemiscale/interface/client.py | 40 +++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index f0f3c829..daa48783 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -14,7 +14,7 @@ import networkx as nx from gufe import AlchemicalNetwork, Transformation, ChemicalSystem from gufe.archival import AlchemicalArchive -from gufe.tokenization import GufeTokenizable, KeyedChain +from gufe.tokenization import GufeTokenizable, KeyedChain, JSON_HANDLER from gufe.protocols import ProtocolResult, ProtocolDAGResult import zstandard as zstd @@ -1750,7 +1750,7 @@ def get_network_failures( ) def get_network_archives( - self, networks: list[ScopedKey] + self, networks: list[ScopedKey], metadata: list[dict | None] = None ) -> list[AlchemicalArchive | None]: """Get the archives for the given ``AlchemicalNetwork`` objects. @@ -1758,14 +1758,48 @@ def get_network_archives( ---------- networks A list of ``AlchemicalNetwork`` ``ScopedKey`` values. The list must contain unique values. + metadata + Metadata to attach to the ``AlchemicalArchive`` + objects. This must be a list of dictionaries that are + compatible with ``GufeTokenizable`` serialization, in the + order of the provided ``AlchemicalNetwork`` ``ScopedKey`` + values. A ``None`` entry in the list will attach no + metadata to the corresponding ``AlchemicalArchive``. A + ``None`` in place of the list is interpretted as a list of + ``None``, which is the default. Returns ------- A list of ``AlchemicalArchive`` instances matching the order of ``networks``. If a network was not found, ``None`` is returned in its place. + + Raises + ------ + A ``ValueError`` is raised if the provided metadata is not serializable or the lenghts of the metadata and networks lists are not the same. + """ - raise NotImplementedError + + metadata = metadata or [None] * len(networks) + + if len(metadata) != len(networks): + raise ValueError("metadata and networks list must be the same length") + + for network, meta in zip(networks, metadata): + if meta: + try: + _ = json.dumps(meta, cls=JSON_HANDLER) + except: + raise ValueError(f"Unable to serialize metadata for {network}") + + # TODO make proper request + raw_archives = get_archives(networks) + + archives = [] + for archive, meta in zip(raw_archives, metadata): + archives.append(archive.copy_with_replacements(metadata=meta) if metadata else archive) + + return archives def get_network_archive(self, network: ScopedKey) -> AlchemicalArchive | None: """Get the archive for a given ``AlchemicalNetwork``. From 41b7d5801cd90d0445c876f6f1ad91fcc06d9caa Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Tue, 17 Feb 2026 11:58:11 -0500 Subject: [PATCH 04/10] Validate metadata types --- alchemiscale/interface/client.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index daa48783..f2e73718 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -1801,21 +1801,40 @@ def get_network_archives( return archives - def get_network_archive(self, network: ScopedKey) -> AlchemicalArchive | None: + def get_network_archive(self, network: ScopedKey, metadata: dict | None = None) -> AlchemicalArchive | None: """Get the archive for a given ``AlchemicalNetwork``. Parameters ---------- network The ``AlchemicalNetwork`` to archive. + metadata + Metadata to attach to the ``AlchemicalArchive``. This must + be a dictionary that is compatible with + ``GufeTokenizable`` serialization. Returns ------- An ``AlchemicalArchive`` for the provided ``AlchemicalNetwork``. If the ntwork was not found, ``None`` is returned. + + Raises + ------ + A ``ValueError`` is raised if the provided metadata is not + serializable. """ - return self.get_network_archives([network])[0] + match metadata: + case {}: + metadata = None + case dict(): + metadata = [metadata] + case None: + pass + case _: + raise ValueError("metadata must be a dictionary or None") + + return self.get_network_archives([network], metadata)[0] def get_transformation_results( self, From 3ef956f3545fa3467040ad501bfa1cd3e0ec7506 Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Thu, 19 Feb 2026 11:08:36 -0500 Subject: [PATCH 05/10] Extract PDR from bytes to function --- alchemiscale/interface/api.py | 6 ++---- alchemiscale/interface/client.py | 8 ++------ alchemiscale/strategist/service.py | 8 ++------ alchemiscale/utils.py | 13 +++++++++++++ 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index 916d996a..bc044b63 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -35,6 +35,7 @@ from ..storage.models import TaskStatusEnum, StrategyState from ..models import Scope, ScopedKey from ..security.models import TokenData, CredentialedUserIdentity +from ..utils import pdr_from_bytes app = FastAPI(title="AlchemiscaleAPI") app.dependency_overrides[get_base_api_settings] = get_api_settings @@ -294,10 +295,7 @@ def _get_network_archive(n4js: Neo4JStore, network: ScopedKey): location=protocoldagresultref.location, ok=True, ) - try: - pdr = decompress_gufe_zstd(pdr_bytes) - except zstd.ZstdError: - pdr = json_to_gufe(pdr_bytes.decode("utf-8")) + pdr = pdr_from_bytes(pdr_bytes) transformation_results[-1][1].append(pdr) archive = AlchemicalArchive( diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index f2e73718..927e198c 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -32,6 +32,7 @@ StrategyState, ) from stratocaster.base import Strategy +from ..utils import pdr_from_bytes from ..validators import validate_network_nonself from warnings import warn @@ -1536,12 +1537,7 @@ async def _async_get_protocoldagresult( pdr_bytes, ) - try: - # Attempt to decompress the ProtocolDAGResult object - pdr = decompress_gufe_zstd(pdr_bytes) - except zstd.ZstdError: - # If decompress fails, assume it's a UTF-8 encoded JSON string - pdr = json_to_gufe(pdr_bytes.decode("utf-8")) + pdr = pdr_from_bytes(pdr_bytes) return pdr diff --git a/alchemiscale/strategist/service.py b/alchemiscale/strategist/service.py index 84a4dc4c..cf05c378 100644 --- a/alchemiscale/strategist/service.py +++ b/alchemiscale/strategist/service.py @@ -36,6 +36,7 @@ from ..settings import Neo4jStoreSettings, S3ObjectStoreSettings from ..compression import compress_keyed_chain_zstd, decompress_gufe_zstd, json_to_gufe from .settings import StrategistSettings +from ..utils import pdr_from_bytes def execute_strategy_worker( @@ -171,12 +172,7 @@ def _get_protocoldagresult_cached( location=result_ref.location, ok=result_ref.ok ) - # Decompress the raw bytes to get ProtocolDAGResult - try: - pdr = decompress_gufe_zstd(pdr_bytes) - except zstd.ZstdError: - # Fallback to JSON deserialization for uncompressed data - pdr = json_to_gufe(pdr_bytes.decode("utf-8")) + pdr = pdr_from_bytes(pdr_bytes) if self._cache_enabled: try: diff --git a/alchemiscale/utils.py b/alchemiscale/utils.py index 37cc0b51..1598479e 100644 --- a/alchemiscale/utils.py +++ b/alchemiscale/utils.py @@ -1,3 +1,4 @@ +from gufe.protocols import ProtocolDAGResult from gufe.tokenization import ( GufeTokenizable, TOKENIZABLE_REGISTRY, @@ -8,6 +9,8 @@ from itertools import chain import networkx as nx +from .compression import decompress_gufe_zstd, json_to_gufe + class RegistryBackup: def __init__(self, gufe_object=None, keep_changes=False): @@ -112,3 +115,13 @@ def add_edges(o): _ = modify_dependencies(sd, add_edges, is_gufe_obj, mode="encode") return graph + + +def pdr_from_bytes(buffer: bytes) -> ProtocolDAGResult: + try: + # Attempt to decompress the ProtocolDAGresult object + pdr = decompress_gufe_zstd(pdr_bytes) + except zstd.ZstdError: + # If decompress fails, assume it's a UTF-8 encoded JSON string + pdr = json_to_gufe(pdr_bytes.decode("utf-8")) + return pdr From 92721a723377c547f77c679cb15e22497a80083b Mon Sep 17 00:00:00 2001 From: Ian Kenney Date: Thu, 19 Feb 2026 11:09:45 -0500 Subject: [PATCH 06/10] Update requests for archives --- alchemiscale/interface/api.py | 2 +- alchemiscale/interface/client.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index bc044b63..25c78f7c 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -328,7 +328,7 @@ def get_networks_archives( archives = [] for sk in network_sks: archive = n4js.get_network_archive(sk) - archives.append(archive) + archives.append(archive.to_json()) return archives diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index 927e198c..15252d6a 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -1746,7 +1746,7 @@ def get_network_failures( ) def get_network_archives( - self, networks: list[ScopedKey], metadata: list[dict | None] = None + self, networks: list[ScopedKey], metadata: list[dict | None] = None ) -> list[AlchemicalArchive | None]: """Get the archives for the given ``AlchemicalNetwork`` objects. @@ -1788,16 +1788,21 @@ def get_network_archives( except: raise ValueError(f"Unable to serialize metadata for {network}") - # TODO make proper request - raw_archives = get_archives(networks) + data = {"networks": list(map(str, networks))} + raw_archives = self._post_resource(f"/bulk/networks/archive", data) archives = [] for archive, meta in zip(raw_archives, metadata): - archives.append(archive.copy_with_replacements(metadata=meta) if metadata else archive) + _archive = AlchemicalArchive.from_json(archive) + archives.append( + _archive.copy_with_replacements(metadata=meta) if metadata else _archive + ) return archives - def get_network_archive(self, network: ScopedKey, metadata: dict | None = None) -> AlchemicalArchive | None: + def get_network_archive( + self, network: ScopedKey, metadata: dict | None = None + ) -> AlchemicalArchive | None: """Get the archive for a given ``AlchemicalNetwork``. Parameters From 180e6d8da8d03674362880c646a4694c2c60e1f8 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Sun, 5 Jul 2026 15:07:13 -0600 Subject: [PATCH 07/10] Implement client-side AlchemicalNetwork archival extracts Add AlchemiscaleClient.get_network_archive / get_network_archives, which bundle an AlchemicalNetwork with all successful ProtocolDAGResults for its Transformations into a gufe AlchemicalArchive, with optional user-supplied metadata. Not-found networks yield None. Implemented client-side atop the existing get_network and get_transformation_results machinery, replacing the abandoned server-side /bulk/networks/archive endpoint (which referenced undefined names and a non-existent statestore method). Also fixes a missing `json` import in client.py and restores pdr_from_bytes in utils.py (zstd import + decode fallback). Closes #246 Co-Authored-By: Claude Opus 4.8 --- alchemiscale/interface/api.py | 75 -------- alchemiscale/interface/client.py | 171 ++++++++++++------ .../interface/client/test_client.py | 96 ++++++++++ alchemiscale/utils.py | 5 +- docs/user_guide/getting_started.rst | 26 +++ news/issue-246.rst | 23 +++ 6 files changed, 267 insertions(+), 129 deletions(-) create mode 100644 news/issue-246.rst diff --git a/alchemiscale/interface/api.py b/alchemiscale/interface/api.py index 25c78f7c..341069e9 100644 --- a/alchemiscale/interface/api.py +++ b/alchemiscale/interface/api.py @@ -27,7 +27,6 @@ _check_store_connectivity, GzipRoute, ) -from ..compression import decompress_gufe_zstd, json_to_gufe from ..settings import get_api_settings from ..settings import get_base_api_settings from ..storage.statestore import Neo4jStore @@ -35,7 +34,6 @@ from ..storage.models import TaskStatusEnum, StrategyState from ..models import Scope, ScopedKey from ..security.models import TokenData, CredentialedUserIdentity -from ..utils import pdr_from_bytes app = FastAPI(title="AlchemiscaleAPI") app.dependency_overrides[get_base_api_settings] = get_api_settings @@ -260,79 +258,6 @@ def get_network_transformations( return [str(sk) for sk in n4js.get_network_transformations(network=sk)] -def _get_network_archive(n4js: Neo4JStore, network: ScopedKey): - try: - an = n4js.get_gufe(network) - except KeyError: - return None - - transformation_sks = n4js.get_network_transformations(network) - - transformation_results = [] - for transformation_sk in transformation_sks: - pdrrs_sks = n4js.get_transformation_results(transformation_sk) - if not pdrs: - continue - transformation = n4js.get_gufe(transformation_sk) - transform_results.append([transformation, []]) - for pdrr_sk in pdrr_sks: - try: - pdrr = n4js.get_gufe(pdrr_sk) - except KeyError: - raise HTTPException( - status_code=http_status.HTTP_400_BAD_REQUEST, - detail=str(e), - ) - pdr_sk = ScopedKey( - gufe_key=protocoldagresultref.obj_key, **sk.scope.to_dict() - ) - try: - pdr_bytes: str = s3os.pull_protocoldagresult( - pdr_sk, transformation_sk, ok=ok - ) - except Exception: - pdr_bytes: str = s3os.pull_protocoldagresult( - location=protocoldagresultref.location, - ok=True, - ) - pdr = pdr_from_bytes(pdr_bytes) - transformation_results[-1][1].append(pdr) - - archive = AlchemicalArchive( - network=an, transform_results=transform_results, metadata={} - ) - return archive - - -@router.get("/bulk/networks/archive") -def get_networks_archives( - *, - networks: list[str] = Body(embed=True), - n4js: Neo4jStore = Depends(get_n4js_depends), - token: TokenData = Depends(get_token_data_depends), -): - - network_sks = [ScopedKey.from_str(sk) for sk in networks] - - # check for repeated entries - for idx in range(len(networks_sks) - 1): - if networks_sks[idx] in networks[idx + 1 :]: - msg = "Provided network ScopedKey list contains repeats" - raise HTTPException( - status_code=http_status.HTTP_400_BAD_REQUEST, detail=msg - ) - - for sk in network_sks: - validate_scopes(sk.scope, token) - - archives = [] - for sk in network_sks: - archive = n4js.get_network_archive(sk) - archives.append(archive.to_json()) - - return archives - - @router.get("/transformations/{transformation_scoped_key}/networks") def get_transformation_networks( transformation_scoped_key, diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index b418f424..c9182f77 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -7,6 +7,7 @@ from __future__ import annotations import asyncio +import json from enum import StrEnum from typing import Any, Literal from collections.abc import Iterable @@ -1765,96 +1766,162 @@ def get_network_failures( ) def get_network_archives( - self, networks: list[ScopedKey], metadata: list[dict | None] = None + self, + networks: list[ScopedKey | str], + metadata: list[dict | None] | None = None, + compress: bool = True, + visualize: bool = True, ) -> list[AlchemicalArchive | None]: - """Get the archives for the given ``AlchemicalNetwork`` objects. + r"""Produce archival-quality extracts for the given ``AlchemicalNetwork``\s. + + Each returned ``AlchemicalArchive`` bundles an + ``AlchemicalNetwork`` together with all successful + ``ProtocolDAGResult``\s currently available for its + ``Transformation``\s, in a form suitable for long-term storage, + sharing, and downstream analysis. Parameters ---------- networks - A list of ``AlchemicalNetwork`` ``ScopedKey`` values. The list must contain unique values. + A list of ``AlchemicalNetwork`` ``ScopedKey`` values. The + list must not contain duplicate entries. metadata - Metadata to attach to the ``AlchemicalArchive`` + Metadata to attach to the produced ``AlchemicalArchive`` objects. This must be a list of dictionaries that are compatible with ``GufeTokenizable`` serialization, in the - order of the provided ``AlchemicalNetwork`` ``ScopedKey`` - values. A ``None`` entry in the list will attach no - metadata to the corresponding ``AlchemicalArchive``. A - ``None`` in place of the list is interpretted as a list of - ``None``, which is the default. + same order as ``networks``. A ``None`` entry in the list + attaches no metadata to the corresponding + ``AlchemicalArchive``. Passing ``None`` in place of the + list is interpreted as a list of ``None``, which is the + default. + compress + If ``True``, compress objects server-side before shipping + them to the client. This is a performance optimization; it + has no bearing on the result of this method call. + visualize + If ``True``, show retrieval progress indicators. Returns ------- - A list of ``AlchemicalArchive`` instances matching the order - of ``networks``. If a network was not found, ``None`` is - returned in its place. + A list of ``AlchemicalArchive`` instances matching the order of + ``networks``. If a network was not found, ``None`` is returned + in its place. Raises ------ - A ``ValueError`` is raised if the provided metadata is not serializable or the lenghts of the metadata and networks lists are not the same. + ValueError + If the provided metadata is not serializable, if the + lengths of the ``metadata`` and ``networks`` lists differ, + or if ``networks`` contains duplicate entries. """ + network_sks = [ + ScopedKey.from_str(network) if isinstance(network, str) else network + for network in networks + ] - metadata = metadata or [None] * len(networks) + if len(set(network_sks)) != len(network_sks): + raise ValueError("`networks` list must not contain duplicate entries") - if len(metadata) != len(networks): - raise ValueError("metadata and networks list must be the same length") + metadata = [None] * len(network_sks) if metadata is None else metadata - for network, meta in zip(networks, metadata): - if meta: - try: - _ = json.dumps(meta, cls=JSON_HANDLER) - except: - raise ValueError(f"Unable to serialize metadata for {network}") + if len(metadata) != len(network_sks): + raise ValueError("`metadata` and `networks` lists must be the same length") - data = {"networks": list(map(str, networks))} - raw_archives = self._post_resource(f"/bulk/networks/archive", data) + # validate that all metadata is serializable up-front, before + # performing any (potentially expensive) retrieval + for network_sk, meta in zip(network_sks, metadata): + if meta is None: + continue + try: + json.dumps(meta, cls=JSON_HANDLER.encoder) + except TypeError as e: + raise ValueError( + f"Unable to serialize metadata for '{network_sk}': {e}" + ) - archives = [] - for archive, meta in zip(raw_archives, metadata): - _archive = AlchemicalArchive.from_json(archive) - archives.append( - _archive.copy_with_replacements(metadata=meta) if metadata else _archive + return [ + self._get_network_archive( + network_sk, meta, compress=compress, visualize=visualize ) - - return archives + for network_sk, meta in zip(network_sks, metadata) + ] def get_network_archive( - self, network: ScopedKey, metadata: dict | None = None + self, + network: ScopedKey | str, + metadata: dict | None = None, + compress: bool = True, + visualize: bool = True, ) -> AlchemicalArchive | None: - """Get the archive for a given ``AlchemicalNetwork``. + r"""Produce an archival-quality extract for a given ``AlchemicalNetwork``. + + The returned ``AlchemicalArchive`` bundles the + ``AlchemicalNetwork`` together with all successful + ``ProtocolDAGResult``\s currently available for its + ``Transformation``\s, in a form suitable for long-term storage, + sharing, and downstream analysis. Parameters ---------- network - The ``AlchemicalNetwork`` to archive. + The ``ScopedKey`` of the ``AlchemicalNetwork`` to archive. metadata - Metadata to attach to the ``AlchemicalArchive``. This must - be a dictionary that is compatible with + Metadata to attach to the produced ``AlchemicalArchive``. + This must be a dictionary that is compatible with ``GufeTokenizable`` serialization. + compress + If ``True``, compress objects server-side before shipping + them to the client. This is a performance optimization; it + has no bearing on the result of this method call. + visualize + If ``True``, show retrieval progress indicators. Returns ------- - An ``AlchemicalArchive`` for the provided - ``AlchemicalNetwork``. If the ntwork was not found, ``None`` - is returned. + An ``AlchemicalArchive`` for the provided ``AlchemicalNetwork``. + If the network was not found, ``None`` is returned. Raises ------ - A ``ValueError`` is raised if the provided metadata is not - serializable. + ValueError + If the provided metadata is not serializable. + """ - match metadata: - case {}: - metadata = None - case dict(): - metadata = [metadata] - case None: - pass - case _: - raise ValueError("metadata must be a dictionary or None") - - return self.get_network_archives([network], metadata)[0] + return self.get_network_archives( + [network], + metadata=None if metadata is None else [metadata], + compress=compress, + visualize=visualize, + )[0] + + def _get_network_archive( + self, + network: ScopedKey, + metadata: dict | None, + compress: bool = True, + visualize: bool = True, + ) -> AlchemicalArchive | None: + # returns None if the network does not exist in the given Scope + if not self.check_exists(network): + return None + + an = self.get_network(network, compress=compress, visualize=visualize) + + transformation_results = [] + for transformation in an.edges: + transformation_sk = ScopedKey( + gufe_key=transformation.key, **network.scope.to_dict() + ) + pdrs = self.get_transformation_results( + transformation_sk, + return_as=ResultFormat.PROTOCOL_DAG_RESULTS, + compress=compress, + visualize=visualize, + ) + transformation_results.append((transformation, pdrs)) + + return AlchemicalArchive(an, transformation_results, metadata=metadata) def get_transformation_results( self, diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 390acb71..23f6197f 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -7,6 +7,7 @@ import json from gufe import AlchemicalNetwork +from gufe.archival import AlchemicalArchive from gufe.tokenization import TOKENIZABLE_REGISTRY, GufeKey, JSON_HANDLER from gufe.protocols import ProtocolResult from gufe.protocols.protocoldag import execute_DAG @@ -2633,6 +2634,101 @@ def test_get_task_failures( # TODO: can we mix in a success in here somewhere? # not possible with current BrokenProtocol, unfortunately + def test_get_network_archives( + self, + scope_test, + n4js_preloaded, + s3os_server_fresh, + user_client_no_cache: client.AlchemiscaleClient, + network_tyk2, + tmpdir, + ): + user_client = user_client_no_cache + n4js = n4js_preloaded + s3os_server = s3os_server_fresh + + an = network_tyk2 + transformation = list(t for t in an.edges if "_solvent" in t.name)[0] + + network_sk = user_client.get_scoped_key(an, scope_test) + transformation_sk = user_client.get_scoped_key(transformation, scope_test) + + # create, action, execute, and push results for three tasks on a + # single transformation + all_tasks = user_client.create_tasks(transformation_sk, count=3) + actioned_tasks = user_client.action_tasks(all_tasks, network_sk) + + with tmpdir.as_cwd(): + protocoldagresults = self._execute_tasks(actioned_tasks, n4js, s3os_server) + self._push_results(actioned_tasks, protocoldagresults, n4js, s3os_server) + + # clear local gufe registry of pdr objects so we exercise + # deserialization of the objects pulled back from the server + for pdr in protocoldagresults: + TOKENIZABLE_REGISTRY.pop(pdr.key, None) + + # produce a single-network archive with metadata attached + metadata = {"project": "tyk2", "note": "archival extract"} + archive = user_client.get_network_archive( + network_sk, metadata=metadata, visualize=False + ) + + assert isinstance(archive, AlchemicalArchive) + assert archive.network == an + assert archive.metadata == metadata + + # every Transformation in the network should be represented, but only + # the one we computed results for should carry ProtocolDAGResults + results_by_key = {tf.key: pdrs for tf, pdrs in archive.transformation_results} + assert set(results_by_key) == {tf.key for tf in an.edges} + + for tf in an.edges: + if tf.key == transformation.key: + assert set(results_by_key[tf.key]) == set(protocoldagresults) + for pdr in results_by_key[tf.key]: + assert pdr.ok() + assert pdr.transformation_key == transformation.key + else: + assert results_by_key[tf.key] == [] + + # the archive should round-trip through its serialized form + roundtrip = AlchemicalArchive.from_json(content=archive.to_json()) + assert roundtrip == archive + assert roundtrip.metadata == metadata + + # a nonexistent network yields None, both in bulk and singly + fake_network_sk = ScopedKey( + gufe_key=GufeKey(f"AlchemicalNetwork-{'0' * 32}"), + **scope_test.to_dict(), + ) + + archives = user_client.get_network_archives( + [network_sk, fake_network_sk], visualize=False + ) + assert len(archives) == 2 + assert isinstance(archives[0], AlchemicalArchive) + assert archives[0].network == an + assert archives[1] is None + + assert user_client.get_network_archive(fake_network_sk, visualize=False) is None + + def test_get_network_archives_validation( + self, scope_test, n4js_preloaded, user_client, network_tyk2 + ): + network_sk = user_client.get_scoped_key(network_tyk2, scope_test) + + # duplicate networks are rejected + with pytest.raises(ValueError, match="duplicate"): + user_client.get_network_archives([network_sk, network_sk]) + + # metadata and networks lists must be the same length + with pytest.raises(ValueError, match="same length"): + user_client.get_network_archives([network_sk], metadata=[{}, {}]) + + # unserializable metadata is rejected before any retrieval + with pytest.raises(ValueError, match="serialize"): + user_client.get_network_archive(network_sk, metadata={"bad": object()}) + class TestTaskRestartPolicy: default_max_retries = 3 diff --git a/alchemiscale/utils.py b/alchemiscale/utils.py index 1598479e..45d7ebe4 100644 --- a/alchemiscale/utils.py +++ b/alchemiscale/utils.py @@ -8,6 +8,7 @@ ) from itertools import chain import networkx as nx +import zstandard as zstd from .compression import decompress_gufe_zstd, json_to_gufe @@ -120,8 +121,8 @@ def add_edges(o): def pdr_from_bytes(buffer: bytes) -> ProtocolDAGResult: try: # Attempt to decompress the ProtocolDAGresult object - pdr = decompress_gufe_zstd(pdr_bytes) + pdr = decompress_gufe_zstd(buffer) except zstd.ZstdError: # If decompress fails, assume it's a UTF-8 encoded JSON string - pdr = json_to_gufe(pdr_bytes.decode("utf-8")) + pdr = json_to_gufe(buffer.decode("utf-8")) return pdr diff --git a/docs/user_guide/getting_started.rst b/docs/user_guide/getting_started.rst index f4460887..6d630aae 100644 --- a/docs/user_guide/getting_started.rst +++ b/docs/user_guide/getting_started.rst @@ -457,3 +457,29 @@ If you wish to pull results for only a single :py:class:`~alchemiscale.storage.m [] You can then iteratively create and action new :py:class:`~alchemiscale.storage.models.Task`\s on your desired :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s based on their current estimate and uncertainty, allocating effort where it will be most beneficial. + + +**************************** +Producing archival extracts +**************************** + +``alchemiscale`` is an execution system, not an archival system. +When you want a durable, self-contained snapshot of an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` together with its existing results — for example to deposit on a `FAIR `_ data repository such as Zenodo, to share with collaborators, or to feed into downstream analysis tooling — you can produce an :external+gufe:py:class:`~gufe.archival.AlchemicalArchive` with:: + + >>> an_sk: ScopedKey + >>> archive = asc.get_network_archive(an_sk) + >>> archive + + +The returned :external+gufe:py:class:`~gufe.archival.AlchemicalArchive` bundles the :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` with all successful :external+gufe:py:class:`~gufe.protocols.protocoldag.ProtocolDAGResult`\s currently available for its :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s. +If the network is not found in the given :py:class:`~alchemiscale.models.Scope`, ``None`` is returned. + +You can attach arbitrary, serializable metadata to the archive:: + + >>> archive = asc.get_network_archive(an_sk, metadata={"doi": "10.5281/zenodo.1234567"}) + +To produce archives for several networks at once, use :py:meth:`~alchemiscale.interface.client.AlchemiscaleClient.get_network_archives`, optionally passing a list of per-network metadata dictionaries in the same order. + +The archive is a :external+gufe:py:class:`~gufe.tokenization.GufeTokenizable`, so it can be written to and read back from disk with its :external+gufe:py:meth:`~gufe.tokenization.GufeTokenizable.to_json` and :external+gufe:py:meth:`~gufe.tokenization.GufeTokenizable.from_json` methods:: + + >>> archive.to_json(file="my_network_archive.json") diff --git a/news/issue-246.rst b/news/issue-246.rst new file mode 100644 index 00000000..86eab94e --- /dev/null +++ b/news/issue-246.rst @@ -0,0 +1,23 @@ +**Added:** + +* ``AlchemiscaleClient.get_network_archive`` and ``AlchemiscaleClient.get_network_archives`` produce archival-quality ``AlchemicalArchive`` extracts, bundling an ``AlchemicalNetwork`` with all successful ``ProtocolDAGResult``\s for its ``Transformation``\s, with optional user-supplied metadata. Networks that are not found yield ``None``. + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* + +**Security:** + +* From 005f3aac7374742b245f9f0df22cb12ff3100657 Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 6 Jul 2026 18:49:10 -0600 Subject: [PATCH 08/10] Join archive results via authoritative ScopedKeys; broaden tests Rework AlchemiscaleClient._get_network_archive to pair each Transformation with its results by fetching results in parallel via get_network_results and then retrieving each Transformation by its authoritative (server-side) ScopedKey. A Transformation deserialized now may not reproduce the GufeKey it had at ingestion (gufe tokenization can change across versions), so a reconstructed ScopedKey is not a reliable join and could silently drop results; the server-side ScopedKey is stable. Also broaden metadata-serialization validation to catch ValueError (e.g. circular references), and extend tests to cover string ScopedKey input, the compress=False path, and per-network metadata ordering in the bulk method. Document the GufeKey-instability gotcha in CLAUDE.md. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 1 + alchemiscale/interface/client.py | 30 ++++++++++++------- .../interface/client/test_client.py | 27 ++++++++++++++++- 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 832a26be..c9307441 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -75,6 +75,7 @@ The system has four main services communicating through Neo4j and S3: - **Frozen Pydantic models**: Core models use `frozen=True` for immutability. - **Module docstrings**: Follow the pattern `:mod:\`alchemiscale.module_name\` --- description`. - **GUFE integration**: Core chemistry types (`AlchemicalNetwork`, `Transformation`, `ChemicalSystem`, `Protocol`) come from the `gufe` library and are stored/retrieved via their tokenization system (`GufeKey`, `GufeTokenizable`). +- **`GufeKey` is not stable across `gufe` versions**: A `ScopedKey` is stable once created (it carries the *ingestion-time* `gufe_key`), but a `GufeTokenizable` deserialized later may recompute a *different* `GufeKey` if the `gufe` tokenization has changed since ingestion. Never derive a `ScopedKey` from a freshly-deserialized object's `.key` (e.g. `ScopedKey(gufe_key=obj.key, **scope.to_dict())`) to join against stored data — such a join can silently miss instead of erroring. Instead, obtain the authoritative `ScopedKey` from the server (e.g. `get_network_transformations`, or the keys of `get_network_results`) and fetch the object *by that `ScopedKey`* (e.g. `get_transformation(sk)`). Comparisons purely among objects deserialized in the same process (e.g. "is this transformation in `network.edges`?") are safe, since both sides use the current-stack key consistently. ## Testing diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index c9182f77..a1688031 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -1835,7 +1835,7 @@ def get_network_archives( continue try: json.dumps(meta, cls=JSON_HANDLER.encoder) - except TypeError as e: + except (TypeError, ValueError) as e: raise ValueError( f"Unable to serialize metadata for '{network_sk}': {e}" ) @@ -1908,16 +1908,26 @@ def _get_network_archive( an = self.get_network(network, compress=compress, visualize=visualize) + # retrieve all successful ProtocolDAGResults for the network's + # Transformations in parallel, keyed by their authoritative + # (server-side) Transformation ScopedKey + results = self.get_network_results( + network, + return_as=ResultFormat.PROTOCOL_DAG_RESULTS, + compress=compress, + visualize=visualize, + ) + + # pair each Transformation with its results by fetching the + # Transformation via its authoritative ScopedKey. We deliberately do + # not derive a ScopedKey from a deserialized Transformation's GufeKey: + # a `Transformation` deserialized now may not reproduce the GufeKey it + # had when ingested (gufe tokenization can change across versions), so + # its key is not a reliable join to the stored ScopedKey. transformation_results = [] - for transformation in an.edges: - transformation_sk = ScopedKey( - gufe_key=transformation.key, **network.scope.to_dict() - ) - pdrs = self.get_transformation_results( - transformation_sk, - return_as=ResultFormat.PROTOCOL_DAG_RESULTS, - compress=compress, - visualize=visualize, + for transformation_sk, pdrs in results.items(): + transformation = self.get_transformation( + transformation_sk, compress=compress, visualize=False ) transformation_results.append((transformation, pdrs)) diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 23f6197f..326d435e 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -2696,18 +2696,43 @@ def test_get_network_archives( assert roundtrip == archive assert roundtrip.metadata == metadata + # a string ScopedKey is accepted and produces an equivalent archive + archive_from_str = user_client.get_network_archive( + str(network_sk), visualize=False + ) + assert archive_from_str.network == an + str_results = { + tf.key: pdrs for tf, pdrs in archive_from_str.transformation_results + } + assert set(str_results[transformation.key]) == set(protocoldagresults) + + # the compress=False path produces an equivalent archive + archive_uncompressed = user_client.get_network_archive( + network_sk, compress=False, visualize=False + ) + assert archive_uncompressed.network == an + uncompressed_results = { + tf.key: pdrs for tf, pdrs in archive_uncompressed.transformation_results + } + assert set(uncompressed_results[transformation.key]) == set(protocoldagresults) + # a nonexistent network yields None, both in bulk and singly fake_network_sk = ScopedKey( gufe_key=GufeKey(f"AlchemicalNetwork-{'0' * 32}"), **scope_test.to_dict(), ) + # per-network metadata is attached in order, and a not-found network + # yields None in its slot archives = user_client.get_network_archives( - [network_sk, fake_network_sk], visualize=False + [network_sk, fake_network_sk], + metadata=[metadata, {"unused": "for missing network"}], + visualize=False, ) assert len(archives) == 2 assert isinstance(archives[0], AlchemicalArchive) assert archives[0].network == an + assert archives[0].metadata == metadata assert archives[1] is None assert user_client.get_network_archive(fake_network_sk, visualize=False) is None From 5ddcffb72d495e783c619839ace22b62f07e871c Mon Sep 17 00:00:00 2001 From: David Dotson Date: Mon, 6 Jul 2026 19:11:02 -0600 Subject: [PATCH 09/10] Fix docs build for AlchemicalArchive The docs build failed to import the package: client.py now imports gufe.archival, absent from the docs env's pinned gufe=1.3.0. Bump it to 1.10.0 to match the runtime pin. That bump surfaced a latent conf.py issue: gufe>=1.10.0 evaluates `PositiveFloat | None` at class-definition time in its settings models, which raises "unsupported operand type(s) for |" when pydantic is mocked via autodoc_mock_imports. Drop pydantic from the mock list (it is a real, installed gufe dependency); autodoc then imports gufe cleanly. Finally, render AlchemicalArchive as an inline literal in getting_started.rst rather than an intersphinx cross-reference: the gufe inventory is pinned to v1.2.0, which predates gufe.archival, so the reference cannot resolve. Verified: `sphinx -W` builds with zero warnings in both the docs.yml env and the full runtime env. Co-Authored-By: Claude Opus 4.8 --- devtools/conda-envs/docs.yml | 2 +- docs/conf.py | 1 - docs/user_guide/getting_started.rst | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/devtools/conda-envs/docs.yml b/devtools/conda-envs/docs.yml index f080a0af..86b7b004 100644 --- a/devtools/conda-envs/docs.yml +++ b/devtools/conda-envs/docs.yml @@ -10,6 +10,6 @@ dependencies: - myst-parser>=0.14 - docutils - sphinx-notfound-page - - gufe=1.3.0 + - gufe=1.10.0 - py2neo - stratocaster diff --git a/docs/conf.py b/docs/conf.py index 16eea6ca..00a3cdcb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,7 +42,6 @@ "httpx", "jose", "neo4j", - "pydantic", "pydantic_settings", "starlette", "yaml", diff --git a/docs/user_guide/getting_started.rst b/docs/user_guide/getting_started.rst index 6d630aae..2fa2313b 100644 --- a/docs/user_guide/getting_started.rst +++ b/docs/user_guide/getting_started.rst @@ -464,14 +464,14 @@ Producing archival extracts **************************** ``alchemiscale`` is an execution system, not an archival system. -When you want a durable, self-contained snapshot of an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` together with its existing results — for example to deposit on a `FAIR `_ data repository such as Zenodo, to share with collaborators, or to feed into downstream analysis tooling — you can produce an :external+gufe:py:class:`~gufe.archival.AlchemicalArchive` with:: +When you want a durable, self-contained snapshot of an :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` together with its existing results — for example to deposit on a `FAIR `_ data repository such as Zenodo, to share with collaborators, or to feed into downstream analysis tooling — you can produce an ``AlchemicalArchive`` with:: >>> an_sk: ScopedKey >>> archive = asc.get_network_archive(an_sk) >>> archive -The returned :external+gufe:py:class:`~gufe.archival.AlchemicalArchive` bundles the :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` with all successful :external+gufe:py:class:`~gufe.protocols.protocoldag.ProtocolDAGResult`\s currently available for its :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s. +The returned ``AlchemicalArchive`` bundles the :external+gufe:py:class:`~gufe.network.AlchemicalNetwork` with all successful :external+gufe:py:class:`~gufe.protocols.protocoldag.ProtocolDAGResult`\s currently available for its :external+gufe:py:class:`~gufe.transformations.transformation.Transformation`\s. If the network is not found in the given :py:class:`~alchemiscale.models.Scope`, ``None`` is returned. You can attach arbitrary, serializable metadata to the archive:: From 8be54a389934b8191c1d6d9cefadaf7d8d75d6ee Mon Sep 17 00:00:00 2001 From: Mike Henry <11765982+mikemhenry@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:30:10 -0700 Subject: [PATCH 10/10] Added more checks for metadata validation --- alchemiscale/interface/client.py | 15 +++++++++-- .../interface/client/test_client.py | 25 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/alchemiscale/interface/client.py b/alchemiscale/interface/client.py index a1688031..f4fb7e0d 100644 --- a/alchemiscale/interface/client.py +++ b/alchemiscale/interface/client.py @@ -10,7 +10,7 @@ import json from enum import StrEnum from typing import Any, Literal -from collections.abc import Iterable +from collections.abc import Iterable, Mapping, Sequence from itertools import chain from functools import lru_cache @@ -1823,7 +1823,12 @@ def get_network_archives( if len(set(network_sks)) != len(network_sks): raise ValueError("`networks` list must not contain duplicate entries") - metadata = [None] * len(network_sks) if metadata is None else metadata + if metadata is None: + metadata = [None] * len(network_sks) + elif isinstance(metadata, Mapping) or not isinstance(metadata, Sequence): + raise ValueError( + "`metadata` must be a list/sequence of dictionaries or None" + ) if len(metadata) != len(network_sks): raise ValueError("`metadata` and `networks` lists must be the same length") @@ -1833,6 +1838,12 @@ def get_network_archives( for network_sk, meta in zip(network_sks, metadata): if meta is None: continue + + if not isinstance(meta, Mapping): + raise ValueError( + f"Metadata for '{network_sk}' must be a dictionary/mapping or None" + ) + try: json.dumps(meta, cls=JSON_HANDLER.encoder) except (TypeError, ValueError) as e: diff --git a/alchemiscale/tests/integration/interface/client/test_client.py b/alchemiscale/tests/integration/interface/client/test_client.py index 326d435e..041deb15 100644 --- a/alchemiscale/tests/integration/interface/client/test_client.py +++ b/alchemiscale/tests/integration/interface/client/test_client.py @@ -2754,6 +2754,31 @@ def test_get_network_archives_validation( with pytest.raises(ValueError, match="serialize"): user_client.get_network_archive(network_sk, metadata={"bad": object()}) + # bulk metadata must be a sequence aligned with networks, not a single dict. + # This is a subtle case because a one-key dict has len == len(networks), + # so a length-only check would pass and iteration would yield the key string. + with pytest.raises(ValueError, match="Metadata.*list|metadata.*sequence"): + user_client.get_network_archives( + [network_sk], + metadata={"project": "tyk2"}, + ) + + # each per-network metadata entry must be a dict or None, even if the value + # is otherwise JSON-serializable + for bad_meta in ["project", ["project", "tyk2"], 1, 1.5, True]: + with pytest.raises(ValueError, match="Metadata.*dict|metadata.*mapping"): + user_client.get_network_archives( + [network_sk], + metadata=[bad_meta], + ) + + # None remains valid as a per-network metadata entry + user_client.get_network_archives( + [network_sk], + metadata=[None], + visualize=False, + ) + class TestTaskRestartPolicy: default_max_retries = 3