From 9b4fa8fef7828c88605c81402e8f76ce096a0bd7 Mon Sep 17 00:00:00 2001 From: simleo Date: Tue, 20 Jan 2026 16:59:19 +0100 Subject: [PATCH 01/32] add test_detached --- rocrate/model/metadata.py | 2 +- test/test_read.py | 65 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/rocrate/model/metadata.py b/rocrate/model/metadata.py index fdbef797..a1b8bfe9 100644 --- a/rocrate/model/metadata.py +++ b/rocrate/model/metadata.py @@ -94,7 +94,7 @@ def _has_writeable_stream(self): return True def write(self, dest_base): - write_path = Path(dest_base) / self.id + write_path = Path(dest_base) / self.id.rsplit("/", 1)[-1] super()._write_from_stream(write_path) @property diff --git a/test/test_read.py b/test/test_read.py index 1a8fd300..4194754d 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -668,6 +668,71 @@ def test_from_dict(tmpdir, version): ROCrate(metadata, init=True) +def test_detached(tmpdir): + base_uri = "http://example.com/" + metadata = { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": f"{base_uri}ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": base_uri}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} + }, + { + "@id": base_uri, + "@type": "Dataset", + "creator": {"@id": "#josiah"}, + "hasPart": {"@id": f"{base_uri}d1"} + }, + { + "@id": f"{base_uri}d1", + "@type": "Dataset", + "hasPart": {"@id": f"{base_uri}d1/d2"} + }, + { + "@id": f"{base_uri}d1/d2", + "@type": "Dataset", + "hasPart": {"@id": f"{base_uri}d1/d2/f1"} + }, + { + "@id": f"{base_uri}d1/d2/f1", + "@type": "File" + }, + { + "@id": "#josiah", + "@type": "Person", + 'name': 'Josiah Carberry' + }, + ] + } + crate = ROCrate(metadata) + d1 = crate.dereference(f"{base_uri}d1") + assert d1 + d2 = crate.dereference(f"{base_uri}d1/d2") + assert d2 + f1 = crate.dereference(f"{base_uri}d1/d2/f1") + assert f1 + p = crate.dereference("#josiah") + assert p + assert set(crate.data_entities) == {d1, d2, f1} + assert set(crate.contextual_entities) == {p} + out_path = tmpdir / 'out_crate' + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + crate = ROCrate(out_path) + d1 = crate.dereference(f"{base_uri}d1") + assert d1 + d2 = crate.dereference(f"{base_uri}d1/d2") + assert d2 + f1 = crate.dereference(f"{base_uri}d1/d2/f1") + assert f1 + p = crate.dereference("#josiah") + assert p + assert set(crate.data_entities) == {d1, d2, f1} + assert set(crate.contextual_entities) == {p} + + @pytest.mark.parametrize("version", ["1.1", "1.2"]) def test_no_data_entity_link_from_file(version): metadata = { From 815542b33ab4bdaca675cbaad839797c211c7cd2 Mon Sep 17 00:00:00 2001 From: simleo Date: Wed, 21 Jan 2026 09:18:15 +0100 Subject: [PATCH 02/32] test_detached: more checks --- test/test_read.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_read.py b/test/test_read.py index 4194754d..2c21107b 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -706,7 +706,10 @@ def test_detached(tmpdir): }, ] } + crate = ROCrate(metadata) + assert crate.root_dataset.id == base_uri + assert crate.metadata.id == f"{base_uri}ro-crate-metadata.json" d1 = crate.dereference(f"{base_uri}d1") assert d1 d2 = crate.dereference(f"{base_uri}d1/d2") @@ -720,7 +723,10 @@ def test_detached(tmpdir): out_path = tmpdir / 'out_crate' crate.write(out_path) assert (out_path / "ro-crate-metadata.json").is_file() + crate = ROCrate(out_path) + assert crate.root_dataset.id == base_uri + assert crate.metadata.id == f"{base_uri}ro-crate-metadata.json" d1 = crate.dereference(f"{base_uri}d1") assert d1 d2 = crate.dereference(f"{base_uri}d1/d2") From fc025ec027772048a7bf861e16f9951708fdde0f Mon Sep 17 00:00:00 2001 From: simleo Date: Wed, 21 Jan 2026 16:05:17 +0100 Subject: [PATCH 03/32] find RDE id as prescribed by the spec --- rocrate/metadata.py | 45 ++---------------------- test/test_metadata.py | 80 ++++--------------------------------------- test/test_read.py | 7 ++-- 3 files changed, 12 insertions(+), 120 deletions(-) diff --git a/rocrate/metadata.py b/rocrate/metadata.py index c32b4af9..1e1c27f4 100644 --- a/rocrate/metadata.py +++ b/rocrate/metadata.py @@ -21,7 +21,6 @@ # limitations under the License. import json -import warnings from .model.metadata import BASENAME, LEGACY_BASENAME @@ -68,48 +67,8 @@ def find_root_entity_id(entities): Return a tuple of the corresponding identifiers (descriptor, root). If the entities are not found, raise KeyError. If they are found, but they don't satisfy the required constraints, raise ValueError. - - In the general case, the metadata file descriptor id can be an - absolute URI whose last path segment is "ro-crate-metadata.json[ld]". - Since there can be more than one such id in the crate, we need to - choose among the corresponding (descriptor, root) entity pairs. First, we - exclude those that don't satisfy other constraints, such as the - descriptor entity being of type CreativeWork, etc.; if this doesn't - leave us with a single pair, we try to pick one with a - heuristic. Suppose we are left with the (m1, r1) and (m2, r2) pairs: - if r1 is the actual root of this crate, then m2 and r2 are regular - files in it, and as such they must appear in r1's hasPart; r2, - however, is not required to have a hasPart property listing other - files. Thus, we look for a pair whose root entity "contains" all - descriptor entities from other pairs. If there is no such pair, or there - is more than one, we just return an arbitrary pair. - """ descriptor = entities.get(BASENAME, entities.get(LEGACY_BASENAME)) - if descriptor: - return _check_descriptor(descriptor, entities) - candidates = [] - for id_, e in entities.items(): - basename = id_.rsplit("/", 1)[-1] - if basename == BASENAME or basename == LEGACY_BASENAME: - try: - candidates.append(_check_descriptor(e, entities)) - except ValueError: - pass - if not candidates: + if not descriptor: raise KeyError("Metadata file descriptor not found") - elif len(candidates) == 1: - return candidates[0] - else: - warnings.warn("Multiple metadata file descriptors, will pick one with a heuristic") - descriptor_ids = set(_[0] for _ in candidates) - for m_id, r_id in candidates: - try: - root = entities[r_id] - part_ids = set(_["@id"] for _ in root["hasPart"]) - except KeyError: - continue - if part_ids >= descriptor_ids - {m_id}: - # if True for more than one candidate, this pick is arbitrary - return m_id, r_id - return candidates[0] # fall back to arbitrary pick + return _check_descriptor(descriptor, entities) diff --git a/test/test_metadata.py b/test/test_metadata.py index 10477171..9219dbc2 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -26,16 +26,14 @@ from rocrate.metadata import find_root_entity_id -@pytest.mark.parametrize("root,basename", [ - ("", "ro-crate-metadata.json"), - ("", "ro-crate-metadata.jsonld"), +@pytest.mark.parametrize("root_id,metadata_id", [ + ("./", "ro-crate-metadata.json"), + ("./", "ro-crate-metadata.jsonld"), ("https://example.org/crate/", "ro-crate-metadata.json"), ("https://example.org/crate/", "ro-crate-metadata.jsonld"), - ("", "bad-name.json"), + ("./", "bad-name.json"), ]) -def test_find_root(root, basename): - metadata_id = root + basename - root_id = root or "./" +def test_find_root(root_id, metadata_id): entities = {_["@id"]: _ for _ in [ { "@id": metadata_id, @@ -51,7 +49,7 @@ def test_find_root(root, basename): "@type": "Dataset", }, ]} - if basename not in {"ro-crate-metadata.json", "ro-crate-metadata.jsonld"}: + if metadata_id not in {"ro-crate-metadata.json", "ro-crate-metadata.jsonld"}: with pytest.raises(KeyError): find_root_entity_id(entities) else: @@ -94,72 +92,6 @@ def test_find_root_bad_entities(): find_root_entity_id(entities) -@pytest.mark.filterwarnings("ignore") -def test_find_root_multiple_entries(): - orig_entities = { - "http://example.org/ro-crate-metadata.json": { - "@id": "http://example.org/ro-crate-metadata.json", - "@type": "CreativeWork", - "about": {"@id": "http://example.org/"}, - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"}, - }, - "http://example.org/": { - "@id": "http://example.org/", - "@type": "Dataset", - "hasPart": [ - {"@id": "http://example.com/"}, - {"@id": "http://example.com/ro-crate-metadata.json"} - ] - }, - "http://example.com/ro-crate-metadata.json": { - "@id": "http://example.com/ro-crate-metadata.json", - "@type": "CreativeWork", - "about": {"@id": "http://example.com/"}, - "conformsTo": {"@id": "https://w3id.com/ro/crate/1.2"}, - }, - "http://example.com/": { - "@id": "http://example.com/", - "@type": "Dataset", - }, - } - - def check_finds_org(entities): - m_id, r_id = find_root_entity_id(entities) - assert m_id == "http://example.org/ro-crate-metadata.json" - assert r_id == "http://example.org/" - - def check_picks_one(entities): - m_id, r_id = find_root_entity_id(entities) - assert m_id in [f"http://example.{_}/ro-crate-metadata.json" for _ in ("org", "com")] - assert r_id in [f"http://example.{_}/" for _ in ("org", "com")] - - check_finds_org(orig_entities) - # no root candidate contains the other one - mod_entities = deepcopy(orig_entities) - del mod_entities["http://example.org/"]["hasPart"] - check_picks_one(mod_entities) - # each root candidate contains the other one - mod_entities = deepcopy(orig_entities) - mod_entities["http://example.com/"]["hasPart"] = [ - {"@id": "http://example.org/"}, - {"@id": "http://example.org/ro-crate-metadata.json"} - ] - check_picks_one(mod_entities) - # "about" does not reference the root entity - mod_entities = deepcopy(orig_entities) - for about in "http://google.com", {"@id": "http://google.com"}: - mod_entities["http://example.com/ro-crate-metadata.json"]["about"] = about - check_finds_org(mod_entities) - # metadata type is not CreativeWork - mod_entities = deepcopy(orig_entities) - mod_entities["http://example.com/ro-crate-metadata.json"]["@type"] = "Thing" - check_finds_org(mod_entities) - # root type is not Dataset - mod_entities = deepcopy(orig_entities) - mod_entities["http://example.com/"]["@type"] = "Thing" - check_finds_org(mod_entities) - - def test_find_root_multiple_types(): entities = {_["@id"]: _ for _ in [ { diff --git a/test/test_read.py b/test/test_read.py index 2c21107b..d905efc5 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -674,7 +674,7 @@ def test_detached(tmpdir): "@context": "https://w3id.org/ro/crate/1.2/context", "@graph": [ { - "@id": f"{base_uri}ro-crate-metadata.json", + "@id": "ro-crate-metadata.json", "@type": "CreativeWork", "about": {"@id": base_uri}, "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} @@ -709,7 +709,7 @@ def test_detached(tmpdir): crate = ROCrate(metadata) assert crate.root_dataset.id == base_uri - assert crate.metadata.id == f"{base_uri}ro-crate-metadata.json" + assert crate.metadata.id == "ro-crate-metadata.json" d1 = crate.dereference(f"{base_uri}d1") assert d1 d2 = crate.dereference(f"{base_uri}d1/d2") @@ -720,13 +720,14 @@ def test_detached(tmpdir): assert p assert set(crate.data_entities) == {d1, d2, f1} assert set(crate.contextual_entities) == {p} + out_path = tmpdir / 'out_crate' crate.write(out_path) assert (out_path / "ro-crate-metadata.json").is_file() crate = ROCrate(out_path) assert crate.root_dataset.id == base_uri - assert crate.metadata.id == f"{base_uri}ro-crate-metadata.json" + assert crate.metadata.id == "ro-crate-metadata.json" d1 = crate.dereference(f"{base_uri}d1") assert d1 d2 = crate.dereference(f"{base_uri}d1/d2") From c2dc3f37b80828727057e5dd30b54acf52db4c7b Mon Sep 17 00:00:00 2001 From: simleo Date: Thu, 22 Jan 2026 16:49:37 +0100 Subject: [PATCH 04/32] support specifying RDE id when creating a new crate --- rocrate/rocrate.py | 10 ++++++++-- test/test_write.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py index 555a110d..ca21a6a0 100644 --- a/rocrate/rocrate.py +++ b/rocrate/rocrate.py @@ -123,7 +123,8 @@ def __init__(self, gen_preview=False, init=False, exclude=None, version=DEFAULT_VERSION, - load_subcrates=False): + load_subcrates=False, + root_dataset_id=None): self.mode = None self.source = source self.exclude = exclude @@ -138,7 +139,12 @@ def __init__(self, self.add(Preview(self)) if not source: self.mode = Mode.CREATE - self.add(RootDataset(self), Metadata(self, version=version)) + if root_dataset_id is not None: + if root_dataset_id != "./" and not is_url(root_dataset_id): + raise ValueError("the root dataset id must be either ./ or an absolute URI") + rde = RootDataset(self, root_dataset_id) + md = Metadata(self, properties={"about": rde}, version=version) + self.add(rde, md) elif init: self.mode = Mode.INIT if isinstance(source, dict): diff --git a/test/test_write.py b/test/test_write.py index 7c518db3..e6503833 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -716,3 +716,27 @@ def test_write_version(tmpdir, helpers, version): with open(out_path / basename, "rt") as f: data = json.load(f) assert data["@context"] == f"https://w3id.org/ro/crate/{version}/context" + + +@pytest.mark.parametrize("to_zip", [False, True]) +def test_detached_creation(tmpdir, to_zip): + base_uri = "http://example.com/crate/" + crate = ROCrate(root_dataset_id=base_uri) + assert crate.root_dataset.id == base_uri + assert crate.metadata.id == "ro-crate-metadata.json" + assert crate.metadata["about"] is crate.root_dataset + + out_path = tmpdir / "ro_crate_out" + if to_zip: + zip_path = tmpdir / 'ro_crate_out.zip' + crate.write_zip(zip_path) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(out_path) + else: + crate.write(out_path) + + assert (out_path / "ro-crate-metadata.json").is_file() + rcrate = ROCrate(out_path) + assert rcrate.root_dataset.id == base_uri + assert rcrate.metadata.id == "ro-crate-metadata.json" + assert rcrate.metadata["about"] is rcrate.root_dataset From f72e7f5d2eb379bd71bf0136e2eca64efea6d783 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 23 Jan 2026 10:21:05 +0100 Subject: [PATCH 05/32] expand test_detached_creation --- test/test_write.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/test_write.py b/test/test_write.py index e6503833..a20fa9a8 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -721,10 +721,18 @@ def test_write_version(tmpdir, helpers, version): @pytest.mark.parametrize("to_zip", [False, True]) def test_detached_creation(tmpdir, to_zip): base_uri = "http://example.com/crate/" + orcid = "https://orcid.org/0000-0002-1825-0097" + name = "Josiah Carberry" crate = ROCrate(root_dataset_id=base_uri) + assert crate.source is None + assert crate.metadata.source is None assert crate.root_dataset.id == base_uri assert crate.metadata.id == "ro-crate-metadata.json" assert crate.metadata["about"] is crate.root_dataset + crate.add_dataset(f"{base_uri}d1") + crate.add_file(f"{base_uri}f1") + p = crate.add(Person(crate, orcid, properties={"name": name})) + crate.root_dataset["creator"] = p out_path = tmpdir / "ro_crate_out" if to_zip: @@ -737,6 +745,20 @@ def test_detached_creation(tmpdir, to_zip): assert (out_path / "ro-crate-metadata.json").is_file() rcrate = ROCrate(out_path) + # this crate is attached, even though all its data entities are web-based + assert rcrate.source == out_path + assert rcrate.metadata.source == "ro-crate-metadata.json" assert rcrate.root_dataset.id == base_uri assert rcrate.metadata.id == "ro-crate-metadata.json" assert rcrate.metadata["about"] is rcrate.root_dataset + rd1 = rcrate.get(f"{base_uri}d1") + assert rd1 + rf1 = rcrate.get(f"{base_uri}f1") + assert rf1 + rp = crate.get(orcid) + assert rp["name"] == name + + with pytest.raises(ValueError): + ROCrate(root_dataset_id="foo/bar") + with pytest.raises(ValueError): + ROCrate(root_dataset_id="/foo/bar") From 3e3a0792f75604c52751d798b6de7fb1b9672505 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 30 Jan 2026 12:51:05 +0100 Subject: [PATCH 06/32] add support for reading a crate from a remote URL --- rocrate/metadata.py | 20 ++++++++++++++++++++ rocrate/model/file.py | 3 ++- rocrate/model/preview.py | 3 +++ rocrate/rocrate.py | 11 +++++++++-- test/test_read.py | 40 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 3 deletions(-) diff --git a/rocrate/metadata.py b/rocrate/metadata.py index 1e1c27f4..2a676724 100644 --- a/rocrate/metadata.py +++ b/rocrate/metadata.py @@ -21,8 +21,19 @@ # limitations under the License. import json +import re +import warnings + +import requests from .model.metadata import BASENAME, LEGACY_BASENAME +from .utils import is_url + +# https://www.researchobject.org/ro-crate/specification/1.2/structure +# "If stored in a file... the filename SHOULD be..." +# https://www.researchobject.org/ro-crate/specification/1.2/data-entities +# "It is NOT RECOMMENDED to resolve a relative root identifier..." +MD_PATTERN = re.compile(r".*[/-]ro-crate-metadata.json(ld)?$") def read_metadata(metadata_path): @@ -34,6 +45,15 @@ def read_metadata(metadata_path): """ if isinstance(metadata_path, dict): metadata = metadata_path + elif is_url(str(metadata_path)): + if not MD_PATTERN.match(metadata_path): + warnings.warn(f"URI {metadata_path} should follow the pattern {MD_PATTERN.pattern!r}") + resp = requests.get(metadata_path) + resp.raise_for_status() + content_type = resp.headers.get("Content-Type", "") + if "application/json" not in content_type.lower(): + warnings.warn(f"URI {metadata_path} does not have a JSON content type") + metadata = resp.json() else: with open(metadata_path, 'r', encoding='utf-8') as f: metadata = json.load(f) diff --git a/rocrate/model/file.py b/rocrate/model/file.py index 5a767462..43acf841 100644 --- a/rocrate/model/file.py +++ b/rocrate/model/file.py @@ -71,7 +71,8 @@ def _copy_file(self, path, out_file_path): self._jsonld['contentSize'] = str(out_file_path.stat().st_size) def write(self, base_path): - out_file_path = Path(base_path) / unquote(self.id) + local_path = self.get("localPath") + out_file_path = Path(base_path) / unquote(local_path or self.id) if isinstance(self.source, (BytesIO, StringIO)) or is_url(str(self.source)): self._write_from_stream(out_file_path) elif self.source is None: diff --git a/rocrate/model/preview.py b/rocrate/model/preview.py index 178989ec..8d1798f0 100644 --- a/rocrate/model/preview.py +++ b/rocrate/model/preview.py @@ -27,6 +27,7 @@ from jinja2 import Template from .file import File +from ..utils import is_url class Preview(File): @@ -98,6 +99,8 @@ def stream(self, chunk_size=8192): yield self.id, str.encode(self.generate_html(), encoding='utf-8') def _has_writeable_stream(self): + if is_url(str(self.source)): + return self.fetch_remote return True def write(self, dest_base): diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py index ca21a6a0..2f19fa94 100644 --- a/rocrate/rocrate.py +++ b/rocrate/rocrate.py @@ -176,7 +176,7 @@ def __init_from_tree(self, top_dir, gen_preview=False, version=DEFAULT_VERSION): self.add(Preview(self, source)) def __read(self, source, gen_preview=False): - if isinstance(source, dict): + if isinstance(source, dict) or is_url(str(source)): metadata_path = source else: source = Path(source) @@ -201,6 +201,8 @@ def __read(self, source, gen_preview=False): def __read_data_entities(self, entities, source, gen_preview): if isinstance(source, dict): source = Path("") + elif is_url(str(source)): + source = source.rsplit("/", 1)[0] + "/" metadata_id, root_id = find_root_entity_id(entities) root_entity = entities.pop(root_id) assert root_id == root_entity.pop('@id') @@ -212,7 +214,8 @@ def __read_data_entities(self, entities, source, gen_preview): preview_entity = entities.pop(Preview.BASENAME, None) if preview_entity and not gen_preview: - self.add(Preview(self, source / Preview.BASENAME, properties=preview_entity)) + preview_source = source + Preview.BASENAME if is_url(str(source)) else source / Preview.BASENAME + self.add(Preview(self, preview_source, properties=preview_entity)) self.__add_parts(parts, entities, source) def __add_parts(self, parts, entities, source): @@ -240,6 +243,8 @@ def __add_parts(self, parts, entities, source): if is_url(id_): instance = Subcrate(self, source=id_, properties=entity) + elif is_url(str(source)): + instance = Subcrate(self, source + id_, id_, properties=entity) else: instance = Subcrate(self, source=source / unquote(id_), properties=entity) @@ -250,6 +255,8 @@ def __add_parts(self, parts, entities, source): # cls is either a File or a Dataset (Directory) if is_url(id_): instance = cls(self, id_, properties=entity) + elif is_url(str(source)): + instance = cls(self, source + id_, id_, properties=entity) else: instance = cls(self, source / unquote(id_), id_, properties=entity) self.add(instance) diff --git a/test/test_read.py b/test/test_read.py index d905efc5..a75b8066 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -905,3 +905,43 @@ def test_not_data_entity_linked(version): assert f1 in crate.contextual_entities else: assert f1 in crate.data_entities + + +@pytest.mark.filterwarnings("ignore") +def test_from_uri(tmpdir): + source = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/read_crate/ro-crate-metadata.json") + base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/read_crate/") + remote_f_uri = ("https://raw.githubusercontent.com/ResearchObject/" + "ro-crate-py/master/test/test-data/sample_file.txt") + test_fn = "test_file_galaxy.txt" + crate = ROCrate(source) + assert crate.source == source + assert crate.preview.source == base_uri + "ro-crate-preview.html" + assert crate.preview.id == "ro-crate-preview.html" + assert test_fn in crate + test_f = crate.get(test_fn) + assert test_f.source == base_uri + test_fn + remote_f = crate.get(remote_f_uri) + assert remote_f.source == remote_f_uri + assert remote_f.id == remote_f_uri + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert not (out_path / "ro-crate-preview.html").exists() + assert not (out_path / test_fn).exists() + assert not (out_path / remote_f_uri.rsplit("/", 1)[1]).exists() + crate.preview.fetch_remote = True + test_f.fetch_remote = True + remote_f.fetch_remote = True + # remote_f.id is the full URI, which by default is interpreted as a local + # path (with a top-level "https:" directory etc.). The id cannot be + # changed, so if we want the file stored in the output crate with its + # basename we have to set localPath, which overrides the id as destination + remote_f["localPath"] = "sample_file.txt" + shutil.rmtree(out_path) + crate.write(out_path) + assert (out_path / "ro-crate-preview.html").is_file() + assert (out_path / test_fn).is_file() + assert (out_path / "sample_file.txt").is_file() From 26a207183fdf0c95554fec900bfc4dfd0c04b01a Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 30 Jan 2026 16:42:10 +0100 Subject: [PATCH 07/32] add test_from_uri_detached --- test/test_read.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/test_read.py b/test/test_read.py index a75b8066..2185ae76 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -945,3 +945,29 @@ def test_from_uri(tmpdir): assert (out_path / "ro-crate-preview.html").is_file() assert (out_path / test_fn).is_file() assert (out_path / "sample_file.txt").is_file() + + +@pytest.mark.filterwarnings("ignore") +def test_from_uri_detached(tmpdir): + source = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "detached_test_data/test/test-data/detached-ro-crate-metadata.json") + base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "detached_test_data/test/test-data/") + crate = ROCrate(source) + assert crate.source == source + assert crate.root_dataset.id == base_uri + sample_file = crate.get(f"{base_uri}sample_file.txt") + assert sample_file.source == f"{base_uri}sample_file.txt" + test_file_galaxy = crate.get(f"{base_uri}test_file_galaxy.txt") + assert test_file_galaxy.source == f"{base_uri}test_file_galaxy.txt" + assert test_file_galaxy.get("localPath") == "test-data/test_file_galaxy.txt" + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert not (out_path / "sample_file.txt").exists() + assert not (out_path / "test_file_galaxy.txt").exists() + assert not (out_path / "test-data" / "test_file_galaxy.txt").exists() + test_file_galaxy.fetch_remote = True + shutil.rmtree(out_path) + crate.write(out_path) + assert (out_path / "test-data" / "test_file_galaxy.txt").exists() From 8fff3ab788106fb87839d879e146971fc54a14b5 Mon Sep 17 00:00:00 2001 From: simleo Date: Mon, 2 Feb 2026 12:10:37 +0100 Subject: [PATCH 08/32] add test sections to check the written crates --- test/test_read.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/test/test_read.py b/test/test_read.py index 2185ae76..a199b7a5 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -668,7 +668,7 @@ def test_from_dict(tmpdir, version): ROCrate(metadata, init=True) -def test_detached(tmpdir): +def test_from_dict_remote_uris(tmpdir): base_uri = "http://example.com/" metadata = { "@context": "https://w3id.org/ro/crate/1.2/context", @@ -913,6 +913,7 @@ def test_from_uri(tmpdir): "master/test/test-data/read_crate/ro-crate-metadata.json") base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" "master/test/test-data/read_crate/") + # this is an absolute URI in the metadata, note it's outside the crate remote_f_uri = ("https://raw.githubusercontent.com/ResearchObject/" "ro-crate-py/master/test/test-data/sample_file.txt") test_fn = "test_file_galaxy.txt" @@ -923,15 +924,22 @@ def test_from_uri(tmpdir): assert test_fn in crate test_f = crate.get(test_fn) assert test_f.source == base_uri + test_fn + assert test_f.id == test_fn remote_f = crate.get(remote_f_uri) assert remote_f.source == remote_f_uri assert remote_f.id == remote_f_uri out_path = tmpdir / "out_crate" crate.write(out_path) assert (out_path / "ro-crate-metadata.json").is_file() + # fetch_remote is False by default, so nothing is fetched assert not (out_path / "ro-crate-preview.html").exists() assert not (out_path / test_fn).exists() assert not (out_path / remote_f_uri.rsplit("/", 1)[1]).exists() + rcrate = ROCrate(out_path) + assert rcrate.preview.id == "ro-crate-preview.html" + assert rcrate.get(test_fn) + assert rcrate.get(remote_f_uri) + # now set some fetch_remote to True crate.preview.fetch_remote = True test_f.fetch_remote = True remote_f.fetch_remote = True @@ -945,6 +953,14 @@ def test_from_uri(tmpdir): assert (out_path / "ro-crate-preview.html").is_file() assert (out_path / test_fn).is_file() assert (out_path / "sample_file.txt").is_file() + rcrate = ROCrate(out_path) + assert rcrate.preview.id == "ro-crate-preview.html" + rtest_f = rcrate.get(test_fn) + assert rtest_f + assert rtest_f.get("contentUrl") == base_uri + test_fn + rremote_f_uri = rcrate.get(remote_f_uri) + assert rremote_f_uri + assert rremote_f_uri.get("localPath") == "sample_file.txt" @pytest.mark.filterwarnings("ignore") @@ -966,8 +982,22 @@ def test_from_uri_detached(tmpdir): assert (out_path / "ro-crate-metadata.json").is_file() assert not (out_path / "sample_file.txt").exists() assert not (out_path / "test_file_galaxy.txt").exists() + # fetch_remote is False, so file is not fetched even if localPath is set assert not (out_path / "test-data" / "test_file_galaxy.txt").exists() + rcrate = ROCrate(out_path) + assert rcrate.get(f"{base_uri}sample_file.txt") + assert rcrate.get(f"{base_uri}test_file_galaxy.txt") + # now set fetch_remote to True test_file_galaxy.fetch_remote = True shutil.rmtree(out_path) crate.write(out_path) - assert (out_path / "test-data" / "test_file_galaxy.txt").exists() + assert (out_path / "test-data" / "test_file_galaxy.txt").is_file() + rcrate = ROCrate(out_path) + rsample_file = rcrate.get(f"{base_uri}sample_file.txt") + rtest_file_galaxy = rcrate.get(f"{base_uri}test_file_galaxy.txt") + assert rtest_file_galaxy + assert rtest_file_galaxy.get("contentUrl") == f"{base_uri}test_file_galaxy.txt" + assert rtest_file_galaxy.get("localPath") == "test-data/test_file_galaxy.txt" + assert set(rcrate.data_entities) == {rsample_file, rtest_file_galaxy} + license = rcrate.get("http://spdx.org/licenses/CC0-1.0") + assert set(rcrate.contextual_entities) == {license} From 8e36253f656e200990c99faf5778008ddf7d1b16 Mon Sep 17 00:00:00 2001 From: simleo Date: Mon, 2 Feb 2026 17:18:02 +0100 Subject: [PATCH 09/32] support for reading crates from file: URIs --- rocrate/metadata.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/rocrate/metadata.py b/rocrate/metadata.py index 2a676724..8e05042d 100644 --- a/rocrate/metadata.py +++ b/rocrate/metadata.py @@ -23,6 +23,7 @@ import json import re import warnings +import urllib.request import requests @@ -48,12 +49,16 @@ def read_metadata(metadata_path): elif is_url(str(metadata_path)): if not MD_PATTERN.match(metadata_path): warnings.warn(f"URI {metadata_path} should follow the pattern {MD_PATTERN.pattern!r}") - resp = requests.get(metadata_path) - resp.raise_for_status() - content_type = resp.headers.get("Content-Type", "") - if "application/json" not in content_type.lower(): - warnings.warn(f"URI {metadata_path} does not have a JSON content type") - metadata = resp.json() + if metadata_path.startswith("file:"): + with urllib.request.urlopen(metadata_path) as resp: + metadata = json.load(resp) + else: + with requests.get(metadata_path) as resp: + resp.raise_for_status() + content_type = resp.headers.get("Content-Type", "") + if "application/json" not in content_type.lower(): + warnings.warn(f"URI {metadata_path} does not have a JSON content type") + metadata = resp.json() else: with open(metadata_path, 'r', encoding='utf-8') as f: metadata = json.load(f) From 6f0cb4712654867faf6527180e4eea81fc4ecb03 Mon Sep 17 00:00:00 2001 From: simleo Date: Tue, 3 Feb 2026 11:19:36 +0100 Subject: [PATCH 10/32] update branch to master in test data URI --- test/test_read.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_read.py b/test/test_read.py index a199b7a5..21fcfa87 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -966,9 +966,9 @@ def test_from_uri(tmpdir): @pytest.mark.filterwarnings("ignore") def test_from_uri_detached(tmpdir): source = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" - "detached_test_data/test/test-data/detached-ro-crate-metadata.json") + "master/test/test-data/detached-ro-crate-metadata.json") base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" - "detached_test_data/test/test-data/") + "master/test/test-data/") crate = ROCrate(source) assert crate.source == source assert crate.root_dataset.id == base_uri From 60be6d5552eec5cc2b49f98f61f08483ab4e44af Mon Sep 17 00:00:00 2001 From: simleo Date: Tue, 3 Feb 2026 12:32:20 +0100 Subject: [PATCH 11/32] test_read.py: check reading from file: URI --- test/test_read.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/test/test_read.py b/test/test_read.py index 21fcfa87..27c595dc 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -32,6 +32,7 @@ _URL = ('https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/' 'test/test-data/sample_file.txt') +THIS_DIR = Path(__file__).absolute().parent @pytest.mark.parametrize("gen_preview,from_zip", [(False, False), (True, False), (True, True)]) @@ -908,11 +909,13 @@ def test_not_data_entity_linked(version): @pytest.mark.filterwarnings("ignore") -def test_from_uri(tmpdir): - source = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" - "master/test/test-data/read_crate/ro-crate-metadata.json") - base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" - "master/test/test-data/read_crate/") +@pytest.mark.parametrize("source_base", [ + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", + f"file://{THIS_DIR}/" +]) +def test_from_uri(tmpdir, source_base): + source = f"{source_base}test-data/read_crate/ro-crate-metadata.json" + base_uri = f"{source_base}test-data/read_crate/" # this is an absolute URI in the metadata, note it's outside the crate remote_f_uri = ("https://raw.githubusercontent.com/ResearchObject/" "ro-crate-py/master/test/test-data/sample_file.txt") @@ -964,9 +967,12 @@ def test_from_uri(tmpdir): @pytest.mark.filterwarnings("ignore") -def test_from_uri_detached(tmpdir): - source = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" - "master/test/test-data/detached-ro-crate-metadata.json") +@pytest.mark.parametrize("source_base", [ + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", + f"file://{THIS_DIR}/" +]) +def test_from_uri_detached(tmpdir, source_base): + source = f"{source_base}test-data/detached-ro-crate-metadata.json" base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" "master/test/test-data/") crate = ROCrate(source) From 190186c57b903f445e1dbdca083923f6e6140ed3 Mon Sep 17 00:00:00 2001 From: simleo Date: Wed, 4 Feb 2026 15:57:00 +0100 Subject: [PATCH 12/32] support for reading crates from local metadata file --- rocrate/rocrate.py | 26 +++++++++++++++++------- test/test_read.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py index 2f19fa94..bb6eacfa 100644 --- a/rocrate/rocrate.py +++ b/rocrate/rocrate.py @@ -188,11 +188,14 @@ def __read(self, source, gen_preview=False): with zipfile.ZipFile(source, "r") as zf: zf.extractall(zip_path) source = Path(zip_path) - metadata_path = source / BASENAME - if not metadata_path.is_file(): - metadata_path = source / LEGACY_BASENAME - if not metadata_path.is_file(): - raise ValueError(f"Not a valid RO-Crate: missing {BASENAME}") + if source.is_file(): + metadata_path = source + else: + metadata_path = source / BASENAME + if not metadata_path.is_file(): + metadata_path = source / LEGACY_BASENAME + if not metadata_path.is_file(): + raise ValueError(f"Not a valid RO-Crate: missing {BASENAME}") _, entities = read_metadata(metadata_path) self.__read_data_entities(entities, source, gen_preview) self.__read_contextual_entities(entities) @@ -214,7 +217,12 @@ def __read_data_entities(self, entities, source, gen_preview): preview_entity = entities.pop(Preview.BASENAME, None) if preview_entity and not gen_preview: - preview_source = source + Preview.BASENAME if is_url(str(source)) else source / Preview.BASENAME + if is_url(str(source)): + preview_source = source + Preview.BASENAME + elif source.is_file(): + preview_source = source.parent / Preview.BASENAME + else: + preview_source = source / Preview.BASENAME self.add(Preview(self, preview_source, properties=preview_entity)) self.__add_parts(parts, entities, source) @@ -245,6 +253,8 @@ def __add_parts(self, parts, entities, source): instance = Subcrate(self, source=id_, properties=entity) elif is_url(str(source)): instance = Subcrate(self, source + id_, id_, properties=entity) + elif source.is_file(): + instance = Subcrate(self, source.parent / unquote(id_), id_, properties=entity) else: instance = Subcrate(self, source=source / unquote(id_), properties=entity) @@ -257,6 +267,8 @@ def __add_parts(self, parts, entities, source): instance = cls(self, id_, properties=entity) elif is_url(str(source)): instance = cls(self, source + id_, id_, properties=entity) + elif source.is_file(): + instance = cls(self, source.parent / unquote(id_), id_, properties=entity) else: instance = cls(self, source / unquote(id_), id_, properties=entity) self.add(instance) @@ -601,7 +613,7 @@ def _copy_unlisted(self, top, base_path): def write(self, base_path): base_path = Path(base_path) base_path.mkdir(parents=True, exist_ok=True) - if self.source and not isinstance(self.source, dict): + if self.source and not isinstance(self.source, dict) and Path(self.source).is_dir(): self._copy_unlisted(self.source, base_path) for writable_entity in self.data_entities + self.default_entities: writable_entity.write(base_path) diff --git a/test/test_read.py b/test/test_read.py index 27c595dc..77aa692f 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -1007,3 +1007,53 @@ def test_from_uri_detached(tmpdir, source_base): assert set(rcrate.data_entities) == {rsample_file, rtest_file_galaxy} license = rcrate.get("http://spdx.org/licenses/CC0-1.0") assert set(rcrate.contextual_entities) == {license} + + +def test_from_file(test_data_dir, tmpdir): + source = test_data_dir / "read_crate" / "ro-crate-metadata.json" + base_path = test_data_dir / "read_crate" + remote_f_uri = ("https://raw.githubusercontent.com/ResearchObject/" + "ro-crate-py/master/test/test-data/sample_file.txt") + test_fn = "test_file_galaxy.txt" + crate = ROCrate(source) + assert crate.source == source + assert crate.preview.source == base_path / "ro-crate-preview.html" + assert crate.preview.id == "ro-crate-preview.html" + assert test_fn in crate + test_f = crate.get(test_fn) + assert test_f.source == base_path / test_fn + assert test_f.id == test_fn + remote_f = crate.get(remote_f_uri) + assert remote_f.source == remote_f_uri + assert remote_f.id == remote_f_uri + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert (out_path / "ro-crate-preview.html").exists() + assert (out_path / test_fn).exists() + assert not (out_path / remote_f_uri.rsplit("/", 1)[1]).exists() + rcrate = ROCrate(out_path) + assert rcrate.preview.id == "ro-crate-preview.html" + assert rcrate.get(test_fn) + assert rcrate.get(remote_f_uri) + assert rcrate.get("test/") + assert not rcrate.get("test/test-metadata.json") + + +def test_from_file_detached(test_data_dir, tmpdir): + source = test_data_dir / "detached-ro-crate-metadata.json" + base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/") + crate = ROCrate(source) + assert crate.source == source + assert crate.root_dataset.id == base_uri + sample_file = crate.get(f"{base_uri}sample_file.txt") + assert sample_file.source == f"{base_uri}sample_file.txt" + test_file_galaxy = crate.get(f"{base_uri}test_file_galaxy.txt") + assert test_file_galaxy.source == f"{base_uri}test_file_galaxy.txt" + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + rcrate = ROCrate(out_path) + assert rcrate.get(f"{base_uri}sample_file.txt") + assert rcrate.get(f"{base_uri}test_file_galaxy.txt") From 137e1066ad97cb15ef14881bca90ff2b4503dce2 Mon Sep 17 00:00:00 2001 From: simleo Date: Thu, 5 Feb 2026 10:38:05 +0100 Subject: [PATCH 13/32] fix test_read on Windows --- test/test_read.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_read.py b/test/test_read.py index 77aa692f..f1da12a7 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -32,7 +32,7 @@ _URL = ('https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/' 'test/test-data/sample_file.txt') -THIS_DIR = Path(__file__).absolute().parent +THIS_DIR = Path(__file__).resolve().parent @pytest.mark.parametrize("gen_preview,from_zip", [(False, False), (True, False), (True, True)]) @@ -911,7 +911,7 @@ def test_not_data_entity_linked(version): @pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("source_base", [ "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", - f"file://{THIS_DIR}/" + f"file:///{THIS_DIR}/" # extra slash needed on some windows systems ]) def test_from_uri(tmpdir, source_base): source = f"{source_base}test-data/read_crate/ro-crate-metadata.json" @@ -969,7 +969,7 @@ def test_from_uri(tmpdir, source_base): @pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("source_base", [ "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", - f"file://{THIS_DIR}/" + f"file:///{THIS_DIR}/" # extra slash needed on some windows systems ]) def test_from_uri_detached(tmpdir, source_base): source = f"{source_base}test-data/detached-ro-crate-metadata.json" From 779762d2117cec851481212ed2dd42e3c04d378d Mon Sep 17 00:00:00 2001 From: simleo Date: Thu, 5 Feb 2026 15:09:44 +0100 Subject: [PATCH 14/32] add write_detached --- rocrate/model/metadata.py | 9 +++++++++ rocrate/rocrate.py | 3 +++ test/test_write.py | 16 ++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/rocrate/model/metadata.py b/rocrate/model/metadata.py index a1b8bfe9..ae33e74d 100644 --- a/rocrate/model/metadata.py +++ b/rocrate/model/metadata.py @@ -24,6 +24,8 @@ import json from pathlib import Path +import re +import warnings from .file import File from .dataset import Dataset @@ -37,6 +39,7 @@ DEFAULT_VERSION = "1.2" BASENAME = "ro-crate-metadata.json" LEGACY_BASENAME = "ro-crate-metadata.jsonld" +DETACHED_MD_NAME = re.compile(r".*-ro-crate-metadata.json$") WORKFLOW_PROFILE = "https://w3id.org/workflowhub/workflow-ro-crate/1.0" @@ -97,6 +100,12 @@ def write(self, dest_base): write_path = Path(dest_base) / self.id.rsplit("/", 1)[-1] super()._write_from_stream(write_path) + def write_detached(self, path): + if not DETACHED_MD_NAME.match(str(path)): + warnings.warn(f"{path} should follow the pattern {DETACHED_MD_NAME.pattern!r}") + path = Path(path) + super()._write_from_stream(path) + @property def root(self) -> Dataset: return self.crate.root_dataset diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py index bb6eacfa..c7b9e2ee 100644 --- a/rocrate/rocrate.py +++ b/rocrate/rocrate.py @@ -627,6 +627,9 @@ def write_zip(self, out_path): f.write(chunk) return out_path + def write_detached(self, metadata_path): + self.metadata.write_detached(metadata_path) + def stream_zip(self, chunk_size=8192): """ Create a stream of bytes representing the RO-Crate as a ZIP file. """ yield from self._stream_zip(chunk_size=chunk_size) diff --git a/test/test_write.py b/test/test_write.py index a20fa9a8..6322d16b 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -758,6 +758,22 @@ def test_detached_creation(tmpdir, to_zip): rp = crate.get(orcid) assert rp["name"] == name + detached_md_path = tmpdir / "example-ro-crate-metadata.json" + crate.write_detached(detached_md_path) + assert detached_md_path.is_file() + rcrate = ROCrate(detached_md_path) + assert rcrate.source == detached_md_path + assert rcrate.metadata.source == "ro-crate-metadata.json" + assert rcrate.root_dataset.id == base_uri + assert rcrate.metadata.id == "ro-crate-metadata.json" + assert rcrate.metadata["about"] is rcrate.root_dataset + rd1 = rcrate.get(f"{base_uri}d1") + assert rd1 + rf1 = rcrate.get(f"{base_uri}f1") + assert rf1 + rp = crate.get(orcid) + assert rp["name"] == name + with pytest.raises(ValueError): ROCrate(root_dataset_id="foo/bar") with pytest.raises(ValueError): From 4d5f6bd95aa3a3a885314fca4d7444726aba020d Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 6 Feb 2026 16:14:15 +0100 Subject: [PATCH 15/32] support referencing detached crates --- rocrate/rocrate.py | 6 ++- .../subcrates-ro-crate-metadata.json | 51 +++++++++++++++++++ test/test_read.py | 19 +++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py index c7b9e2ee..86546635 100644 --- a/rocrate/rocrate.py +++ b/rocrate/rocrate.py @@ -945,7 +945,11 @@ def _load_subcrate(self): """ if self._crate is None: # load_subcrates=True to load further nested RO-Crate (on-demand / lazily too) - self._crate = ROCrate(self.source, load_subcrates=True) + if subject_of := self.get("subjectOf"): + subcrate_uri = subject_of.id if isinstance(subject_of, Entity) else subject_of + self._crate = ROCrate(subcrate_uri, load_subcrates=True) + else: + self._crate = ROCrate(self.source, load_subcrates=True) def write(self, base_path): super().write(base_path) diff --git a/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json b/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json new file mode 100644 index 00000000..ae514025 --- /dev/null +++ b/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json @@ -0,0 +1,51 @@ +{ + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached_crate_with_subcrates/", + "@type": "Dataset", + "name": "Detached crate with subcrates", + "description": "A detached RO-Crate that references other crates", + "license": "https://spdx.org/licenses/MIT.html", + "datePublished": "2026-02-06", + "hasPart": [ + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/sample_file.txt" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/" + } + ] + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached_crate_with_subcrates/" + }, + "conformsTo": { + "@id": "https://w3id.org/ro/crate/1.2" + } + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/sample_file.txt", + "@type": "File" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/", + "@type": "Dataset", + "conformsTo": { + "@id": "https://w3id.org/ro/crate" + }, + "subjectOf": { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached-ro-crate-metadata.json" + } + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached-ro-crate-metadata.json", + "@type": "CreativeWork", + "encodingFormat": "application/ld+json", + "sdDatePublished": "2026-02-06" + } + ] +} diff --git a/test/test_read.py b/test/test_read.py index f1da12a7..465237ae 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -240,6 +240,25 @@ def test_crate_with_subcrates(test_data_dir): assert subcrate._crate is nested_crate +@pytest.mark.filterwarnings("ignore") +def test_detached_crate_with_subcrates(test_data_dir): + main_crate = ROCrate( + test_data_dir / "detached_crate_with_subcrates/subcrates-ro-crate-metadata.json", + load_subcrates=True + ) + subcrate = main_crate.get("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/") + assert isinstance(subcrate, Subcrate) + assert set(main_crate.subcrate_entities) == {subcrate} + assert subcrate.get("conformsTo") == "https://w3id.org/ro/crate" + assert subcrate._crate is None + other_crate = subcrate.get_crate() + assert isinstance(other_crate, ROCrate) + assert subcrate._crate is other_crate + assert other_crate.get( + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/test_file_galaxy.txt" + ) + + @pytest.mark.parametrize("override", [False, True]) def test_init(test_data_dir, tmpdir, helpers, override): crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase" From 567f169c2061d238737a6bfdcbe2f84379036f14 Mon Sep 17 00:00:00 2001 From: simleo Date: Tue, 10 Feb 2026 12:13:31 +0100 Subject: [PATCH 16/32] remove non-json content type warning when reading from url --- rocrate/metadata.py | 3 --- test/test_read.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/rocrate/metadata.py b/rocrate/metadata.py index 8e05042d..521f108f 100644 --- a/rocrate/metadata.py +++ b/rocrate/metadata.py @@ -55,9 +55,6 @@ def read_metadata(metadata_path): else: with requests.get(metadata_path) as resp: resp.raise_for_status() - content_type = resp.headers.get("Content-Type", "") - if "application/json" not in content_type.lower(): - warnings.warn(f"URI {metadata_path} does not have a JSON content type") metadata = resp.json() else: with open(metadata_path, 'r', encoding='utf-8') as f: diff --git a/test/test_read.py b/test/test_read.py index 465237ae..97fd0499 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -240,7 +240,6 @@ def test_crate_with_subcrates(test_data_dir): assert subcrate._crate is nested_crate -@pytest.mark.filterwarnings("ignore") def test_detached_crate_with_subcrates(test_data_dir): main_crate = ROCrate( test_data_dir / "detached_crate_with_subcrates/subcrates-ro-crate-metadata.json", @@ -927,7 +926,6 @@ def test_not_data_entity_linked(version): assert f1 in crate.data_entities -@pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("source_base", [ "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", f"file:///{THIS_DIR}/" # extra slash needed on some windows systems @@ -985,7 +983,6 @@ def test_from_uri(tmpdir, source_base): assert rremote_f_uri.get("localPath") == "sample_file.txt" -@pytest.mark.filterwarnings("ignore") @pytest.mark.parametrize("source_base", [ "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", f"file:///{THIS_DIR}/" # extra slash needed on some windows systems From 37a53d8e18081578e322f8d7d30f2eda7f0d1a33 Mon Sep 17 00:00:00 2001 From: simleo Date: Tue, 10 Feb 2026 16:21:54 +0100 Subject: [PATCH 17/32] add a section on detached crates to the docs --- README.md | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/README.md b/README.md index 3710cf48..0012c150 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,95 @@ article = crate.dereference("paper.pdf") ## Advanced features +### Detached crates + +[RO-Crate 1.2](https://www.researchobject.org/ro-crate/whats-changed-in-1-2) introduces the concept of _detached_ RO-Crates, which have no defined root directory: in detached crates, the metadata is accessed independently, for instance via an API or from a standalone metadata file. By contrast, "traditional" crates that describe a payload of files and directories contained in a root directory are called _attached_. + +Both detached and attached crates can have a root data entity with an absolute URI as `@id`. To create an RO-Crate whose root data entity `@id` is different from the default `./`, use the `root_dataset_id` argument in the constructor: + +```python +from rocrate.rocrate import ROCrate + +url = "http://example.com/crate/" +crate = ROCrate(root_dataset_id=url) +``` + +In detached crates, _all_ data entities must be web-based, i.e., have an absolute URI as `@id`: + +``` +f1 = crate.add_file(f"{url}f1") +``` + +The [recommended way](https://www.researchobject.org/ro-crate/specification/1.2/structure.html#types-of-ro-crate) to store a detached crate on disk is to write a single metadata file called `${prefix}-ro-crate-metadata.json`: + +``` +crate.write_detached("/tmp/example-ro-crate-metadata.json") +``` + +One of the ways to consume a detached crate is to read the metadata from a local file: + +``` +rcrate = ROCrate("/tmp/example-ro-crate-metadata.json") +rf1 = rcrate.dereference(f"{url}f1") +``` + +This also works with a local `file://` URI: + +```python +rcrate = ROCrate("file:///tmp/example-ro-crate-metadata.json") +``` + +and with a remote URI: + +```python +base = "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/" +rcrate = ROCrate(f"{base}detached-ro-crate-metadata.json") +assert rcrate.root_dataset.id == base +sample_file = rcrate.dereference(f"{base}sample_file.txt") +test_file_galaxy = rcrate.dereference(f"{base}test_file_galaxy.txt") +``` + +Suppose you now want to save the crate to the local file system. You could use `write_detached` as shown above: + +```python +rcrate.write_detached("/tmp/detached-ro-crate-metadata.json") +``` + +but you could also write the crate as attached, after tweaking the data entities a bit: + +```python +sample_file.fetch_remote = True +sample_file["localPath"] = "sample_file.txt" +test_file_galaxy.fetch_remote = True +rcrate.write("/tmp/crate") +``` + +This leads to the following structure on the file system: + +```python +/tmp/crate/ +|-- ro-crate-metadata.json +|-- sample_file.txt +`-- test-data + `-- test_file_galaxy.txt +``` + +Note that we did not have to set `localPath` for `test_file_galaxy` because it was already set in the original crate that we read from the remote url. + +Another way to read a detached crate is to pass a JSON dictionary with the RO-Crate metadata directly to `ROCrate`. For instance: + +```python +import json +from rocrate.rocrate import ROCrate + +with open("/tmp/example-ro-crate-metadata.json") as f: + metadata = json.load(f) +crate = ROCrate(metadata) +``` + +In the above example we read the metadata from a local file, but you could get it from an API endpoint or any other source. + + ### Subcrates An RO-Crate can contain one or more nested RO-Crates. For instance, consider the following layout: From 5ca8aefc61513b3d71d6ea67e14b3656cfda40be Mon Sep 17 00:00:00 2001 From: simleo Date: Tue, 10 Feb 2026 16:30:06 +0100 Subject: [PATCH 18/32] fix code highlighting in the docs --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0012c150..7e08aedc 100644 --- a/README.md +++ b/README.md @@ -308,19 +308,19 @@ crate = ROCrate(root_dataset_id=url) In detached crates, _all_ data entities must be web-based, i.e., have an absolute URI as `@id`: -``` +```python f1 = crate.add_file(f"{url}f1") ``` The [recommended way](https://www.researchobject.org/ro-crate/specification/1.2/structure.html#types-of-ro-crate) to store a detached crate on disk is to write a single metadata file called `${prefix}-ro-crate-metadata.json`: -``` +```python crate.write_detached("/tmp/example-ro-crate-metadata.json") ``` One of the ways to consume a detached crate is to read the metadata from a local file: -``` +```python rcrate = ROCrate("/tmp/example-ro-crate-metadata.json") rf1 = rcrate.dereference(f"{url}f1") ``` @@ -358,7 +358,7 @@ rcrate.write("/tmp/crate") This leads to the following structure on the file system: -```python +``` /tmp/crate/ |-- ro-crate-metadata.json |-- sample_file.txt From 8382d290893c5467294ae1be57da75d15255ffe6 Mon Sep 17 00:00:00 2001 From: Simone Leo Date: Thu, 12 Feb 2026 12:19:54 +0100 Subject: [PATCH 19/32] Apply suggestions from code review Co-authored-by: Eli Chadwick --- test/test_write.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_write.py b/test/test_write.py index 6322d16b..a98f25a9 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -755,7 +755,7 @@ def test_detached_creation(tmpdir, to_zip): assert rd1 rf1 = rcrate.get(f"{base_uri}f1") assert rf1 - rp = crate.get(orcid) + rp = rcrate.get(orcid) assert rp["name"] == name detached_md_path = tmpdir / "example-ro-crate-metadata.json" @@ -771,7 +771,7 @@ def test_detached_creation(tmpdir, to_zip): assert rd1 rf1 = rcrate.get(f"{base_uri}f1") assert rf1 - rp = crate.get(orcid) + rp = rcrate.get(orcid) assert rp["name"] == name with pytest.raises(ValueError): From 0955c89ae5eff86e5f7c508d386e73e4ad392744 Mon Sep 17 00:00:00 2001 From: simleo Date: Thu, 12 Feb 2026 14:30:07 +0100 Subject: [PATCH 20/32] clarify some bits in the docs --- README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 7e08aedc..db474bdb 100644 --- a/README.md +++ b/README.md @@ -309,42 +309,42 @@ crate = ROCrate(root_dataset_id=url) In detached crates, _all_ data entities must be web-based, i.e., have an absolute URI as `@id`: ```python -f1 = crate.add_file(f"{url}f1") +file_1 = crate.add_file(f"{url}file_1") # http://example.com/crate/file_1 ``` -The [recommended way](https://www.researchobject.org/ro-crate/specification/1.2/structure.html#types-of-ro-crate) to store a detached crate on disk is to write a single metadata file called `${prefix}-ro-crate-metadata.json`: +The [recommended way](https://www.researchobject.org/ro-crate/specification/1.2/structure.html#types-of-ro-crate) to store a detached crate on disk is to write a single metadata file called `${prefix}-ro-crate-metadata.json`, where `${prefix}` is a variable. The library supports this through the `write_detached` method, which takes as argument an arbitrary path (a warning will be issued if the path does not follow the above pattern): ```python crate.write_detached("/tmp/example-ro-crate-metadata.json") ``` -One of the ways to consume a detached crate is to read the metadata from a local file: +One of the ways to consume a detached crate is to read the metadata from a local file. For instance, to read the crate that we just wrote: ```python -rcrate = ROCrate("/tmp/example-ro-crate-metadata.json") -rf1 = rcrate.dereference(f"{url}f1") +read_crate = ROCrate("/tmp/example-ro-crate-metadata.json") +read_file_1 = read_crate.dereference(f"{url}file_1") ``` This also works with a local `file://` URI: ```python -rcrate = ROCrate("file:///tmp/example-ro-crate-metadata.json") +read_crate = ROCrate("file:///tmp/example-ro-crate-metadata.json") ``` and with a remote URI: ```python base = "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/" -rcrate = ROCrate(f"{base}detached-ro-crate-metadata.json") -assert rcrate.root_dataset.id == base -sample_file = rcrate.dereference(f"{base}sample_file.txt") -test_file_galaxy = rcrate.dereference(f"{base}test_file_galaxy.txt") +read_crate = ROCrate(f"{base}detached-ro-crate-metadata.json") +assert read_crate.root_dataset.id == base +sample_file = read_crate.dereference(f"{base}sample_file.txt") +test_file_galaxy = read_crate.dereference(f"{base}test_file_galaxy.txt") ``` Suppose you now want to save the crate to the local file system. You could use `write_detached` as shown above: ```python -rcrate.write_detached("/tmp/detached-ro-crate-metadata.json") +read_crate.write_detached("/tmp/detached-ro-crate-metadata.json") ``` but you could also write the crate as attached, after tweaking the data entities a bit: @@ -353,7 +353,7 @@ but you could also write the crate as attached, after tweaking the data entities sample_file.fetch_remote = True sample_file["localPath"] = "sample_file.txt" test_file_galaxy.fetch_remote = True -rcrate.write("/tmp/crate") +read_crate.write("/tmp/crate") ``` This leads to the following structure on the file system: From 312a2a52f529092046df827f5c241272dd7432d1 Mon Sep 17 00:00:00 2001 From: simleo Date: Thu, 12 Feb 2026 15:42:36 +0100 Subject: [PATCH 21/32] file.write: cut out path to the basename if it's a url and fetch_remote is True --- rocrate/model/file.py | 6 ++++-- test/test_read.py | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/rocrate/model/file.py b/rocrate/model/file.py index 43acf841..30580aa3 100644 --- a/rocrate/model/file.py +++ b/rocrate/model/file.py @@ -71,8 +71,10 @@ def _copy_file(self, path, out_file_path): self._jsonld['contentSize'] = str(out_file_path.stat().st_size) def write(self, base_path): - local_path = self.get("localPath") - out_file_path = Path(base_path) / unquote(local_path or self.id) + relative_dest_uri = self.get("localPath") or self.id + if self.fetch_remote and is_url(relative_dest_uri): + relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] + out_file_path = Path(base_path) / unquote(relative_dest_uri) if isinstance(self.source, (BytesIO, StringIO)) or is_url(str(self.source)): self._write_from_stream(out_file_path) elif self.source is None: diff --git a/test/test_read.py b/test/test_read.py index 97fd0499..a84e465b 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -963,16 +963,21 @@ def test_from_uri(tmpdir, source_base): crate.preview.fetch_remote = True test_f.fetch_remote = True remote_f.fetch_remote = True - # remote_f.id is the full URI, which by default is interpreted as a local - # path (with a top-level "https:" directory etc.). The id cannot be - # changed, so if we want the file stored in the output crate with its - # basename we have to set localPath, which overrides the id as destination - remote_f["localPath"] = "sample_file.txt" + assert remote_f.id == remote_f_uri + # remote_f.id is the full URI. Check that the destination path is + # automatically set to the output dir / basename upon writing. shutil.rmtree(out_path) crate.write(out_path) + assert (out_path / "sample_file.txt").is_file() + # check that localPath overrides the default destination + remote_f["localPath"] = "other/sample_file.txt" + shutil.rmtree(out_path) + crate.write(out_path) + assert (out_path / "other" / "sample_file.txt").is_file() + # check more paths assert (out_path / "ro-crate-preview.html").is_file() assert (out_path / test_fn).is_file() - assert (out_path / "sample_file.txt").is_file() + # read back the crate rcrate = ROCrate(out_path) assert rcrate.preview.id == "ro-crate-preview.html" rtest_f = rcrate.get(test_fn) @@ -980,7 +985,7 @@ def test_from_uri(tmpdir, source_base): assert rtest_f.get("contentUrl") == base_uri + test_fn rremote_f_uri = rcrate.get(remote_f_uri) assert rremote_f_uri - assert rremote_f_uri.get("localPath") == "sample_file.txt" + assert rremote_f_uri.get("localPath") == "other/sample_file.txt" @pytest.mark.parametrize("source_base", [ From 4e46b2c1b61ad4cae97167f10657c4880daa2582 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 13 Feb 2026 16:10:42 +0100 Subject: [PATCH 22/32] file write: set localPath when dest path is set to basename --- rocrate/model/file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rocrate/model/file.py b/rocrate/model/file.py index 30580aa3..15223da9 100644 --- a/rocrate/model/file.py +++ b/rocrate/model/file.py @@ -74,6 +74,7 @@ def write(self, base_path): relative_dest_uri = self.get("localPath") or self.id if self.fetch_remote and is_url(relative_dest_uri): relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] + self["localPath"] = relative_dest_uri out_file_path = Path(base_path) / unquote(relative_dest_uri) if isinstance(self.source, (BytesIO, StringIO)) or is_url(str(self.source)): self._write_from_stream(out_file_path) From dbd5e338f55f70fec9274ced09f4635bc3e3f80e Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 13 Feb 2026 16:14:33 +0100 Subject: [PATCH 23/32] docs: clarify localPath usage --- README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index db474bdb..9f010eeb 100644 --- a/README.md +++ b/README.md @@ -351,7 +351,6 @@ but you could also write the crate as attached, after tweaking the data entities ```python sample_file.fetch_remote = True -sample_file["localPath"] = "sample_file.txt" test_file_galaxy.fetch_remote = True read_crate.write("/tmp/crate") ``` @@ -366,7 +365,22 @@ This leads to the following structure on the file system: `-- test_file_galaxy.txt ``` -Note that we did not have to set `localPath` for `test_file_galaxy` because it was already set in the original crate that we read from the remote url. +Note that `test_file_galaxy` was written as `test-data/test-file-galaxy.txt` relative to the crate root: this is due to the fact that the original crate specifies `"localPath": "test-data/test-file-galaxy.txt"`. In contrast, `localPath` is not specified for the other file, so the library applies the default behavior of using the basename as the relative path. This can be changed by setting `localPath` on the file entity: + +```python +sample_file["localPath"] = "test-data/sample_file.txt" +read_crate.write("/tmp/crate2") +``` + +Which leads to: + +``` +/tmp/crate2/ +|-- ro-crate-metadata.json +`-- test-data + |-- sample_file.txt + `-- test_file_galaxy.txt +``` Another way to read a detached crate is to pass a JSON dictionary with the RO-Crate metadata directly to `ROCrate`. For instance: From 8c16c88640c2742667e5811408b7ab5adc4f312a Mon Sep 17 00:00:00 2001 From: simleo Date: Mon, 16 Feb 2026 14:49:33 +0100 Subject: [PATCH 24/32] file.write: strip rde id to get relative path when applicable --- rocrate/model/file.py | 5 ++++- .../test-data/detached-ro-crate-metadata.json | 14 ++++++++++++ test/test_read.py | 22 +++++++++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/rocrate/model/file.py b/rocrate/model/file.py index 15223da9..35a4cbf0 100644 --- a/rocrate/model/file.py +++ b/rocrate/model/file.py @@ -73,7 +73,10 @@ def _copy_file(self, path, out_file_path): def write(self, base_path): relative_dest_uri = self.get("localPath") or self.id if self.fetch_remote and is_url(relative_dest_uri): - relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] + if relative_dest_uri.startswith(self.crate.root_dataset.id): + relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] + else: + relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] self["localPath"] = relative_dest_uri out_file_path = Path(base_path) / unquote(relative_dest_uri) if isinstance(self.source, (BytesIO, StringIO)) or is_url(str(self.source)): diff --git a/test/test-data/detached-ro-crate-metadata.json b/test/test-data/detached-ro-crate-metadata.json index 1da64d29..63aefb96 100644 --- a/test/test-data/detached-ro-crate-metadata.json +++ b/test/test-data/detached-ro-crate-metadata.json @@ -26,6 +26,12 @@ }, { "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/test_file_galaxy.txt" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/read_extra/listed.txt" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test_read.py" } ] }, @@ -40,6 +46,14 @@ "name": "Test data", "localPath": "test-data/test_file_galaxy.txt" }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/read_extra/listed.txt", + "@type": "File" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test_read.py", + "@type": "File" + }, { "@id": "http://spdx.org/licenses/CC0-1.0", "@type": "CreativeWork", diff --git a/test/test_read.py b/test/test_read.py index a84e465b..c10de43e 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -965,7 +965,7 @@ def test_from_uri(tmpdir, source_base): remote_f.fetch_remote = True assert remote_f.id == remote_f_uri # remote_f.id is the full URI. Check that the destination path is - # automatically set to the output dir / basename upon writing. + # the output dir / remote_f.id minus the root dataset id. shutil.rmtree(out_path) crate.write(out_path) assert (out_path / "sample_file.txt").is_file() @@ -1021,11 +1021,11 @@ def test_from_uri_detached(tmpdir, source_base): assert (out_path / "test-data" / "test_file_galaxy.txt").is_file() rcrate = ROCrate(out_path) rsample_file = rcrate.get(f"{base_uri}sample_file.txt") + assert rsample_file rtest_file_galaxy = rcrate.get(f"{base_uri}test_file_galaxy.txt") assert rtest_file_galaxy assert rtest_file_galaxy.get("contentUrl") == f"{base_uri}test_file_galaxy.txt" assert rtest_file_galaxy.get("localPath") == "test-data/test_file_galaxy.txt" - assert set(rcrate.data_entities) == {rsample_file, rtest_file_galaxy} license = rcrate.get("http://spdx.org/licenses/CC0-1.0") assert set(rcrate.contextual_entities) == {license} @@ -1072,9 +1072,27 @@ def test_from_file_detached(test_data_dir, tmpdir): assert sample_file.source == f"{base_uri}sample_file.txt" test_file_galaxy = crate.get(f"{base_uri}test_file_galaxy.txt") assert test_file_galaxy.source == f"{base_uri}test_file_galaxy.txt" + listed = crate.get(f"{base_uri}read_extra/listed.txt") + assert listed.source == f"{base_uri}read_extra/listed.txt" + test_read_base = ("https://raw.githubusercontent.com/ResearchObject/" + "ro-crate-py/master/test/") + test_read = crate.get(f"{test_read_base}test_read.py") + assert test_read.source == f"{test_read_base}test_read.py" out_path = tmpdir / "out_crate" crate.write(out_path) assert (out_path / "ro-crate-metadata.json").is_file() rcrate = ROCrate(out_path) assert rcrate.get(f"{base_uri}sample_file.txt") assert rcrate.get(f"{base_uri}test_file_galaxy.txt") + assert rcrate.get(f"{base_uri}read_extra/listed.txt") + assert rcrate.get(f"{test_read_base}test_read.py") + + # set fetch_remote to True and check local paths + for e in crate.data_entities: + e.fetch_remote = True + shutil.rmtree(out_path) + crate.write(out_path) + assert (out_path / "sample_file.txt").is_file() + assert (out_path / "test-data" / "test_file_galaxy.txt").is_file() + assert (out_path / "read_extra" / "listed.txt").is_file() + assert (out_path / "test_read.py").is_file() From ff6a26762c0d77641957714c64f6693fbadeea27 Mon Sep 17 00:00:00 2001 From: simleo Date: Tue, 17 Feb 2026 15:19:36 +0100 Subject: [PATCH 25/32] override dest with localPath only when downloading a remote file --- rocrate/model/file.py | 17 ++++++++++------- test/test_write.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/rocrate/model/file.py b/rocrate/model/file.py index 35a4cbf0..7e07fb5f 100644 --- a/rocrate/model/file.py +++ b/rocrate/model/file.py @@ -71,13 +71,16 @@ def _copy_file(self, path, out_file_path): self._jsonld['contentSize'] = str(out_file_path.stat().st_size) def write(self, base_path): - relative_dest_uri = self.get("localPath") or self.id - if self.fetch_remote and is_url(relative_dest_uri): - if relative_dest_uri.startswith(self.crate.root_dataset.id): - relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] - else: - relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] - self["localPath"] = relative_dest_uri + if self.fetch_remote and is_url(str(self.source)): + relative_dest_uri = self.get("localPath") or self.id + if is_url(relative_dest_uri): + if relative_dest_uri.startswith(self.crate.root_dataset.id): + relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] + else: + relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] + self["localPath"] = relative_dest_uri + else: + relative_dest_uri = self.id out_file_path = Path(base_path) / unquote(relative_dest_uri) if isinstance(self.source, (BytesIO, StringIO)) or is_url(str(self.source)): self._write_from_stream(out_file_path) diff --git a/test/test_write.py b/test/test_write.py index a98f25a9..6f13a569 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -177,6 +177,25 @@ def test_remote_uri(tmpdir, helpers, fetch_remote, validate_url, to_zip): assert "sdDatePublished" in props +def test_local_path(test_data_dir, tmpdir): + crate = ROCrate() + url = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/sample_file.txt") + sample_file = crate.add_file(url) + assert sample_file.id == url + sample_file["localPath"] = "test-data/sample_file.txt" + sample_file.fetch_remote = True + test_file_galaxy = crate.add_file(test_data_dir / "test_file_galaxy.txt") + assert test_file_galaxy.id == "test_file_galaxy.txt" + test_file_galaxy["localPath"] = "foo/bar.txt" + out_path = tmpdir / "ro_crate_out" + crate.write(out_path) + assert (out_path / "test-data" / "sample_file.txt").is_file() + assert not (out_path / "sample_file.txt").exists() + assert not (out_path / "foo" / "bar.txt").exists() + assert (out_path / "test_file_galaxy.txt").is_file() + + def test_file_uri(tmpdir): f_name = uuid.uuid4().hex f_path = (tmpdir / f_name).resolve() From deda277191e2a08369511375f7ed66469b76466a Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 27 Mar 2026 12:59:01 +0100 Subject: [PATCH 26/32] fix remote dir handling --- rocrate/model/dataset.py | 20 +++++++++++------- test/test_read.py | 45 ++++++++++++++++++++++++++++++++++++++++ test/test_write.py | 7 ++++--- 3 files changed, 62 insertions(+), 10 deletions(-) diff --git a/rocrate/model/dataset.py b/rocrate/model/dataset.py index 28131761..a58d6326 100644 --- a/rocrate/model/dataset.py +++ b/rocrate/model/dataset.py @@ -121,17 +121,23 @@ def _stream_folder_from_url(self, chunk_size=8192): with urlopen(self.source) as _: self._jsonld['sdDatePublished'] = iso_now() else: - base = self.source.rstrip("/") + relative_dest_uri = self.get("localPath") or self.id + if is_url(relative_dest_uri): + if relative_dest_uri.startswith(self.crate.root_dataset.id): + relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] + else: + relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] + self["localPath"] = relative_dest_uri + out_dir_path = Path(unquote(relative_dest_uri)) + for entry in self._jsonld.get("hasPart", []): try: part = entry["@id"] - if is_url(part) or part.startswith("/"): - raise RuntimeError(f"'{self.source}': part '{part}' is not a relative path") - part_uri = f"{base}/{part}" - rel_out_path = Path(self.id) / part - + if not is_url(part): + raise RuntimeError(f"'{self.source}' is a URL, but part '{part}' is not a URL") + rel_out_path = out_dir_path / part.rsplit("/", 1)[-1] is_empty = True - with urlopen(part_uri) as response: + with urlopen(part) as response: while chunk := response.read(chunk_size): is_empty = False yield str(rel_out_path), chunk diff --git a/test/test_read.py b/test/test_read.py index c10de43e..67b56d58 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -1096,3 +1096,48 @@ def test_from_file_detached(test_data_dir, tmpdir): assert (out_path / "test-data" / "test_file_galaxy.txt").is_file() assert (out_path / "read_extra" / "listed.txt").is_file() assert (out_path / "test_read.py").is_file() + + +def test_read_remote_dir(tmpdir): + base_uri = "https://ftp.mozilla.org/pub/" + metadata = { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": base_uri}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} + }, + { + "@id": base_uri, + "@type": "Dataset", + "datePublished": "2026-02-19", + "hasPart": [{"@id": f"{base_uri}misc/errorpages/"}] + }, + { + "@id": f"{base_uri}misc/errorpages/", + "@type": "Dataset", + "hasPart": [ + {"@id": f"{base_uri}misc/errorpages/404.html"}, + {"@id": f"{base_uri}misc/errorpages/500.html"} + ] + }, + { + "@id": f"{base_uri}misc/errorpages/404.html", + "@type": "File" + }, + { + "@id": f"{base_uri}misc/errorpages/500.html", + "@type": "File" + } + ] + } + crate = ROCrate(metadata) + errorpages = crate.get(f"{base_uri}misc/errorpages/") + errorpages.fetch_remote = True + out_path = tmpdir / 'out_crate' + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert (out_path / "misc" / "errorpages" / "404.html").is_file() + assert (out_path / "misc" / "errorpages" / "500.html").is_file() diff --git a/test/test_write.py b/test/test_write.py index 6f13a569..5d77a7a8 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -281,8 +281,8 @@ def test_remote_dir(tmpdir, helpers, fetch_remote, validate_url): relpath = "pub/misc/errorpages/" properties = { "hasPart": [ - {"@id": "404.html"}, - {"@id": "500.html"}, + {"@id": "https://ftp.mozilla.org/pub/misc/errorpages/404.html"}, + {"@id": "https://ftp.mozilla.org/pub/misc/errorpages/500.html"}, ], } kw = { @@ -305,7 +305,8 @@ def test_remote_dir(tmpdir, helpers, fetch_remote, validate_url): out_dataset = out_crate.dereference(relpath) assert (out_path / relpath).is_dir() for entry in properties["hasPart"]: - assert (out_path / relpath / entry["@id"]).is_file() + basename = entry["@id"].rsplit("/", 1)[-1] + assert (out_path / relpath / basename).is_file() else: out_dataset = out_crate.dereference(url) assert not (out_path / relpath).exists() From 5eeccdceeef5e209074831f03431a9de93803a36 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 27 Mar 2026 15:41:29 +0100 Subject: [PATCH 27/32] remove docs on writing a detached crate "as attached" --- README.md | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/README.md b/README.md index c10480c9..3fd58888 100644 --- a/README.md +++ b/README.md @@ -341,47 +341,6 @@ sample_file = read_crate.dereference(f"{base}sample_file.txt") test_file_galaxy = read_crate.dereference(f"{base}test_file_galaxy.txt") ``` -Suppose you now want to save the crate to the local file system. You could use `write_detached` as shown above: - -```python -read_crate.write_detached("/tmp/detached-ro-crate-metadata.json") -``` - -but you could also write the crate as attached, after tweaking the data entities a bit: - -```python -sample_file.fetch_remote = True -test_file_galaxy.fetch_remote = True -read_crate.write("/tmp/crate") -``` - -This leads to the following structure on the file system: - -``` -/tmp/crate/ -|-- ro-crate-metadata.json -|-- sample_file.txt -`-- test-data - `-- test_file_galaxy.txt -``` - -Note that `test_file_galaxy` was written as `test-data/test-file-galaxy.txt` relative to the crate root: this is due to the fact that the original crate specifies `"localPath": "test-data/test-file-galaxy.txt"`. In contrast, `localPath` is not specified for the other file, so the library applies the default behavior of using the basename as the relative path. This can be changed by setting `localPath` on the file entity: - -```python -sample_file["localPath"] = "test-data/sample_file.txt" -read_crate.write("/tmp/crate2") -``` - -Which leads to: - -``` -/tmp/crate2/ -|-- ro-crate-metadata.json -`-- test-data - |-- sample_file.txt - `-- test_file_galaxy.txt -``` - Another way to read a detached crate is to pass a JSON dictionary with the RO-Crate metadata directly to `ROCrate`. For instance: ```python From 35cb0b8bbaaa4a4eb72841bdb14ef79fd1b0e4c0 Mon Sep 17 00:00:00 2001 From: simleo Date: Mon, 30 Mar 2026 10:30:31 +0200 Subject: [PATCH 28/32] don't override a relative dest_path with localPath (create mode) --- rocrate/model/dataset.py | 5 ++++- rocrate/model/file.py | 5 ++++- test/test_write.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/rocrate/model/dataset.py b/rocrate/model/dataset.py index 979229ed..e32c69f4 100644 --- a/rocrate/model/dataset.py +++ b/rocrate/model/dataset.py @@ -122,7 +122,10 @@ def _stream_folder_from_url(self, chunk_size=8192): with urlopen(self.source) as _: self._jsonld['sdDatePublished'] = iso_now() else: - relative_dest_uri = self.get("localPath") or self.id + if is_url(self.id): + relative_dest_uri = self.get("localPath") or self.id + else: + relative_dest_uri = self.id if is_url(relative_dest_uri): if relative_dest_uri.startswith(self.crate.root_dataset.id): relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] diff --git a/rocrate/model/file.py b/rocrate/model/file.py index a42e7c69..2e7122d0 100644 --- a/rocrate/model/file.py +++ b/rocrate/model/file.py @@ -73,7 +73,10 @@ def _copy_file(self, path, out_file_path): def write(self, base_path): if self.fetch_remote and is_url(str(self.source)): - relative_dest_uri = self.get("localPath") or self.id + if is_url(self.id): + relative_dest_uri = self.get("localPath") or self.id + else: + relative_dest_uri = self.id if is_url(relative_dest_uri): if relative_dest_uri.startswith(self.crate.root_dataset.id): relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] diff --git a/test/test_write.py b/test/test_write.py index 66ce6cf3..a2f91a16 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -197,6 +197,40 @@ def test_local_path(test_data_dir, tmpdir): assert (out_path / "test_file_galaxy.txt").is_file() +def test_local_path_vs_relative_id(test_data_dir, tmpdir): + crate = ROCrate() + url = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/sample_file.txt") + sample_file = crate.add_file(url, dest_path="examples/sample_file.txt") + assert sample_file.id == "examples/sample_file.txt" + sample_file["localPath"] = "test-data/sample_file.txt" + sample_file.fetch_remote = True + out_path = tmpdir / "ro_crate_out" + crate.write(out_path) + assert (out_path / "examples" / "sample_file.txt").is_file() + assert not (out_path / "test-data" / "sample_file.txt").exists() + + +def test_local_path_vs_relative_id_dataset(test_data_dir, tmpdir): + crate = ROCrate() + f1 = crate.add_file("https://ftp.mozilla.org/pub/misc/errorpages/404.html") + f2 = crate.add_file("https://ftp.mozilla.org/pub/misc/errorpages/500.html") + dataset = crate.add_dataset( + "https://ftp.mozilla.org/pub/misc/errorpages/", + dest_path="errorpages/", + fetch_remote=True, + ) + assert dataset.id == "errorpages/" + dataset["hasPart"] = [f1, f2] + dataset["localPath"] = "mozilla_error_pages" + out_path = tmpdir / "ro_crate_out" + crate.write(out_path) + assert (out_path / "errorpages").is_dir() + assert (out_path / "errorpages" / "404.html").is_file() + assert (out_path / "errorpages" / "500.html").is_file() + assert not (out_path / "mozilla_error_pages").exists() + + def test_file_uri(tmpdir): f_name = uuid.uuid4().hex f_path = (tmpdir / f_name).resolve() From 6b8439523603398059f95380b50cdb205bd1e0c0 Mon Sep 17 00:00:00 2001 From: simleo Date: Wed, 1 Apr 2026 10:51:55 +0200 Subject: [PATCH 29/32] override dataset localPath with part localPath if set --- rocrate/model/dataset.py | 7 +++++- test/test_read.py | 49 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/rocrate/model/dataset.py b/rocrate/model/dataset.py index e32c69f4..b8706390 100644 --- a/rocrate/model/dataset.py +++ b/rocrate/model/dataset.py @@ -31,7 +31,7 @@ from urllib.parse import unquote from .file_or_dir import FileOrDir -from ..utils import is_url, iso_now, Mode +from ..utils import as_list, is_url, iso_now, Mode class Dataset(FileOrDir): @@ -140,6 +140,11 @@ def _stream_folder_from_url(self, chunk_size=8192): if not is_url(part): raise RuntimeError(f"'{self.source}' is a URL, but part '{part}' is not a URL") rel_out_path = out_dir_path / part.rsplit("/", 1)[-1] + # override with file localPath if set + if part_file := self.crate.get(part): + if "File" in as_list(part_file.type): + if file_local_path := part_file.get("localPath"): + rel_out_path = file_local_path is_empty = True with urlopen(part) as response: while chunk := response.read(chunk_size): diff --git a/test/test_read.py b/test/test_read.py index a77dcbe6..931c4a7f 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -1142,3 +1142,52 @@ def test_read_remote_dir(tmpdir): assert (out_path / "ro-crate-metadata.json").is_file() assert (out_path / "misc" / "errorpages" / "404.html").is_file() assert (out_path / "misc" / "errorpages" / "500.html").is_file() + + +def test_read_remote_dir_local_path(tmpdir): + base_uri = "https://ftp.mozilla.org/pub/" + metadata = { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": base_uri}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} + }, + { + "@id": base_uri, + "@type": "Dataset", + "datePublished": "2026-02-19", + "hasPart": [{"@id": f"{base_uri}misc/errorpages/"}] + }, + { + "@id": f"{base_uri}misc/errorpages/", + "@type": "Dataset", + "localPath": "errorpages/", + "hasPart": [ + {"@id": f"{base_uri}misc/errorpages/404.html"}, + {"@id": f"{base_uri}misc/errorpages/500.html"} + ] + }, + { + "@id": f"{base_uri}misc/errorpages/404.html", + "@type": "File" + }, + { + "@id": f"{base_uri}misc/errorpages/500.html", + "@type": "File", + "localPath": "errorpages/500/500.html" + } + ] + } + crate = ROCrate(metadata) + errorpages = crate.get(f"{base_uri}misc/errorpages/") + errorpages.fetch_remote = True + out_path = tmpdir / 'out_crate' + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert not (out_path / "misc").exists() + assert (out_path / "errorpages" / "404.html").is_file() + assert not (out_path / "errorpages" / "500.html").exists() + assert (out_path / "errorpages" / "500" / "500.html").is_file() From 59753610d4d0163edd60cb5d1ac11d92ba1e5421 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 10 Apr 2026 11:51:40 +0200 Subject: [PATCH 30/32] fetch_remote on Dataset: skip Dataset parts --- rocrate/model/dataset.py | 10 ++++--- test/test_read.py | 58 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/rocrate/model/dataset.py b/rocrate/model/dataset.py index b8706390..83a27728 100644 --- a/rocrate/model/dataset.py +++ b/rocrate/model/dataset.py @@ -140,10 +140,12 @@ def _stream_folder_from_url(self, chunk_size=8192): if not is_url(part): raise RuntimeError(f"'{self.source}' is a URL, but part '{part}' is not a URL") rel_out_path = out_dir_path / part.rsplit("/", 1)[-1] - # override with file localPath if set - if part_file := self.crate.get(part): - if "File" in as_list(part_file.type): - if file_local_path := part_file.get("localPath"): + if part_entity := self.crate.get(part): + if "Dataset" in as_list(part_entity.type): + continue + # override with file localPath if set + if "File" in as_list(part_entity.type): + if file_local_path := part_entity.get("localPath"): rel_out_path = file_local_path is_empty = True with urlopen(part) as response: diff --git a/test/test_read.py b/test/test_read.py index 931c4a7f..95062900 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -1191,3 +1191,61 @@ def test_read_remote_dir_local_path(tmpdir): assert (out_path / "errorpages" / "404.html").is_file() assert not (out_path / "errorpages" / "500.html").exists() assert (out_path / "errorpages" / "500" / "500.html").is_file() + + +def test_read_remote_dir_skip_dir_part(tmpdir): + base_uri = "https://ftp.mozilla.org/pub/" + metadata = { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": base_uri}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} + }, + { + "@id": base_uri, + "@type": "Dataset", + "datePublished": "2026-02-19", + "hasPart": [{"@id": f"{base_uri}misc/"}] + }, + { + "@id": f"{base_uri}misc/", + "@type": "Dataset", + "hasPart": [ + # part is dir, should not try to download anything + {"@id": f"{base_uri}misc/errorpages/"}, + ] + }, + { + "@id": f"{base_uri}misc/errorpages/", + "@type": "Dataset", + "hasPart": [ + {"@id": f"{base_uri}misc/errorpages/404.html"}, + {"@id": f"{base_uri}misc/errorpages/500.html"} + ] + }, + { + "@id": f"{base_uri}misc/errorpages/404.html", + "@type": "File" + }, + { + "@id": f"{base_uri}misc/errorpages/500.html", + "@type": "File" + } + ] + } + crate = ROCrate(metadata) + misc = crate.get(f"{base_uri}misc/") + misc.fetch_remote = True + errorpages = crate.get(f"{base_uri}misc/errorpages/") + errorpages.fetch_remote = True + out_path = tmpdir / 'out_crate' + crate.write(out_path) + + assert (out_path / "ro-crate-metadata.json").is_file() + assert (out_path / "misc").is_dir() + assert (out_path / "misc" / "errorpages").is_dir() + assert (out_path / "misc" / "errorpages" / "404.html").is_file() + assert (out_path / "misc" / "errorpages" / "500.html").is_file() From fc617479a70846dfb0102a5ec498a7667e612cd8 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 10 Apr 2026 12:12:03 +0200 Subject: [PATCH 31/32] don't auto-set localPath after a download --- rocrate/model/dataset.py | 1 - rocrate/model/file.py | 1 - 2 files changed, 2 deletions(-) diff --git a/rocrate/model/dataset.py b/rocrate/model/dataset.py index 83a27728..859c672b 100644 --- a/rocrate/model/dataset.py +++ b/rocrate/model/dataset.py @@ -131,7 +131,6 @@ def _stream_folder_from_url(self, chunk_size=8192): relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] else: relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] - self["localPath"] = relative_dest_uri out_dir_path = Path(unquote(relative_dest_uri)) for entry in self._jsonld.get("hasPart", []): diff --git a/rocrate/model/file.py b/rocrate/model/file.py index 2e7122d0..0c087433 100644 --- a/rocrate/model/file.py +++ b/rocrate/model/file.py @@ -82,7 +82,6 @@ def write(self, base_path): relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] else: relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] - self["localPath"] = relative_dest_uri else: relative_dest_uri = self.id out_file_path = Path(base_path) / unquote(relative_dest_uri) From ba531a6e6ee9bec35db2d121abba7daa646ed4b2 Mon Sep 17 00:00:00 2001 From: simleo Date: Fri, 10 Apr 2026 16:02:50 +0200 Subject: [PATCH 32/32] split long tests into smaller ones --- test/test_read.py | 39 +++++++++++++++++++++++++++++++++------ test/test_write.py | 23 +++++++++++++++-------- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/test/test_read.py b/test/test_read.py index 95062900..075f90d0 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -960,6 +960,22 @@ def test_from_uri(tmpdir, source_base): assert rcrate.preview.id == "ro-crate-preview.html" assert rcrate.get(test_fn) assert rcrate.get(remote_f_uri) + + +@pytest.mark.parametrize("source_base", [ + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", + f"file:///{THIS_DIR}/" # extra slash needed on some windows systems +]) +def test_from_uri_fetch_remote(tmpdir, source_base): + source = f"{source_base}test-data/read_crate/ro-crate-metadata.json" + base_uri = f"{source_base}test-data/read_crate/" + # this is an absolute URI in the metadata, note it's outside the crate + remote_f_uri = ("https://raw.githubusercontent.com/ResearchObject/" + "ro-crate-py/master/test/test-data/sample_file.txt") + test_fn = "test_file_galaxy.txt" + crate = ROCrate(source) + remote_f = crate.get(remote_f_uri) + test_f = crate.get(test_fn) # now set some fetch_remote to True crate.preview.fetch_remote = True test_f.fetch_remote = True @@ -967,7 +983,7 @@ def test_from_uri(tmpdir, source_base): assert remote_f.id == remote_f_uri # remote_f.id is the full URI. Check that the destination path is # the output dir / remote_f.id minus the root dataset id. - shutil.rmtree(out_path) + out_path = tmpdir / "out_crate" crate.write(out_path) assert (out_path / "sample_file.txt").is_file() # check that localPath overrides the default destination @@ -1015,20 +1031,31 @@ def test_from_uri_detached(tmpdir, source_base): rcrate = ROCrate(out_path) assert rcrate.get(f"{base_uri}sample_file.txt") assert rcrate.get(f"{base_uri}test_file_galaxy.txt") + license = rcrate.get("http://spdx.org/licenses/CC0-1.0") + assert set(rcrate.contextual_entities) == {license} + + +@pytest.mark.parametrize("source_base", [ + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", + f"file:///{THIS_DIR}/" # extra slash needed on some windows systems +]) +def test_from_uri_detached_fetch_remote(tmpdir, source_base): + source = f"{source_base}test-data/detached-ro-crate-metadata.json" + base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/") + crate = ROCrate(source) + test_file_galaxy = crate.get(f"{base_uri}test_file_galaxy.txt") # now set fetch_remote to True test_file_galaxy.fetch_remote = True - shutil.rmtree(out_path) + assert test_file_galaxy.get("localPath") == "test-data/test_file_galaxy.txt" + out_path = tmpdir / "out_crate" crate.write(out_path) assert (out_path / "test-data" / "test_file_galaxy.txt").is_file() rcrate = ROCrate(out_path) - rsample_file = rcrate.get(f"{base_uri}sample_file.txt") - assert rsample_file rtest_file_galaxy = rcrate.get(f"{base_uri}test_file_galaxy.txt") assert rtest_file_galaxy assert rtest_file_galaxy.get("contentUrl") == f"{base_uri}test_file_galaxy.txt" assert rtest_file_galaxy.get("localPath") == "test-data/test_file_galaxy.txt" - license = rcrate.get("http://spdx.org/licenses/CC0-1.0") - assert set(rcrate.contextual_entities) == {license} def test_from_file(test_data_dir, tmpdir): diff --git a/test/test_write.py b/test/test_write.py index a2f91a16..0f3f5bee 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -806,13 +806,20 @@ def test_detached_creation(tmpdir, to_zip): assert rcrate.root_dataset.id == base_uri assert rcrate.metadata.id == "ro-crate-metadata.json" assert rcrate.metadata["about"] is rcrate.root_dataset - rd1 = rcrate.get(f"{base_uri}d1") - assert rd1 - rf1 = rcrate.get(f"{base_uri}f1") - assert rf1 + assert rcrate.get(f"{base_uri}d1") + assert rcrate.get(f"{base_uri}f1") rp = rcrate.get(orcid) assert rp["name"] == name + +def test_detached_creation_write_detached(tmpdir): + base_uri = "http://example.com/crate/" + orcid = "https://orcid.org/0000-0002-1825-0097" + name = "Josiah Carberry" + crate = ROCrate(root_dataset_id=base_uri) + crate.add_dataset(f"{base_uri}d1") + crate.add_file(f"{base_uri}f1") + crate.add(Person(crate, orcid, properties={"name": name})) detached_md_path = tmpdir / "example-ro-crate-metadata.json" crate.write_detached(detached_md_path) assert detached_md_path.is_file() @@ -822,13 +829,13 @@ def test_detached_creation(tmpdir, to_zip): assert rcrate.root_dataset.id == base_uri assert rcrate.metadata.id == "ro-crate-metadata.json" assert rcrate.metadata["about"] is rcrate.root_dataset - rd1 = rcrate.get(f"{base_uri}d1") - assert rd1 - rf1 = rcrate.get(f"{base_uri}f1") - assert rf1 + assert rcrate.get(f"{base_uri}d1") + assert rcrate.get(f"{base_uri}f1") rp = rcrate.get(orcid) assert rp["name"] == name + +def test_detached_creation_exceptions(): with pytest.raises(ValueError): ROCrate(root_dataset_id="foo/bar") with pytest.raises(ValueError):