diff --git a/README.md b/README.md index 20e06137..3fd58888 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,68 @@ article = crate.dereference("paper.pdf") ## Advanced features +### Detached crates + +[RO-Crate 1.2](https://www.researchobject.org/ro-crate/whats-changed-in-1-2) introduces the concept of _detached_ RO-Crates, which have no defined root directory: in detached crates, the metadata is accessed independently, for instance via an API or from a standalone metadata file. By contrast, "traditional" crates that describe a payload of files and directories contained in a root directory are called _attached_. + +Both detached and attached crates can have a root data entity with an absolute URI as `@id`. To create an RO-Crate whose root data entity `@id` is different from the default `./`, use the `root_dataset_id` argument in the constructor: + +```python +from rocrate.rocrate import ROCrate + +url = "http://example.com/crate/" +crate = ROCrate(root_dataset_id=url) +``` + +In detached crates, _all_ data entities must be web-based, i.e., have an absolute URI as `@id`: + +```python +file_1 = crate.add_file(f"{url}file_1") # http://example.com/crate/file_1 +``` + +The [recommended way](https://www.researchobject.org/ro-crate/specification/1.2/structure.html#types-of-ro-crate) to store a detached crate on disk is to write a single metadata file called `${prefix}-ro-crate-metadata.json`, where `${prefix}` is a variable. The library supports this through the `write_detached` method, which takes as argument an arbitrary path (a warning will be issued if the path does not follow the above pattern): + +```python +crate.write_detached("/tmp/example-ro-crate-metadata.json") +``` + +One of the ways to consume a detached crate is to read the metadata from a local file. For instance, to read the crate that we just wrote: + +```python +read_crate = ROCrate("/tmp/example-ro-crate-metadata.json") +read_file_1 = read_crate.dereference(f"{url}file_1") +``` + +This also works with a local `file://` URI: + +```python +read_crate = ROCrate("file:///tmp/example-ro-crate-metadata.json") +``` + +and with a remote URI: + +```python +base = "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/" +read_crate = ROCrate(f"{base}detached-ro-crate-metadata.json") +assert read_crate.root_dataset.id == base +sample_file = read_crate.dereference(f"{base}sample_file.txt") +test_file_galaxy = read_crate.dereference(f"{base}test_file_galaxy.txt") +``` + +Another way to read a detached crate is to pass a JSON dictionary with the RO-Crate metadata directly to `ROCrate`. For instance: + +```python +import json +from rocrate.rocrate import ROCrate + +with open("/tmp/example-ro-crate-metadata.json") as f: + metadata = json.load(f) +crate = ROCrate(metadata) +``` + +In the above example we read the metadata from a local file, but you could get it from an API endpoint or any other source. + + ### Subcrates An RO-Crate can contain one or more nested RO-Crates. For instance, consider the following layout: diff --git a/rocrate/metadata.py b/rocrate/metadata.py index 69aacf74..dfa009ae 100644 --- a/rocrate/metadata.py +++ b/rocrate/metadata.py @@ -22,9 +22,20 @@ # limitations under the License. import json +import re import warnings +import urllib.request + +import requests from .model.metadata import BASENAME, LEGACY_BASENAME +from .utils import is_url + +# https://www.researchobject.org/ro-crate/specification/1.2/structure +# "If stored in a file... the filename SHOULD be..." +# https://www.researchobject.org/ro-crate/specification/1.2/data-entities +# "It is NOT RECOMMENDED to resolve a relative root identifier..." +MD_PATTERN = re.compile(r".*[/-]ro-crate-metadata.json(ld)?$") def read_metadata(metadata_path): @@ -36,6 +47,16 @@ def read_metadata(metadata_path): """ if isinstance(metadata_path, dict): metadata = metadata_path + elif is_url(str(metadata_path)): + if not MD_PATTERN.match(metadata_path): + warnings.warn(f"URI {metadata_path} should follow the pattern {MD_PATTERN.pattern!r}") + if metadata_path.startswith("file:"): + with urllib.request.urlopen(metadata_path) as resp: + metadata = json.load(resp) + else: + with requests.get(metadata_path) as resp: + resp.raise_for_status() + metadata = resp.json() else: with open(metadata_path, 'r', encoding='utf-8') as f: metadata = json.load(f) @@ -69,48 +90,8 @@ def find_root_entity_id(entities): Return a tuple of the corresponding identifiers (descriptor, root). If the entities are not found, raise KeyError. If they are found, but they don't satisfy the required constraints, raise ValueError. - - In the general case, the metadata file descriptor id can be an - absolute URI whose last path segment is "ro-crate-metadata.json[ld]". - Since there can be more than one such id in the crate, we need to - choose among the corresponding (descriptor, root) entity pairs. First, we - exclude those that don't satisfy other constraints, such as the - descriptor entity being of type CreativeWork, etc.; if this doesn't - leave us with a single pair, we try to pick one with a - heuristic. Suppose we are left with the (m1, r1) and (m2, r2) pairs: - if r1 is the actual root of this crate, then m2 and r2 are regular - files in it, and as such they must appear in r1's hasPart; r2, - however, is not required to have a hasPart property listing other - files. Thus, we look for a pair whose root entity "contains" all - descriptor entities from other pairs. If there is no such pair, or there - is more than one, we just return an arbitrary pair. - """ descriptor = entities.get(BASENAME, entities.get(LEGACY_BASENAME)) - if descriptor: - return _check_descriptor(descriptor, entities) - candidates = [] - for id_, e in entities.items(): - basename = id_.rsplit("/", 1)[-1] - if basename == BASENAME or basename == LEGACY_BASENAME: - try: - candidates.append(_check_descriptor(e, entities)) - except ValueError: - pass - if not candidates: + if not descriptor: raise KeyError("Metadata file descriptor not found") - elif len(candidates) == 1: - return candidates[0] - else: - warnings.warn("Multiple metadata file descriptors, will pick one with a heuristic") - descriptor_ids = set(_[0] for _ in candidates) - for m_id, r_id in candidates: - try: - root = entities[r_id] - part_ids = set(_["@id"] for _ in root["hasPart"]) - except KeyError: - continue - if part_ids >= descriptor_ids - {m_id}: - # if True for more than one candidate, this pick is arbitrary - return m_id, r_id - return candidates[0] # fall back to arbitrary pick + return _check_descriptor(descriptor, entities) diff --git a/rocrate/model/dataset.py b/rocrate/model/dataset.py index 8317d2eb..859c672b 100644 --- a/rocrate/model/dataset.py +++ b/rocrate/model/dataset.py @@ -31,7 +31,7 @@ from urllib.parse import unquote from .file_or_dir import FileOrDir -from ..utils import is_url, iso_now, Mode +from ..utils import as_list, is_url, iso_now, Mode class Dataset(FileOrDir): @@ -122,17 +122,32 @@ def _stream_folder_from_url(self, chunk_size=8192): with urlopen(self.source) as _: self._jsonld['sdDatePublished'] = iso_now() else: - base = self.source.rstrip("/") + if is_url(self.id): + relative_dest_uri = self.get("localPath") or self.id + else: + relative_dest_uri = self.id + if is_url(relative_dest_uri): + if relative_dest_uri.startswith(self.crate.root_dataset.id): + relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] + else: + relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] + out_dir_path = Path(unquote(relative_dest_uri)) + for entry in self._jsonld.get("hasPart", []): try: part = entry["@id"] - if is_url(part) or part.startswith("/"): - raise RuntimeError(f"'{self.source}': part '{part}' is not a relative path") - part_uri = f"{base}/{part}" - rel_out_path = Path(self.id) / part - + if not is_url(part): + raise RuntimeError(f"'{self.source}' is a URL, but part '{part}' is not a URL") + rel_out_path = out_dir_path / part.rsplit("/", 1)[-1] + if part_entity := self.crate.get(part): + if "Dataset" in as_list(part_entity.type): + continue + # override with file localPath if set + if "File" in as_list(part_entity.type): + if file_local_path := part_entity.get("localPath"): + rel_out_path = file_local_path is_empty = True - with urlopen(part_uri) as response: + with urlopen(part) as response: while chunk := response.read(chunk_size): is_empty = False yield str(rel_out_path), chunk diff --git a/rocrate/model/file.py b/rocrate/model/file.py index be042cbc..0c087433 100644 --- a/rocrate/model/file.py +++ b/rocrate/model/file.py @@ -72,7 +72,19 @@ def _copy_file(self, path, out_file_path): self._jsonld['contentSize'] = str(out_file_path.stat().st_size) def write(self, base_path): - out_file_path = Path(base_path) / unquote(self.id) + if self.fetch_remote and is_url(str(self.source)): + if is_url(self.id): + relative_dest_uri = self.get("localPath") or self.id + else: + relative_dest_uri = self.id + if is_url(relative_dest_uri): + if relative_dest_uri.startswith(self.crate.root_dataset.id): + relative_dest_uri = relative_dest_uri[len(self.crate.root_dataset.id):] + else: + relative_dest_uri = relative_dest_uri.rsplit("/", 1)[-1] + else: + relative_dest_uri = self.id + out_file_path = Path(base_path) / unquote(relative_dest_uri) if isinstance(self.source, (BytesIO, StringIO)) or is_url(str(self.source)): self._write_from_stream(out_file_path) elif self.source is None: diff --git a/rocrate/model/metadata.py b/rocrate/model/metadata.py index 14ac280a..af16bdb2 100644 --- a/rocrate/model/metadata.py +++ b/rocrate/model/metadata.py @@ -25,6 +25,8 @@ import json from pathlib import Path +import re +import warnings from .file import File from .dataset import Dataset @@ -38,6 +40,7 @@ DEFAULT_VERSION = "1.2" BASENAME = "ro-crate-metadata.json" LEGACY_BASENAME = "ro-crate-metadata.jsonld" +DETACHED_MD_NAME = re.compile(r".*-ro-crate-metadata.json$") WORKFLOW_PROFILE = "https://w3id.org/workflowhub/workflow-ro-crate/1.0" @@ -95,9 +98,15 @@ def _has_writeable_stream(self): return True def write(self, dest_base): - write_path = Path(dest_base) / self.id + write_path = Path(dest_base) / self.id.rsplit("/", 1)[-1] super()._write_from_stream(write_path) + def write_detached(self, path): + if not DETACHED_MD_NAME.match(str(path)): + warnings.warn(f"{path} should follow the pattern {DETACHED_MD_NAME.pattern!r}") + path = Path(path) + super()._write_from_stream(path) + @property def root(self) -> Dataset: return self.crate.root_dataset diff --git a/rocrate/model/preview.py b/rocrate/model/preview.py index ba284e2a..86d4f6d7 100644 --- a/rocrate/model/preview.py +++ b/rocrate/model/preview.py @@ -28,6 +28,7 @@ from jinja2 import Template from .file import File +from ..utils import is_url class Preview(File): @@ -99,6 +100,8 @@ def stream(self, chunk_size=8192): yield self.id, str.encode(self.generate_html(), encoding='utf-8') def _has_writeable_stream(self): + if is_url(str(self.source)): + return self.fetch_remote return True def write(self, dest_base): diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py index 5808bfb6..005595f0 100644 --- a/rocrate/rocrate.py +++ b/rocrate/rocrate.py @@ -124,7 +124,8 @@ def __init__(self, gen_preview=False, init=False, exclude=None, version=DEFAULT_VERSION, - load_subcrates=False): + load_subcrates=False, + root_dataset_id=None): self.mode = None self.source = source self.exclude = exclude @@ -139,7 +140,12 @@ def __init__(self, self.add(Preview(self)) if not source: self.mode = Mode.CREATE - self.add(RootDataset(self), Metadata(self, version=version)) + if root_dataset_id is not None: + if root_dataset_id != "./" and not is_url(root_dataset_id): + raise ValueError("the root dataset id must be either ./ or an absolute URI") + rde = RootDataset(self, root_dataset_id) + md = Metadata(self, properties={"about": rde}, version=version) + self.add(rde, md) elif init: self.mode = Mode.INIT if isinstance(source, dict): @@ -171,7 +177,7 @@ def __init_from_tree(self, top_dir, gen_preview=False, version=DEFAULT_VERSION): self.add(Preview(self, source)) def __read(self, source, gen_preview=False): - if isinstance(source, dict): + if isinstance(source, dict) or is_url(str(source)): metadata_path = source else: source = Path(source) @@ -183,11 +189,14 @@ def __read(self, source, gen_preview=False): with zipfile.ZipFile(source, "r") as zf: zf.extractall(zip_path) source = Path(zip_path) - metadata_path = source / BASENAME - if not metadata_path.is_file(): - metadata_path = source / LEGACY_BASENAME - if not metadata_path.is_file(): - raise ValueError(f"Not a valid RO-Crate: missing {BASENAME}") + if source.is_file(): + metadata_path = source + else: + metadata_path = source / BASENAME + if not metadata_path.is_file(): + metadata_path = source / LEGACY_BASENAME + if not metadata_path.is_file(): + raise ValueError(f"Not a valid RO-Crate: missing {BASENAME}") _, entities = read_metadata(metadata_path) self.__read_data_entities(entities, source, gen_preview) self.__read_contextual_entities(entities) @@ -196,6 +205,8 @@ def __read(self, source, gen_preview=False): def __read_data_entities(self, entities, source, gen_preview): if isinstance(source, dict): source = Path("") + elif is_url(str(source)): + source = source.rsplit("/", 1)[0] + "/" metadata_id, root_id = find_root_entity_id(entities) root_entity = entities.pop(root_id) assert root_id == root_entity.pop('@id') @@ -207,7 +218,13 @@ def __read_data_entities(self, entities, source, gen_preview): preview_entity = entities.pop(Preview.BASENAME, None) if preview_entity and not gen_preview: - self.add(Preview(self, source / Preview.BASENAME, properties=preview_entity)) + if is_url(str(source)): + preview_source = source + Preview.BASENAME + elif source.is_file(): + preview_source = source.parent / Preview.BASENAME + else: + preview_source = source / Preview.BASENAME + self.add(Preview(self, preview_source, properties=preview_entity)) self.__add_parts(parts, entities, source) def __add_parts(self, parts, entities, source): @@ -235,6 +252,10 @@ def __add_parts(self, parts, entities, source): if is_url(id_): instance = Subcrate(self, source=id_, properties=entity) + elif is_url(str(source)): + instance = Subcrate(self, source + id_, id_, properties=entity) + elif source.is_file(): + instance = Subcrate(self, source.parent / unquote(id_), id_, properties=entity) else: instance = Subcrate(self, source=source / unquote(id_), properties=entity) @@ -245,6 +266,10 @@ def __add_parts(self, parts, entities, source): # cls is either a File or a Dataset (Directory) if is_url(id_): instance = cls(self, id_, properties=entity) + elif is_url(str(source)): + instance = cls(self, source + id_, id_, properties=entity) + elif source.is_file(): + instance = cls(self, source.parent / unquote(id_), id_, properties=entity) else: instance = cls(self, source / unquote(id_), id_, properties=entity) self.add(instance) @@ -589,7 +614,7 @@ def _copy_unlisted(self, top, base_path): def write(self, base_path): base_path = Path(base_path) base_path.mkdir(parents=True, exist_ok=True) - if self.source and not isinstance(self.source, dict): + if self.source and not isinstance(self.source, dict) and Path(self.source).is_dir(): self._copy_unlisted(self.source, base_path) for writable_entity in self.data_entities + self.default_entities: writable_entity.write(base_path) @@ -603,6 +628,9 @@ def write_zip(self, out_path): f.write(chunk) return out_path + def write_detached(self, metadata_path): + self.metadata.write_detached(metadata_path) + def stream_zip(self, chunk_size=8192): """ Create a stream of bytes representing the RO-Crate as a ZIP file. """ yield from self._stream_zip(chunk_size=chunk_size) @@ -918,7 +946,11 @@ def _load_subcrate(self): """ if self._crate is None: # load_subcrates=True to load further nested RO-Crate (on-demand / lazily too) - self._crate = ROCrate(self.source, load_subcrates=True) + if subject_of := self.get("subjectOf"): + subcrate_uri = subject_of.id if isinstance(subject_of, Entity) else subject_of + self._crate = ROCrate(subcrate_uri, load_subcrates=True) + else: + self._crate = ROCrate(self.source, load_subcrates=True) def write(self, base_path): super().write(base_path) diff --git a/test/test-data/detached-ro-crate-metadata.json b/test/test-data/detached-ro-crate-metadata.json index 1da64d29..63aefb96 100644 --- a/test/test-data/detached-ro-crate-metadata.json +++ b/test/test-data/detached-ro-crate-metadata.json @@ -26,6 +26,12 @@ }, { "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/test_file_galaxy.txt" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/read_extra/listed.txt" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test_read.py" } ] }, @@ -40,6 +46,14 @@ "name": "Test data", "localPath": "test-data/test_file_galaxy.txt" }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/read_extra/listed.txt", + "@type": "File" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test_read.py", + "@type": "File" + }, { "@id": "http://spdx.org/licenses/CC0-1.0", "@type": "CreativeWork", diff --git a/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json b/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json new file mode 100644 index 00000000..ae514025 --- /dev/null +++ b/test/test-data/detached_crate_with_subcrates/subcrates-ro-crate-metadata.json @@ -0,0 +1,51 @@ +{ + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached_crate_with_subcrates/", + "@type": "Dataset", + "name": "Detached crate with subcrates", + "description": "A detached RO-Crate that references other crates", + "license": "https://spdx.org/licenses/MIT.html", + "datePublished": "2026-02-06", + "hasPart": [ + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/sample_file.txt" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/" + } + ] + }, + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached_crate_with_subcrates/" + }, + "conformsTo": { + "@id": "https://w3id.org/ro/crate/1.2" + } + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/sample_file.txt", + "@type": "File" + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/", + "@type": "Dataset", + "conformsTo": { + "@id": "https://w3id.org/ro/crate" + }, + "subjectOf": { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached-ro-crate-metadata.json" + } + }, + { + "@id": "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/detached-ro-crate-metadata.json", + "@type": "CreativeWork", + "encodingFormat": "application/ld+json", + "sdDatePublished": "2026-02-06" + } + ] +} diff --git a/test/test_metadata.py b/test/test_metadata.py index 85a2558c..a2dd1b8b 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -27,16 +27,14 @@ from rocrate.metadata import find_root_entity_id -@pytest.mark.parametrize("root,basename", [ - ("", "ro-crate-metadata.json"), - ("", "ro-crate-metadata.jsonld"), +@pytest.mark.parametrize("root_id,metadata_id", [ + ("./", "ro-crate-metadata.json"), + ("./", "ro-crate-metadata.jsonld"), ("https://example.org/crate/", "ro-crate-metadata.json"), ("https://example.org/crate/", "ro-crate-metadata.jsonld"), - ("", "bad-name.json"), + ("./", "bad-name.json"), ]) -def test_find_root(root, basename): - metadata_id = root + basename - root_id = root or "./" +def test_find_root(root_id, metadata_id): entities = {_["@id"]: _ for _ in [ { "@id": metadata_id, @@ -52,7 +50,7 @@ def test_find_root(root, basename): "@type": "Dataset", }, ]} - if basename not in {"ro-crate-metadata.json", "ro-crate-metadata.jsonld"}: + if metadata_id not in {"ro-crate-metadata.json", "ro-crate-metadata.jsonld"}: with pytest.raises(KeyError): find_root_entity_id(entities) else: @@ -95,72 +93,6 @@ def test_find_root_bad_entities(): find_root_entity_id(entities) -@pytest.mark.filterwarnings("ignore") -def test_find_root_multiple_entries(): - orig_entities = { - "http://example.org/ro-crate-metadata.json": { - "@id": "http://example.org/ro-crate-metadata.json", - "@type": "CreativeWork", - "about": {"@id": "http://example.org/"}, - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"}, - }, - "http://example.org/": { - "@id": "http://example.org/", - "@type": "Dataset", - "hasPart": [ - {"@id": "http://example.com/"}, - {"@id": "http://example.com/ro-crate-metadata.json"} - ] - }, - "http://example.com/ro-crate-metadata.json": { - "@id": "http://example.com/ro-crate-metadata.json", - "@type": "CreativeWork", - "about": {"@id": "http://example.com/"}, - "conformsTo": {"@id": "https://w3id.com/ro/crate/1.2"}, - }, - "http://example.com/": { - "@id": "http://example.com/", - "@type": "Dataset", - }, - } - - def check_finds_org(entities): - m_id, r_id = find_root_entity_id(entities) - assert m_id == "http://example.org/ro-crate-metadata.json" - assert r_id == "http://example.org/" - - def check_picks_one(entities): - m_id, r_id = find_root_entity_id(entities) - assert m_id in [f"http://example.{_}/ro-crate-metadata.json" for _ in ("org", "com")] - assert r_id in [f"http://example.{_}/" for _ in ("org", "com")] - - check_finds_org(orig_entities) - # no root candidate contains the other one - mod_entities = deepcopy(orig_entities) - del mod_entities["http://example.org/"]["hasPart"] - check_picks_one(mod_entities) - # each root candidate contains the other one - mod_entities = deepcopy(orig_entities) - mod_entities["http://example.com/"]["hasPart"] = [ - {"@id": "http://example.org/"}, - {"@id": "http://example.org/ro-crate-metadata.json"} - ] - check_picks_one(mod_entities) - # "about" does not reference the root entity - mod_entities = deepcopy(orig_entities) - for about in "http://google.com", {"@id": "http://google.com"}: - mod_entities["http://example.com/ro-crate-metadata.json"]["about"] = about - check_finds_org(mod_entities) - # metadata type is not CreativeWork - mod_entities = deepcopy(orig_entities) - mod_entities["http://example.com/ro-crate-metadata.json"]["@type"] = "Thing" - check_finds_org(mod_entities) - # root type is not Dataset - mod_entities = deepcopy(orig_entities) - mod_entities["http://example.com/"]["@type"] = "Thing" - check_finds_org(mod_entities) - - def test_find_root_multiple_types(): entities = {_["@id"]: _ for _ in [ { diff --git a/test/test_read.py b/test/test_read.py index 9055ece3..075f90d0 100644 --- a/test/test_read.py +++ b/test/test_read.py @@ -33,6 +33,7 @@ _URL = ('https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/' 'test/test-data/sample_file.txt') +THIS_DIR = Path(__file__).resolve().parent @pytest.mark.parametrize("gen_preview,from_zip", [(False, False), (True, False), (True, True)]) @@ -240,6 +241,24 @@ def test_crate_with_subcrates(test_data_dir): assert subcrate._crate is nested_crate +def test_detached_crate_with_subcrates(test_data_dir): + main_crate = ROCrate( + test_data_dir / "detached_crate_with_subcrates/subcrates-ro-crate-metadata.json", + load_subcrates=True + ) + subcrate = main_crate.get("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/") + assert isinstance(subcrate, Subcrate) + assert set(main_crate.subcrate_entities) == {subcrate} + assert subcrate.get("conformsTo") == "https://w3id.org/ro/crate" + assert subcrate._crate is None + other_crate = subcrate.get_crate() + assert isinstance(other_crate, ROCrate) + assert subcrate._crate is other_crate + assert other_crate.get( + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/test-data/test_file_galaxy.txt" + ) + + @pytest.mark.parametrize("override", [False, True]) def test_init(test_data_dir, tmpdir, helpers, override): crate_dir = test_data_dir / "ro-crate-galaxy-sortchangecase" @@ -669,6 +688,78 @@ def test_from_dict(tmpdir, version): ROCrate(metadata, init=True) +def test_from_dict_remote_uris(tmpdir): + base_uri = "http://example.com/" + metadata = { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": base_uri}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} + }, + { + "@id": base_uri, + "@type": "Dataset", + "creator": {"@id": "#josiah"}, + "hasPart": {"@id": f"{base_uri}d1"} + }, + { + "@id": f"{base_uri}d1", + "@type": "Dataset", + "hasPart": {"@id": f"{base_uri}d1/d2"} + }, + { + "@id": f"{base_uri}d1/d2", + "@type": "Dataset", + "hasPart": {"@id": f"{base_uri}d1/d2/f1"} + }, + { + "@id": f"{base_uri}d1/d2/f1", + "@type": "File" + }, + { + "@id": "#josiah", + "@type": "Person", + 'name': 'Josiah Carberry' + }, + ] + } + + crate = ROCrate(metadata) + assert crate.root_dataset.id == base_uri + assert crate.metadata.id == "ro-crate-metadata.json" + d1 = crate.dereference(f"{base_uri}d1") + assert d1 + d2 = crate.dereference(f"{base_uri}d1/d2") + assert d2 + f1 = crate.dereference(f"{base_uri}d1/d2/f1") + assert f1 + p = crate.dereference("#josiah") + assert p + assert set(crate.data_entities) == {d1, d2, f1} + assert set(crate.contextual_entities) == {p} + + out_path = tmpdir / 'out_crate' + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + + crate = ROCrate(out_path) + assert crate.root_dataset.id == base_uri + assert crate.metadata.id == "ro-crate-metadata.json" + d1 = crate.dereference(f"{base_uri}d1") + assert d1 + d2 = crate.dereference(f"{base_uri}d1/d2") + assert d2 + f1 = crate.dereference(f"{base_uri}d1/d2/f1") + assert f1 + p = crate.dereference("#josiah") + assert p + assert set(crate.data_entities) == {d1, d2, f1} + assert set(crate.contextual_entities) == {p} + + @pytest.mark.parametrize("version", ["1.1", "1.2"]) def test_no_data_entity_link_from_file(version): metadata = { @@ -834,3 +925,354 @@ def test_not_data_entity_linked(version): assert f1 in crate.contextual_entities else: assert f1 in crate.data_entities + + +@pytest.mark.parametrize("source_base", [ + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", + f"file:///{THIS_DIR}/" # extra slash needed on some windows systems +]) +def test_from_uri(tmpdir, source_base): + source = f"{source_base}test-data/read_crate/ro-crate-metadata.json" + base_uri = f"{source_base}test-data/read_crate/" + # this is an absolute URI in the metadata, note it's outside the crate + remote_f_uri = ("https://raw.githubusercontent.com/ResearchObject/" + "ro-crate-py/master/test/test-data/sample_file.txt") + test_fn = "test_file_galaxy.txt" + crate = ROCrate(source) + assert crate.source == source + assert crate.preview.source == base_uri + "ro-crate-preview.html" + assert crate.preview.id == "ro-crate-preview.html" + assert test_fn in crate + test_f = crate.get(test_fn) + assert test_f.source == base_uri + test_fn + assert test_f.id == test_fn + remote_f = crate.get(remote_f_uri) + assert remote_f.source == remote_f_uri + assert remote_f.id == remote_f_uri + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + # fetch_remote is False by default, so nothing is fetched + assert not (out_path / "ro-crate-preview.html").exists() + assert not (out_path / test_fn).exists() + assert not (out_path / remote_f_uri.rsplit("/", 1)[1]).exists() + rcrate = ROCrate(out_path) + assert rcrate.preview.id == "ro-crate-preview.html" + assert rcrate.get(test_fn) + assert rcrate.get(remote_f_uri) + + +@pytest.mark.parametrize("source_base", [ + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", + f"file:///{THIS_DIR}/" # extra slash needed on some windows systems +]) +def test_from_uri_fetch_remote(tmpdir, source_base): + source = f"{source_base}test-data/read_crate/ro-crate-metadata.json" + base_uri = f"{source_base}test-data/read_crate/" + # this is an absolute URI in the metadata, note it's outside the crate + remote_f_uri = ("https://raw.githubusercontent.com/ResearchObject/" + "ro-crate-py/master/test/test-data/sample_file.txt") + test_fn = "test_file_galaxy.txt" + crate = ROCrate(source) + remote_f = crate.get(remote_f_uri) + test_f = crate.get(test_fn) + # now set some fetch_remote to True + crate.preview.fetch_remote = True + test_f.fetch_remote = True + remote_f.fetch_remote = True + assert remote_f.id == remote_f_uri + # remote_f.id is the full URI. Check that the destination path is + # the output dir / remote_f.id minus the root dataset id. + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "sample_file.txt").is_file() + # check that localPath overrides the default destination + remote_f["localPath"] = "other/sample_file.txt" + shutil.rmtree(out_path) + crate.write(out_path) + assert (out_path / "other" / "sample_file.txt").is_file() + # check more paths + assert (out_path / "ro-crate-preview.html").is_file() + assert (out_path / test_fn).is_file() + # read back the crate + rcrate = ROCrate(out_path) + assert rcrate.preview.id == "ro-crate-preview.html" + rtest_f = rcrate.get(test_fn) + assert rtest_f + assert rtest_f.get("contentUrl") == base_uri + test_fn + rremote_f_uri = rcrate.get(remote_f_uri) + assert rremote_f_uri + assert rremote_f_uri.get("localPath") == "other/sample_file.txt" + + +@pytest.mark.parametrize("source_base", [ + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", + f"file:///{THIS_DIR}/" # extra slash needed on some windows systems +]) +def test_from_uri_detached(tmpdir, source_base): + source = f"{source_base}test-data/detached-ro-crate-metadata.json" + base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/") + crate = ROCrate(source) + assert crate.source == source + assert crate.root_dataset.id == base_uri + sample_file = crate.get(f"{base_uri}sample_file.txt") + assert sample_file.source == f"{base_uri}sample_file.txt" + test_file_galaxy = crate.get(f"{base_uri}test_file_galaxy.txt") + assert test_file_galaxy.source == f"{base_uri}test_file_galaxy.txt" + assert test_file_galaxy.get("localPath") == "test-data/test_file_galaxy.txt" + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert not (out_path / "sample_file.txt").exists() + assert not (out_path / "test_file_galaxy.txt").exists() + # fetch_remote is False, so file is not fetched even if localPath is set + assert not (out_path / "test-data" / "test_file_galaxy.txt").exists() + rcrate = ROCrate(out_path) + assert rcrate.get(f"{base_uri}sample_file.txt") + assert rcrate.get(f"{base_uri}test_file_galaxy.txt") + license = rcrate.get("http://spdx.org/licenses/CC0-1.0") + assert set(rcrate.contextual_entities) == {license} + + +@pytest.mark.parametrize("source_base", [ + "https://raw.githubusercontent.com/ResearchObject/ro-crate-py/master/test/", + f"file:///{THIS_DIR}/" # extra slash needed on some windows systems +]) +def test_from_uri_detached_fetch_remote(tmpdir, source_base): + source = f"{source_base}test-data/detached-ro-crate-metadata.json" + base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/") + crate = ROCrate(source) + test_file_galaxy = crate.get(f"{base_uri}test_file_galaxy.txt") + # now set fetch_remote to True + test_file_galaxy.fetch_remote = True + assert test_file_galaxy.get("localPath") == "test-data/test_file_galaxy.txt" + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "test-data" / "test_file_galaxy.txt").is_file() + rcrate = ROCrate(out_path) + rtest_file_galaxy = rcrate.get(f"{base_uri}test_file_galaxy.txt") + assert rtest_file_galaxy + assert rtest_file_galaxy.get("contentUrl") == f"{base_uri}test_file_galaxy.txt" + assert rtest_file_galaxy.get("localPath") == "test-data/test_file_galaxy.txt" + + +def test_from_file(test_data_dir, tmpdir): + source = test_data_dir / "read_crate" / "ro-crate-metadata.json" + base_path = test_data_dir / "read_crate" + remote_f_uri = ("https://raw.githubusercontent.com/ResearchObject/" + "ro-crate-py/master/test/test-data/sample_file.txt") + test_fn = "test_file_galaxy.txt" + crate = ROCrate(source) + assert crate.source == source + assert crate.preview.source == base_path / "ro-crate-preview.html" + assert crate.preview.id == "ro-crate-preview.html" + assert test_fn in crate + test_f = crate.get(test_fn) + assert test_f.source == base_path / test_fn + assert test_f.id == test_fn + remote_f = crate.get(remote_f_uri) + assert remote_f.source == remote_f_uri + assert remote_f.id == remote_f_uri + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert (out_path / "ro-crate-preview.html").exists() + assert (out_path / test_fn).exists() + assert not (out_path / remote_f_uri.rsplit("/", 1)[1]).exists() + rcrate = ROCrate(out_path) + assert rcrate.preview.id == "ro-crate-preview.html" + assert rcrate.get(test_fn) + assert rcrate.get(remote_f_uri) + assert rcrate.get("test/") + assert not rcrate.get("test/test-metadata.json") + + +def test_from_file_detached(test_data_dir, tmpdir): + source = test_data_dir / "detached-ro-crate-metadata.json" + base_uri = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/") + crate = ROCrate(source) + assert crate.source == source + assert crate.root_dataset.id == base_uri + sample_file = crate.get(f"{base_uri}sample_file.txt") + assert sample_file.source == f"{base_uri}sample_file.txt" + test_file_galaxy = crate.get(f"{base_uri}test_file_galaxy.txt") + assert test_file_galaxy.source == f"{base_uri}test_file_galaxy.txt" + listed = crate.get(f"{base_uri}read_extra/listed.txt") + assert listed.source == f"{base_uri}read_extra/listed.txt" + test_read_base = ("https://raw.githubusercontent.com/ResearchObject/" + "ro-crate-py/master/test/") + test_read = crate.get(f"{test_read_base}test_read.py") + assert test_read.source == f"{test_read_base}test_read.py" + out_path = tmpdir / "out_crate" + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + rcrate = ROCrate(out_path) + assert rcrate.get(f"{base_uri}sample_file.txt") + assert rcrate.get(f"{base_uri}test_file_galaxy.txt") + assert rcrate.get(f"{base_uri}read_extra/listed.txt") + assert rcrate.get(f"{test_read_base}test_read.py") + + # set fetch_remote to True and check local paths + for e in crate.data_entities: + e.fetch_remote = True + shutil.rmtree(out_path) + crate.write(out_path) + assert (out_path / "sample_file.txt").is_file() + assert (out_path / "test-data" / "test_file_galaxy.txt").is_file() + assert (out_path / "read_extra" / "listed.txt").is_file() + assert (out_path / "test_read.py").is_file() + + +def test_read_remote_dir(tmpdir): + base_uri = "https://ftp.mozilla.org/pub/" + metadata = { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": base_uri}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} + }, + { + "@id": base_uri, + "@type": "Dataset", + "datePublished": "2026-02-19", + "hasPart": [{"@id": f"{base_uri}misc/errorpages/"}] + }, + { + "@id": f"{base_uri}misc/errorpages/", + "@type": "Dataset", + "hasPart": [ + {"@id": f"{base_uri}misc/errorpages/404.html"}, + {"@id": f"{base_uri}misc/errorpages/500.html"} + ] + }, + { + "@id": f"{base_uri}misc/errorpages/404.html", + "@type": "File" + }, + { + "@id": f"{base_uri}misc/errorpages/500.html", + "@type": "File" + } + ] + } + crate = ROCrate(metadata) + errorpages = crate.get(f"{base_uri}misc/errorpages/") + errorpages.fetch_remote = True + out_path = tmpdir / 'out_crate' + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert (out_path / "misc" / "errorpages" / "404.html").is_file() + assert (out_path / "misc" / "errorpages" / "500.html").is_file() + + +def test_read_remote_dir_local_path(tmpdir): + base_uri = "https://ftp.mozilla.org/pub/" + metadata = { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": base_uri}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} + }, + { + "@id": base_uri, + "@type": "Dataset", + "datePublished": "2026-02-19", + "hasPart": [{"@id": f"{base_uri}misc/errorpages/"}] + }, + { + "@id": f"{base_uri}misc/errorpages/", + "@type": "Dataset", + "localPath": "errorpages/", + "hasPart": [ + {"@id": f"{base_uri}misc/errorpages/404.html"}, + {"@id": f"{base_uri}misc/errorpages/500.html"} + ] + }, + { + "@id": f"{base_uri}misc/errorpages/404.html", + "@type": "File" + }, + { + "@id": f"{base_uri}misc/errorpages/500.html", + "@type": "File", + "localPath": "errorpages/500/500.html" + } + ] + } + crate = ROCrate(metadata) + errorpages = crate.get(f"{base_uri}misc/errorpages/") + errorpages.fetch_remote = True + out_path = tmpdir / 'out_crate' + crate.write(out_path) + assert (out_path / "ro-crate-metadata.json").is_file() + assert not (out_path / "misc").exists() + assert (out_path / "errorpages" / "404.html").is_file() + assert not (out_path / "errorpages" / "500.html").exists() + assert (out_path / "errorpages" / "500" / "500.html").is_file() + + +def test_read_remote_dir_skip_dir_part(tmpdir): + base_uri = "https://ftp.mozilla.org/pub/" + metadata = { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": base_uri}, + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"} + }, + { + "@id": base_uri, + "@type": "Dataset", + "datePublished": "2026-02-19", + "hasPart": [{"@id": f"{base_uri}misc/"}] + }, + { + "@id": f"{base_uri}misc/", + "@type": "Dataset", + "hasPart": [ + # part is dir, should not try to download anything + {"@id": f"{base_uri}misc/errorpages/"}, + ] + }, + { + "@id": f"{base_uri}misc/errorpages/", + "@type": "Dataset", + "hasPart": [ + {"@id": f"{base_uri}misc/errorpages/404.html"}, + {"@id": f"{base_uri}misc/errorpages/500.html"} + ] + }, + { + "@id": f"{base_uri}misc/errorpages/404.html", + "@type": "File" + }, + { + "@id": f"{base_uri}misc/errorpages/500.html", + "@type": "File" + } + ] + } + crate = ROCrate(metadata) + misc = crate.get(f"{base_uri}misc/") + misc.fetch_remote = True + errorpages = crate.get(f"{base_uri}misc/errorpages/") + errorpages.fetch_remote = True + out_path = tmpdir / 'out_crate' + crate.write(out_path) + + assert (out_path / "ro-crate-metadata.json").is_file() + assert (out_path / "misc").is_dir() + assert (out_path / "misc" / "errorpages").is_dir() + assert (out_path / "misc" / "errorpages" / "404.html").is_file() + assert (out_path / "misc" / "errorpages" / "500.html").is_file() diff --git a/test/test_write.py b/test/test_write.py index bb60b7be..0f3f5bee 100644 --- a/test/test_write.py +++ b/test/test_write.py @@ -178,6 +178,59 @@ def test_remote_uri(tmpdir, helpers, fetch_remote, validate_url, to_zip): assert "sdDatePublished" in props +def test_local_path(test_data_dir, tmpdir): + crate = ROCrate() + url = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/sample_file.txt") + sample_file = crate.add_file(url) + assert sample_file.id == url + sample_file["localPath"] = "test-data/sample_file.txt" + sample_file.fetch_remote = True + test_file_galaxy = crate.add_file(test_data_dir / "test_file_galaxy.txt") + assert test_file_galaxy.id == "test_file_galaxy.txt" + test_file_galaxy["localPath"] = "foo/bar.txt" + out_path = tmpdir / "ro_crate_out" + crate.write(out_path) + assert (out_path / "test-data" / "sample_file.txt").is_file() + assert not (out_path / "sample_file.txt").exists() + assert not (out_path / "foo" / "bar.txt").exists() + assert (out_path / "test_file_galaxy.txt").is_file() + + +def test_local_path_vs_relative_id(test_data_dir, tmpdir): + crate = ROCrate() + url = ("https://raw.githubusercontent.com/ResearchObject/ro-crate-py/" + "master/test/test-data/sample_file.txt") + sample_file = crate.add_file(url, dest_path="examples/sample_file.txt") + assert sample_file.id == "examples/sample_file.txt" + sample_file["localPath"] = "test-data/sample_file.txt" + sample_file.fetch_remote = True + out_path = tmpdir / "ro_crate_out" + crate.write(out_path) + assert (out_path / "examples" / "sample_file.txt").is_file() + assert not (out_path / "test-data" / "sample_file.txt").exists() + + +def test_local_path_vs_relative_id_dataset(test_data_dir, tmpdir): + crate = ROCrate() + f1 = crate.add_file("https://ftp.mozilla.org/pub/misc/errorpages/404.html") + f2 = crate.add_file("https://ftp.mozilla.org/pub/misc/errorpages/500.html") + dataset = crate.add_dataset( + "https://ftp.mozilla.org/pub/misc/errorpages/", + dest_path="errorpages/", + fetch_remote=True, + ) + assert dataset.id == "errorpages/" + dataset["hasPart"] = [f1, f2] + dataset["localPath"] = "mozilla_error_pages" + out_path = tmpdir / "ro_crate_out" + crate.write(out_path) + assert (out_path / "errorpages").is_dir() + assert (out_path / "errorpages" / "404.html").is_file() + assert (out_path / "errorpages" / "500.html").is_file() + assert not (out_path / "mozilla_error_pages").exists() + + def test_file_uri(tmpdir): f_name = uuid.uuid4().hex f_path = (tmpdir / f_name).resolve() @@ -263,8 +316,8 @@ def test_remote_dir(tmpdir, helpers, fetch_remote, validate_url): relpath = "pub/misc/errorpages/" properties = { "hasPart": [ - {"@id": "404.html"}, - {"@id": "500.html"}, + {"@id": "https://ftp.mozilla.org/pub/misc/errorpages/404.html"}, + {"@id": "https://ftp.mozilla.org/pub/misc/errorpages/500.html"}, ], } kw = { @@ -287,7 +340,8 @@ def test_remote_dir(tmpdir, helpers, fetch_remote, validate_url): out_dataset = out_crate.dereference(relpath) assert (out_path / relpath).is_dir() for entry in properties["hasPart"]: - assert (out_path / relpath / entry["@id"]).is_file() + basename = entry["@id"].rsplit("/", 1)[-1] + assert (out_path / relpath / basename).is_file() else: out_dataset = out_crate.dereference(url) assert not (out_path / relpath).exists() @@ -719,6 +773,75 @@ def test_write_version(tmpdir, helpers, version): assert data["@context"] == f"https://w3id.org/ro/crate/{version}/context" +@pytest.mark.parametrize("to_zip", [False, True]) +def test_detached_creation(tmpdir, to_zip): + base_uri = "http://example.com/crate/" + orcid = "https://orcid.org/0000-0002-1825-0097" + name = "Josiah Carberry" + crate = ROCrate(root_dataset_id=base_uri) + assert crate.source is None + assert crate.metadata.source is None + assert crate.root_dataset.id == base_uri + assert crate.metadata.id == "ro-crate-metadata.json" + assert crate.metadata["about"] is crate.root_dataset + crate.add_dataset(f"{base_uri}d1") + crate.add_file(f"{base_uri}f1") + p = crate.add(Person(crate, orcid, properties={"name": name})) + crate.root_dataset["creator"] = p + + out_path = tmpdir / "ro_crate_out" + if to_zip: + zip_path = tmpdir / 'ro_crate_out.zip' + crate.write_zip(zip_path) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(out_path) + else: + crate.write(out_path) + + assert (out_path / "ro-crate-metadata.json").is_file() + rcrate = ROCrate(out_path) + # this crate is attached, even though all its data entities are web-based + assert rcrate.source == out_path + assert rcrate.metadata.source == "ro-crate-metadata.json" + assert rcrate.root_dataset.id == base_uri + assert rcrate.metadata.id == "ro-crate-metadata.json" + assert rcrate.metadata["about"] is rcrate.root_dataset + assert rcrate.get(f"{base_uri}d1") + assert rcrate.get(f"{base_uri}f1") + rp = rcrate.get(orcid) + assert rp["name"] == name + + +def test_detached_creation_write_detached(tmpdir): + base_uri = "http://example.com/crate/" + orcid = "https://orcid.org/0000-0002-1825-0097" + name = "Josiah Carberry" + crate = ROCrate(root_dataset_id=base_uri) + crate.add_dataset(f"{base_uri}d1") + crate.add_file(f"{base_uri}f1") + crate.add(Person(crate, orcid, properties={"name": name})) + detached_md_path = tmpdir / "example-ro-crate-metadata.json" + crate.write_detached(detached_md_path) + assert detached_md_path.is_file() + rcrate = ROCrate(detached_md_path) + assert rcrate.source == detached_md_path + assert rcrate.metadata.source == "ro-crate-metadata.json" + assert rcrate.root_dataset.id == base_uri + assert rcrate.metadata.id == "ro-crate-metadata.json" + assert rcrate.metadata["about"] is rcrate.root_dataset + assert rcrate.get(f"{base_uri}d1") + assert rcrate.get(f"{base_uri}f1") + rp = rcrate.get(orcid) + assert rp["name"] == name + + +def test_detached_creation_exceptions(): + with pytest.raises(ValueError): + ROCrate(root_dataset_id="foo/bar") + with pytest.raises(ValueError): + ROCrate(root_dataset_id="/foo/bar") + + def test_metadata_utf8_encoding(tmpdir, helpers): crate = ROCrate() crate_name = 'Test crate with non-ASCII characters'