From 00525cbec44ae63a2bd5b21eb9b8b74bf416b6b2 Mon Sep 17 00:00:00 2001 From: saipraneeth <2506664+msaipraneeth@users.noreply.github.com> Date: Mon, 7 Sep 2026 07:11:17 +0100 Subject: [PATCH 1/3] Fix(CMEM-8068): preserve non-ASCII characters in JSON output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit json.dumps() defaults to ensure_ascii=True, which escaped non-ASCII characters (e.g. ö) into unicode escape sequences when valid JSON objects were written to the target JSON dataset. Also fixed test_source_and_target_dataset, which exercises this exact code path: it was never collected by pytest because its name didn't start with test_ (validate_test_... instead of test_...) and it had no @needs_cmem marker, unlike its siblings. --- CHANGELOG.md | 5 + .../validate_entities/task.py | 4 +- tests/test_validate_entities.py | 96 ++++++++++++++++++- 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9243cb4..eee4ba4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p - updated dependencies and template +### Fixed + +- Validate Entities task no longer converts non-ASCII characters to unicode escape sequences + when writing valid JSON objects to the target dataset + ## [1.3.0] 2026-08-19 ### Changed diff --git a/cmem_plugin_validation/validate_entities/task.py b/cmem_plugin_validation/validate_entities/task.py index 9661a14..8e2c83e 100644 --- a/cmem_plugin_validation/validate_entities/task.py +++ b/cmem_plugin_validation/validate_entities/task.py @@ -286,7 +286,9 @@ def execute( Client.from_context(context=context).datasets.post_file_resource( project_id=context.task.project_id(), dataset_id=self.target_dataset, - file_resource=io.BytesIO(json.dumps(valid_json_objects).encode("utf-8")), + file_resource=io.BytesIO( + json.dumps(valid_json_objects, ensure_ascii=False).encode("utf-8") + ), ) return None diff --git a/tests/test_validate_entities.py b/tests/test_validate_entities.py index 633260b..cb77fde 100644 --- a/tests/test_validate_entities.py +++ b/tests/test_validate_entities.py @@ -5,12 +5,16 @@ from dataclasses import dataclass from os import environ from pathlib import Path +from types import SimpleNamespace +from typing import IO, Any import pytest from cmem_client.client import Client from cmem_client.models.dataset import Dataset, DatasetData, DatasetMetadata from cmem_client.models.project import Project from cmem_client.repositories.protocols.import_item import ImportConflictPolicy +from cmem_plugin_base.dataintegration.context import ExecutionContext, ReportContext +from cmem_plugin_base.dataintegration.entity import Entities, Entity, EntityPath, EntitySchema from cmem_plugin_base.testing import TestExecutionContext from cmem_plugin_validation.validate_entities.task import SOURCE, TARGET, ValidateEntity @@ -138,7 +142,8 @@ def test_execute_with_source_dataset(project: TestSetup) -> None: assert len(list(entities.entities)) == 1 -def validate_test_source_target_dataset(project: TestSetup) -> None: +@needs_cmem +def test_source_and_target_dataset(project: TestSetup) -> None: """Test source and target dataset mode""" _ = project @@ -154,3 +159,92 @@ def validate_test_source_target_dataset(project: TestSetup) -> None: client = get_client(_.project_name) data = json.loads(client.files.read(f"{_.project_name}:{_.target_dataset_file}")) assert len(data) == _.valid_source_object_count + + +class _FakeDatasetItem: + """Stand-in for the dataset item cmem_client.datasets.get_item() returns""" + + def __init__(self, file_name: str) -> None: + self.data = SimpleNamespace(parameters={"file": file_name}) + + +class _FakeDatasets: + """Stand-in for cmem_client.client.Client.datasets""" + + def __init__(self, schema_file_name: str, written: dict[str, bytes]) -> None: + self._schema_file_name = schema_file_name + self._written = written + + def get_item(self, project_id: str, dataset_id: str) -> _FakeDatasetItem: + """Return the schema dataset's file name, the only lookup task.py performs""" + return _FakeDatasetItem(self._schema_file_name) + + def post_file_resource( + self, project_id: str, dataset_id: str, file_resource: IO[bytes] + ) -> None: + """Record the raw bytes written to the target dataset instead of uploading them""" + _ = project_id, dataset_id + self._written["content"] = file_resource.read() + + +class _FakeFiles: + """Stand-in for cmem_client.client.Client.files""" + + def __init__(self, schema_bytes: bytes) -> None: + self._schema_bytes = schema_bytes + + def read(self, key: str) -> bytes: + """Return the JSON schema content, the only file this task reads""" + _ = key + return self._schema_bytes + + +class _FakeClient: + """Stand-in for cmem_client.client.Client, avoiding a real Corporate Memory connection""" + + def __init__(self, schema_bytes: bytes, schema_file_name: str, written: dict[str, bytes]): + self.datasets = _FakeDatasets(schema_file_name, written) + self.files = _FakeFiles(schema_bytes) + + +class _StubExecutionContext(ExecutionContext): + """An execution context which needs no Corporate Memory connection. + + task.py only reads ``context.task.project_id()`` and calls + ``context.report.update()`` - it never touches ``context.user``. + """ + + def __init__(self, project_id: str) -> None: + self.report = ReportContext() + self.task = SimpleNamespace(project_id=lambda: project_id) + + +def test_target_dataset_keeps_unicode_characters(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that non-ASCII characters in entity values are not escaped in the target dataset + + Uses a fake Client so this runs without a Corporate Memory connection: task.py only + calls Client.datasets.get_item() and Client.files.read() to resolve the JSON schema, + and Client.datasets.post_file_resource() to write the target dataset. + """ + schema = json.dumps({"type": "object", "properties": {"city": {"type": "string"}}}).encode() + written: dict[str, Any] = {} + monkeypatch.setattr( + "cmem_plugin_validation.validate_entities.task.Client.from_context", + lambda context: _FakeClient(schema, "schema.json", written), + ) + + entities = Entities( + entities=[Entity(uri="urn:x-1", values=[["Köln"]])], + schema=EntitySchema(type_uri="", paths=[EntityPath(path="city", is_single_value=True)]), + ) + ValidateEntity( + source_mode=SOURCE.entities, + target_mode=TARGET.dataset, + json_schema_dataset="schema_dataset", + fail_on_violations=True, + target_dataset="target_dataset", + ).execute([entities], _StubExecutionContext(project_id="validate_entities_unit_test")) + + raw_content = written["content"].decode("utf-8") + assert "\\u00f6" not in raw_content + assert json.loads(raw_content) == [{"city": "Köln"}] From b517f1fd62352eb9922f03fdb921dadbebf9f03a Mon Sep 17 00:00:00 2001 From: saipraneeth <2506664+msaipraneeth@users.noreply.github.com> Date: Mon, 7 Sep 2026 09:55:49 +0100 Subject: [PATCH 2/3] Replace fake-Client unit test with a live-CMEM regression test A fully mocked Client can't exercise the real upload path, so it wouldn't have caught issues like the io.StringIO/io.BytesIO mismatch just found in cmem-plugin-graphql's equivalent code. Use a dedicated non-ASCII fixture dataset and the existing source-dataset test flow instead, checking the raw target file bytes for escape sequences. --- tests/fixtures/source.unicode.json | 1 + tests/test_validate_entities.py | 106 ++++++----------------------- 2 files changed, 21 insertions(+), 86 deletions(-) create mode 100644 tests/fixtures/source.unicode.json diff --git a/tests/fixtures/source.unicode.json b/tests/fixtures/source.unicode.json new file mode 100644 index 0000000..8dacb9c --- /dev/null +++ b/tests/fixtures/source.unicode.json @@ -0,0 +1 @@ +[{"name" : "Käse", "price" : 5.5}, {"name" : "Müsli", "price" : 3.2}] diff --git a/tests/test_validate_entities.py b/tests/test_validate_entities.py index cb77fde..76e34bf 100644 --- a/tests/test_validate_entities.py +++ b/tests/test_validate_entities.py @@ -5,16 +5,12 @@ from dataclasses import dataclass from os import environ from pathlib import Path -from types import SimpleNamespace -from typing import IO, Any import pytest from cmem_client.client import Client from cmem_client.models.dataset import Dataset, DatasetData, DatasetMetadata from cmem_client.models.project import Project from cmem_client.repositories.protocols.import_item import ImportConflictPolicy -from cmem_plugin_base.dataintegration.context import ExecutionContext, ReportContext -from cmem_plugin_base.dataintegration.entity import Entities, Entity, EntityPath, EntitySchema from cmem_plugin_base.testing import TestExecutionContext from cmem_plugin_validation.validate_entities.task import SOURCE, TARGET, ValidateEntity @@ -29,8 +25,10 @@ class TestSetup: schema_dataset: str = "schema_dataset" valid_source_dataset_file: Path = FIXTURE_DIR / "source.valid.json" invalid_source_dataset_file: Path = FIXTURE_DIR / "source.invalid.json" + unicode_source_dataset_file: Path = FIXTURE_DIR / "source.unicode.json" valid_source_dataset: str = "valid_source_dataset" invalid_source_dataset: str = "invalid_source_dataset" + unicode_source_dataset: str = "unicode_source_dataset" target_dataset_file: str = "target.json" target_dataset: str = "target_dataset" project_name: str = "validate_entities_test_project" @@ -72,6 +70,7 @@ def project() -> Generator[TestSetup]: for dataset_name, dataset_file in ( (_.valid_source_dataset, _.valid_source_dataset_file), (_.invalid_source_dataset, _.invalid_source_dataset_file), + (_.unicode_source_dataset, _.unicode_source_dataset_file), (_.schema_dataset, _.schema_dataset_file), ): _make_dataset(client, _.project_name, dataset_name, dataset_file.name) @@ -161,90 +160,25 @@ def test_source_and_target_dataset(project: TestSetup) -> None: assert len(data) == _.valid_source_object_count -class _FakeDatasetItem: - """Stand-in for the dataset item cmem_client.datasets.get_item() returns""" - - def __init__(self, file_name: str) -> None: - self.data = SimpleNamespace(parameters={"file": file_name}) - - -class _FakeDatasets: - """Stand-in for cmem_client.client.Client.datasets""" - - def __init__(self, schema_file_name: str, written: dict[str, bytes]) -> None: - self._schema_file_name = schema_file_name - self._written = written - - def get_item(self, project_id: str, dataset_id: str) -> _FakeDatasetItem: - """Return the schema dataset's file name, the only lookup task.py performs""" - return _FakeDatasetItem(self._schema_file_name) - - def post_file_resource( - self, project_id: str, dataset_id: str, file_resource: IO[bytes] - ) -> None: - """Record the raw bytes written to the target dataset instead of uploading them""" - _ = project_id, dataset_id - self._written["content"] = file_resource.read() - - -class _FakeFiles: - """Stand-in for cmem_client.client.Client.files""" - - def __init__(self, schema_bytes: bytes) -> None: - self._schema_bytes = schema_bytes - - def read(self, key: str) -> bytes: - """Return the JSON schema content, the only file this task reads""" - _ = key - return self._schema_bytes - - -class _FakeClient: - """Stand-in for cmem_client.client.Client, avoiding a real Corporate Memory connection""" - - def __init__(self, schema_bytes: bytes, schema_file_name: str, written: dict[str, bytes]): - self.datasets = _FakeDatasets(schema_file_name, written) - self.files = _FakeFiles(schema_bytes) - - -class _StubExecutionContext(ExecutionContext): - """An execution context which needs no Corporate Memory connection. - - task.py only reads ``context.task.project_id()`` and calls - ``context.report.update()`` - it never touches ``context.user``. - """ - - def __init__(self, project_id: str) -> None: - self.report = ReportContext() - self.task = SimpleNamespace(project_id=lambda: project_id) - - -def test_target_dataset_keeps_unicode_characters(monkeypatch: pytest.MonkeyPatch) -> None: - """Test that non-ASCII characters in entity values are not escaped in the target dataset - - Uses a fake Client so this runs without a Corporate Memory connection: task.py only - calls Client.datasets.get_item() and Client.files.read() to resolve the JSON schema, - and Client.datasets.post_file_resource() to write the target dataset. - """ - schema = json.dumps({"type": "object", "properties": {"city": {"type": "string"}}}).encode() - written: dict[str, Any] = {} - monkeypatch.setattr( - "cmem_plugin_validation.validate_entities.task.Client.from_context", - lambda context: _FakeClient(schema, "schema.json", written), - ) +@needs_cmem +def test_target_dataset_keeps_unicode_characters(project: TestSetup) -> None: + """Test that non-ASCII characters in the source data are not escaped in the target dataset""" + _ = project - entities = Entities( - entities=[Entity(uri="urn:x-1", values=[["Köln"]])], - schema=EntitySchema(type_uri="", paths=[EntityPath(path="city", is_single_value=True)]), - ) ValidateEntity( - source_mode=SOURCE.entities, + source_mode=SOURCE.dataset, target_mode=TARGET.dataset, - json_schema_dataset="schema_dataset", + json_schema_dataset=_.schema_dataset, fail_on_violations=True, - target_dataset="target_dataset", - ).execute([entities], _StubExecutionContext(project_id="validate_entities_unit_test")) + source_dataset=_.unicode_source_dataset, + target_dataset=_.target_dataset, + ).execute([], TestExecutionContext(project_id=_.project_name)) - raw_content = written["content"].decode("utf-8") - assert "\\u00f6" not in raw_content - assert json.loads(raw_content) == [{"city": "Köln"}] + client = get_client(_.project_name) + raw_content = client.files.read(f"{_.project_name}:{_.target_dataset_file}").decode("utf-8") + assert "\\u00e4" not in raw_content # ä + assert "\\u00fc" not in raw_content # ü + assert json.loads(raw_content) == [ + {"name": "Käse", "price": 5.5}, + {"name": "Müsli", "price": 3.2}, + ] From 8bdeb5fcc0cc1a5a573feb550b43cca2b8166cbe Mon Sep 17 00:00:00 2001 From: saipraneeth <2506664+msaipraneeth@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:30:46 +0100 Subject: [PATCH 3/3] docs: fix and clarify plugin documentation and parameter descriptions Corrects fail_on_violations descriptions that claimed early termination on the first violation, when both tasks actually validate everything before failing the report. Fills in missing descriptions for boolean parameters in validate_graph, fixes grammar bugs in the validate_entities choice labels, and rewrites the Input/Output Modes sections to describe port shape instead of restating the parameters. --- CHANGELOG.md | 2 + .../validate_entities/task.py | 50 +++++++++---------- cmem_plugin_validation/validate_graph/task.py | 29 +++++++++-- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eee4ba4..7637493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p ### Changed - updated dependencies and template +- corrected and clarified the documentation and parameter descriptions of the + `Validate Entities` and `Validate Knowledge Graph` plugins ### Fixed diff --git a/cmem_plugin_validation/validate_entities/task.py b/cmem_plugin_validation/validate_entities/task.py index 8e2c83e..46e3dc9 100644 --- a/cmem_plugin_validation/validate_entities/task.py +++ b/cmem_plugin_validation/validate_entities/task.py @@ -39,32 +39,26 @@ ### Input Modes -The plugin supports two input modes for validation: - -1. **Validate Entities**: Validates entities received from the input port in the workflow. -2. **Validate JSON Dataset**: Validates a JSON dataset stored in the project. - - If the JSON dataset is a JSON array, the schema will validate each object inside the array. - - If the JSON dataset is a JSON object, it will be validated against the schema directly. - -Validated data objects can be sent to an output port for further processing in the workflow -or saved in a JSON dataset in the project. +Entities arrive on the input port by default. Switching the input mode to a JSON dataset +removes the input port, and the resources to validate are instead read from a JSON dataset in +the project. A JSON array in that dataset has each of its objects validated individually; a +JSON object is validated directly. ### Output Modes -1. **Valid JSON objects sent to Output Port**: Valid JSON objects can be sent as entities - to the output port. -2. **Saved in JSON Dataset**: Valid JSON objects can be stored in a specified JSON dataset - in the project. +Valid JSON objects are sent to an output port by default, ready for further processing in the +workflow. Switching the output mode to a JSON dataset removes the output port, and the valid +JSON objects are instead saved to a JSON dataset in the project, replacing its existing content. ### Error Handling -The task can either: +Every entity or object is validated regardless of the outcome. The task can then either: -- Fail instantly if there is a data violation, halting the workflow. -- Provide warnings in the workflow report, allowing follow-up tasks to run based on the +- Fail once validation completes if any entity has violations, halting the workflow. +- Report violations only as warnings, allowing follow-up tasks to run based on the validated data. -The error handling behavior is configurable through the `Fail on violations` parameter. +The error handling behavior is configurable through the **Fail on violations** parameter. """ @@ -88,9 +82,9 @@ TARGET.options = OrderedDict( { TARGET.dataset: f"{TARGET.dataset}: " - "Valid JSON objects will be is saved in a JSON dataset (see advanced options).", + "Valid JSON objects are saved in a JSON dataset (see advanced options).", TARGET.entities: f"{TARGET.entities}: " - "Valid JSON objects will be send as entities to the output port.", + "Valid JSON objects are sent as entities to the output port.", } ) @@ -105,21 +99,24 @@ PluginParameter( name="source_mode", label="Source / Input Mode", - description="", + description="Selects where entities to validate come from: the input port or" + " a JSON dataset.", param_type=ChoiceParameterType(SOURCE.options), default_value=SOURCE.entities, ), PluginParameter( name="target_mode", label="Target / Output Mode", - description="", + description="Selects where valid JSON objects are written to: the output port" + " or a JSON dataset.", param_type=ChoiceParameterType(TARGET.options), default_value=TARGET.entities, ), PluginParameter( name="source_dataset", label="Source JSON Dataset", - description="This dataset holds the resources you want to validate.", + description="This dataset holds the resources you want to validate. Required when" + " Source / Input Mode is set to dataset; leave it empty when using entities.", param_type=DatasetParameterType(dataset_type="json"), advanced=True, default_value="", @@ -127,8 +124,9 @@ PluginParameter( name="target_dataset", label="Target JSON Dataset", - description="This dataset will be used to store the valid JSON objects" - " after validation.", + description="This dataset stores the valid JSON objects after validation," + " replacing any existing content. Required when Target / Output Mode is set to" + " dataset; leave it empty when using entities.", param_type=DatasetParameterType(dataset_type="json"), default_value="", advanced=True, @@ -142,7 +140,9 @@ PluginParameter( name="fail_on_violations", label="Fail on violations", - description="If enabled, the task will fail on the first data violation.", + description="If enabled, the workflow fails once validation completes if any" + " entity has violations. All entities are validated either way; disabling this" + " instead reports the violations as warnings.", default_value=DEFAULT_FAIL_ON_VIOLATION, ), ], diff --git a/cmem_plugin_validation/validate_graph/task.py b/cmem_plugin_validation/validate_graph/task.py index 40fd2e4..943cecf 100644 --- a/cmem_plugin_validation/validate_graph/task.py +++ b/cmem_plugin_validation/validate_graph/task.py @@ -27,8 +27,24 @@ from cmem_plugin_validation.validate_graph.state import State DOCUMENTATION = """ -Start a graph validation process which verifies, that resources in a specific graph are valid -according to the node shapes in a shape catalog graph. +Starts a graph validation process which verifies that resources in a specific graph are valid +according to the node shapes in a shape catalog graph. The task waits for the validation to +finish before completing. + +The task has no input port. Violations found during validation are always summarized in the +workflow report, and can also be materialized into a result graph in the project. Sending each +violation as an entity to an output port is optional; when that is disabled, the task has no +output port either. + +### Error Handling + +Every resource is validated regardless of the outcome. The task can then either: + +- Fail once validation completes if any resource has violations, halting the workflow. +- Report violations only as warnings, allowing follow-up tasks to run based on the results. + +The error handling behavior is configurable through the **Fail workflow on violations** +parameter. """ DEFAULT_SHAPE_GRAPH = "https://vocab.eccenca.com/shacl/" @@ -85,23 +101,30 @@ PluginParameter( name="clear_result_graph", label="Clear result graph before validation", + description="If enabled, the existing content of the result graph is deleted" + " before validation starts. Has no effect when Result graph is left empty.", default_value=DEFAULT_CLEAR_RESULT_GRAPH, ), PluginParameter( name="fail_on_violations", label="Fail workflow on violations", + description="If enabled, the workflow fails once validation completes if any" + " resource has violations. All resources are validated either way; disabling this" + " instead reports the violations as warnings.", default_value=DEFAULT_FAIL_ON_VIOLATION, ), PluginParameter( name="output_results", label="Output violations as entities", + description="If enabled, each violation is sent as an entity to an output port" + " for further processing in the workflow. Disabling this removes the output port.", default_value=DEFAULT_OUTPUT_RESULTS, ), PluginParameter( name="sparql_query", label="Resource Selection Query", description="The query to select the resources to validate. " - "Use {{context_graph}} as a placeholder for the select context graph for validation.", + "Use `{{context_graph}}` as a placeholder for the selected Context Graph.", default_value=DEFAULT_SPARQL_QUERY, advanced=True, ),