Fix validation plugin converting characters to Unicode escape sequences (CMEM-8068)

CHANGELOG.md

@@ -10,6 +10,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p
 ### Changed
 
 - updated dependencies and template
+- corrected and clarified the documentation and parameter descriptions of the
+  `Validate Entities` and `Validate Knowledge Graph` plugins
+
+### Fixed
+
+- Validate Entities task no longer converts non-ASCII characters to unicode escape sequences
+  when writing valid JSON objects to the target dataset
 
 ## [1.3.0] 2026-08-19
 

cmem_plugin_validation/validate_entities/task.py

@@ -39,32 +39,26 @@
 
 ### Input Modes
 
-The plugin supports two input modes for validation:
-
-1. **Validate Entities**: Validates entities received from the input port in the workflow.
-2. **Validate JSON Dataset**: Validates a JSON dataset stored in the project.
-   - If the JSON dataset is a JSON array, the schema will validate each object inside the array.
-   - If the JSON dataset is a JSON object, it will be validated against the schema directly.
-
-Validated data objects can be sent to an output port for further processing in the workflow
-or saved in a JSON dataset in the project.
+Entities arrive on the input port by default. Switching the input mode to a JSON dataset
+removes the input port, and the resources to validate are instead read from a JSON dataset in
+the project. A JSON array in that dataset has each of its objects validated individually; a
+JSON object is validated directly.
 
 ### Output Modes
 
-1. **Valid JSON objects sent to Output Port**: Valid JSON objects can be sent as entities
-   to the output port.
-2. **Saved in JSON Dataset**: Valid JSON objects can be stored in a specified JSON dataset
-   in the project.
+Valid JSON objects are sent to an output port by default, ready for further processing in the
+workflow. Switching the output mode to a JSON dataset removes the output port, and the valid
+JSON objects are instead saved to a JSON dataset in the project, replacing its existing content.
 
 ### Error Handling
 
-The task can either:
+Every entity or object is validated regardless of the outcome. The task can then either:
 
-- Fail instantly if there is a data violation, halting the workflow.
-- Provide warnings in the workflow report, allowing follow-up tasks to run based on the
+- Fail once validation completes if any entity has violations, halting the workflow.
+- Report violations only as warnings, allowing follow-up tasks to run based on the
   validated data.
 
-The error handling behavior is configurable through the `Fail on violations` parameter.
+The error handling behavior is configurable through the **Fail on violations** parameter.
 """
 
 
@@ -88,9 +82,9 @@
 TARGET.options = OrderedDict(
     {
         TARGET.dataset: f"{TARGET.dataset}: "
-        "Valid JSON objects will be is saved in a JSON dataset (see advanced options).",
+        "Valid JSON objects are saved in a JSON dataset (see advanced options).",
         TARGET.entities: f"{TARGET.entities}: "
-        "Valid JSON objects will be send as entities to the output port.",
+        "Valid JSON objects are sent as entities to the output port.",
     }
 )
 
@@ -105,30 +99,34 @@
         PluginParameter(
             name="source_mode",
             label="Source / Input Mode",
-            description="",
+            description="Selects where entities to validate come from: the input port or"
+            " a JSON dataset.",
             param_type=ChoiceParameterType(SOURCE.options),
             default_value=SOURCE.entities,
         ),
         PluginParameter(
             name="target_mode",
             label="Target / Output Mode",
-            description="",
+            description="Selects where valid JSON objects are written to: the output port"
+            " or a JSON dataset.",
             param_type=ChoiceParameterType(TARGET.options),
             default_value=TARGET.entities,
         ),
         PluginParameter(
             name="source_dataset",
             label="Source JSON Dataset",
-            description="This dataset holds the resources you want to validate.",
+            description="This dataset holds the resources you want to validate. Required when"
+            " Source / Input Mode is set to dataset; leave it empty when using entities.",
             param_type=DatasetParameterType(dataset_type="json"),
             advanced=True,
             default_value="",
         ),
         PluginParameter(
             name="target_dataset",
             label="Target JSON Dataset",
-            description="This dataset will be used to store the valid JSON objects"
-            " after validation.",
+            description="This dataset stores the valid JSON objects after validation,"
+            " replacing any existing content. Required when Target / Output Mode is set to"
+            " dataset; leave it empty when using entities.",
             param_type=DatasetParameterType(dataset_type="json"),
             default_value="",
             advanced=True,
@@ -142,7 +140,9 @@
         PluginParameter(
             name="fail_on_violations",
             label="Fail on violations",
-            description="If enabled, the task will fail on the first data violation.",
+            description="If enabled, the workflow fails once validation completes if any"
+            " entity has violations. All entities are validated either way; disabling this"
+            " instead reports the violations as warnings.",
             default_value=DEFAULT_FAIL_ON_VIOLATION,
         ),
     ],
@@ -286,7 +286,9 @@ def execute(
             Client.from_context(context=context).datasets.post_file_resource(
                 project_id=context.task.project_id(),
                 dataset_id=self.target_dataset,
-                file_resource=io.BytesIO(json.dumps(valid_json_objects).encode("utf-8")),
+                file_resource=io.BytesIO(
+                    json.dumps(valid_json_objects, ensure_ascii=False).encode("utf-8")
+                ),
             )
             return None
 

cmem_plugin_validation/validate_graph/task.py

@@ -27,8 +27,24 @@
 from cmem_plugin_validation.validate_graph.state import State
 
 DOCUMENTATION = """
-Start a graph validation process which verifies, that resources in a specific graph are valid
-according to the node shapes in a shape catalog graph.
+Starts a graph validation process which verifies that resources in a specific graph are valid
+according to the node shapes in a shape catalog graph. The task waits for the validation to
+finish before completing.
+
+The task has no input port. Violations found during validation are always summarized in the
+workflow report, and can also be materialized into a result graph in the project. Sending each
+violation as an entity to an output port is optional; when that is disabled, the task has no
+output port either.
+
+### Error Handling
+
+Every resource is validated regardless of the outcome. The task can then either:
+
+- Fail once validation completes if any resource has violations, halting the workflow.
+- Report violations only as warnings, allowing follow-up tasks to run based on the results.
+
+The error handling behavior is configurable through the **Fail workflow on violations**
+parameter.
 """
 
 DEFAULT_SHAPE_GRAPH = "https://vocab.eccenca.com/shacl/"
@@ -85,23 +101,30 @@
         PluginParameter(
             name="clear_result_graph",
             label="Clear result graph before validation",
+            description="If enabled, the existing content of the result graph is deleted"
+            " before validation starts. Has no effect when Result graph is left empty.",
             default_value=DEFAULT_CLEAR_RESULT_GRAPH,
         ),
         PluginParameter(
             name="fail_on_violations",
             label="Fail workflow on violations",
+            description="If enabled, the workflow fails once validation completes if any"
+            " resource has violations. All resources are validated either way; disabling this"
+            " instead reports the violations as warnings.",
             default_value=DEFAULT_FAIL_ON_VIOLATION,
         ),
         PluginParameter(
             name="output_results",
             label="Output violations as entities",
+            description="If enabled, each violation is sent as an entity to an output port"
+            " for further processing in the workflow. Disabling this removes the output port.",
             default_value=DEFAULT_OUTPUT_RESULTS,
         ),
         PluginParameter(
             name="sparql_query",
             label="Resource Selection Query",
             description="The query to select the resources to validate. "
-            "Use {{context_graph}} as a placeholder for the select context graph for validation.",
+            "Use `{{context_graph}}` as a placeholder for the selected Context Graph.",
             default_value=DEFAULT_SPARQL_QUERY,
             advanced=True,
         ),

tests/fixtures/source.unicode.json

@@ -0,0 +1 @@
+[{"name" : "Käse", "price" : 5.5}, {"name" : "Müsli", "price" : 3.2}]

tests/test_validate_entities.py

@@ -25,8 +25,10 @@ class TestSetup:
     schema_dataset: str = "schema_dataset"
     valid_source_dataset_file: Path = FIXTURE_DIR / "source.valid.json"
     invalid_source_dataset_file: Path = FIXTURE_DIR / "source.invalid.json"
+    unicode_source_dataset_file: Path = FIXTURE_DIR / "source.unicode.json"
     valid_source_dataset: str = "valid_source_dataset"
     invalid_source_dataset: str = "invalid_source_dataset"
+    unicode_source_dataset: str = "unicode_source_dataset"
     target_dataset_file: str = "target.json"
     target_dataset: str = "target_dataset"
     project_name: str = "validate_entities_test_project"
@@ -68,6 +70,7 @@ def project() -> Generator[TestSetup]:
     for dataset_name, dataset_file in (
         (_.valid_source_dataset, _.valid_source_dataset_file),
         (_.invalid_source_dataset, _.invalid_source_dataset_file),
+        (_.unicode_source_dataset, _.unicode_source_dataset_file),
         (_.schema_dataset, _.schema_dataset_file),
     ):
         _make_dataset(client, _.project_name, dataset_name, dataset_file.name)
@@ -138,7 +141,8 @@ def test_execute_with_source_dataset(project: TestSetup) -> None:
         assert len(list(entities.entities)) == 1
 
 
-def validate_test_source_target_dataset(project: TestSetup) -> None:
+@needs_cmem
+def test_source_and_target_dataset(project: TestSetup) -> None:
     """Test source and target dataset mode"""
     _ = project
 
@@ -154,3 +158,27 @@ def validate_test_source_target_dataset(project: TestSetup) -> None:
     client = get_client(_.project_name)
     data = json.loads(client.files.read(f"{_.project_name}:{_.target_dataset_file}"))
     assert len(data) == _.valid_source_object_count
+
+
+@needs_cmem
+def test_target_dataset_keeps_unicode_characters(project: TestSetup) -> None:
+    """Test that non-ASCII characters in the source data are not escaped in the target dataset"""
+    _ = project
+
+    ValidateEntity(
+        source_mode=SOURCE.dataset,
+        target_mode=TARGET.dataset,
+        json_schema_dataset=_.schema_dataset,
+        fail_on_violations=True,
+        source_dataset=_.unicode_source_dataset,
+        target_dataset=_.target_dataset,
+    ).execute([], TestExecutionContext(project_id=_.project_name))
+
+    client = get_client(_.project_name)
+    raw_content = client.files.read(f"{_.project_name}:{_.target_dataset_file}").decode("utf-8")
+    assert "\\u00e4" not in raw_content  # ä
+    assert "\\u00fc" not in raw_content  # ü
+    assert json.loads(raw_content) == [
+        {"name": "Käse", "price": 5.5},
+        {"name": "Müsli", "price": 3.2},
+    ]