diff --git a/CHANGELOG.md b/CHANGELOG.md index 9243cb4..7637493 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this p ### Changed - updated dependencies and template +- corrected and clarified the documentation and parameter descriptions of the + `Validate Entities` and `Validate Knowledge Graph` plugins + +### Fixed + +- Validate Entities task no longer converts non-ASCII characters to unicode escape sequences + when writing valid JSON objects to the target dataset ## [1.3.0] 2026-08-19 diff --git a/cmem_plugin_validation/validate_entities/task.py b/cmem_plugin_validation/validate_entities/task.py index 9661a14..46e3dc9 100644 --- a/cmem_plugin_validation/validate_entities/task.py +++ b/cmem_plugin_validation/validate_entities/task.py @@ -39,32 +39,26 @@ ### Input Modes -The plugin supports two input modes for validation: - -1. **Validate Entities**: Validates entities received from the input port in the workflow. -2. **Validate JSON Dataset**: Validates a JSON dataset stored in the project. - - If the JSON dataset is a JSON array, the schema will validate each object inside the array. - - If the JSON dataset is a JSON object, it will be validated against the schema directly. - -Validated data objects can be sent to an output port for further processing in the workflow -or saved in a JSON dataset in the project. +Entities arrive on the input port by default. Switching the input mode to a JSON dataset +removes the input port, and the resources to validate are instead read from a JSON dataset in +the project. A JSON array in that dataset has each of its objects validated individually; a +JSON object is validated directly. ### Output Modes -1. **Valid JSON objects sent to Output Port**: Valid JSON objects can be sent as entities - to the output port. -2. **Saved in JSON Dataset**: Valid JSON objects can be stored in a specified JSON dataset - in the project. +Valid JSON objects are sent to an output port by default, ready for further processing in the +workflow. Switching the output mode to a JSON dataset removes the output port, and the valid +JSON objects are instead saved to a JSON dataset in the project, replacing its existing content. ### Error Handling -The task can either: +Every entity or object is validated regardless of the outcome. The task can then either: -- Fail instantly if there is a data violation, halting the workflow. -- Provide warnings in the workflow report, allowing follow-up tasks to run based on the +- Fail once validation completes if any entity has violations, halting the workflow. +- Report violations only as warnings, allowing follow-up tasks to run based on the validated data. -The error handling behavior is configurable through the `Fail on violations` parameter. +The error handling behavior is configurable through the **Fail on violations** parameter. """ @@ -88,9 +82,9 @@ TARGET.options = OrderedDict( { TARGET.dataset: f"{TARGET.dataset}: " - "Valid JSON objects will be is saved in a JSON dataset (see advanced options).", + "Valid JSON objects are saved in a JSON dataset (see advanced options).", TARGET.entities: f"{TARGET.entities}: " - "Valid JSON objects will be send as entities to the output port.", + "Valid JSON objects are sent as entities to the output port.", } ) @@ -105,21 +99,24 @@ PluginParameter( name="source_mode", label="Source / Input Mode", - description="", + description="Selects where entities to validate come from: the input port or" + " a JSON dataset.", param_type=ChoiceParameterType(SOURCE.options), default_value=SOURCE.entities, ), PluginParameter( name="target_mode", label="Target / Output Mode", - description="", + description="Selects where valid JSON objects are written to: the output port" + " or a JSON dataset.", param_type=ChoiceParameterType(TARGET.options), default_value=TARGET.entities, ), PluginParameter( name="source_dataset", label="Source JSON Dataset", - description="This dataset holds the resources you want to validate.", + description="This dataset holds the resources you want to validate. Required when" + " Source / Input Mode is set to dataset; leave it empty when using entities.", param_type=DatasetParameterType(dataset_type="json"), advanced=True, default_value="", @@ -127,8 +124,9 @@ PluginParameter( name="target_dataset", label="Target JSON Dataset", - description="This dataset will be used to store the valid JSON objects" - " after validation.", + description="This dataset stores the valid JSON objects after validation," + " replacing any existing content. Required when Target / Output Mode is set to" + " dataset; leave it empty when using entities.", param_type=DatasetParameterType(dataset_type="json"), default_value="", advanced=True, @@ -142,7 +140,9 @@ PluginParameter( name="fail_on_violations", label="Fail on violations", - description="If enabled, the task will fail on the first data violation.", + description="If enabled, the workflow fails once validation completes if any" + " entity has violations. All entities are validated either way; disabling this" + " instead reports the violations as warnings.", default_value=DEFAULT_FAIL_ON_VIOLATION, ), ], @@ -286,7 +286,9 @@ def execute( Client.from_context(context=context).datasets.post_file_resource( project_id=context.task.project_id(), dataset_id=self.target_dataset, - file_resource=io.BytesIO(json.dumps(valid_json_objects).encode("utf-8")), + file_resource=io.BytesIO( + json.dumps(valid_json_objects, ensure_ascii=False).encode("utf-8") + ), ) return None diff --git a/cmem_plugin_validation/validate_graph/task.py b/cmem_plugin_validation/validate_graph/task.py index 40fd2e4..943cecf 100644 --- a/cmem_plugin_validation/validate_graph/task.py +++ b/cmem_plugin_validation/validate_graph/task.py @@ -27,8 +27,24 @@ from cmem_plugin_validation.validate_graph.state import State DOCUMENTATION = """ -Start a graph validation process which verifies, that resources in a specific graph are valid -according to the node shapes in a shape catalog graph. +Starts a graph validation process which verifies that resources in a specific graph are valid +according to the node shapes in a shape catalog graph. The task waits for the validation to +finish before completing. + +The task has no input port. Violations found during validation are always summarized in the +workflow report, and can also be materialized into a result graph in the project. Sending each +violation as an entity to an output port is optional; when that is disabled, the task has no +output port either. + +### Error Handling + +Every resource is validated regardless of the outcome. The task can then either: + +- Fail once validation completes if any resource has violations, halting the workflow. +- Report violations only as warnings, allowing follow-up tasks to run based on the results. + +The error handling behavior is configurable through the **Fail workflow on violations** +parameter. """ DEFAULT_SHAPE_GRAPH = "https://vocab.eccenca.com/shacl/" @@ -85,23 +101,30 @@ PluginParameter( name="clear_result_graph", label="Clear result graph before validation", + description="If enabled, the existing content of the result graph is deleted" + " before validation starts. Has no effect when Result graph is left empty.", default_value=DEFAULT_CLEAR_RESULT_GRAPH, ), PluginParameter( name="fail_on_violations", label="Fail workflow on violations", + description="If enabled, the workflow fails once validation completes if any" + " resource has violations. All resources are validated either way; disabling this" + " instead reports the violations as warnings.", default_value=DEFAULT_FAIL_ON_VIOLATION, ), PluginParameter( name="output_results", label="Output violations as entities", + description="If enabled, each violation is sent as an entity to an output port" + " for further processing in the workflow. Disabling this removes the output port.", default_value=DEFAULT_OUTPUT_RESULTS, ), PluginParameter( name="sparql_query", label="Resource Selection Query", description="The query to select the resources to validate. " - "Use {{context_graph}} as a placeholder for the select context graph for validation.", + "Use `{{context_graph}}` as a placeholder for the selected Context Graph.", default_value=DEFAULT_SPARQL_QUERY, advanced=True, ), diff --git a/tests/fixtures/source.unicode.json b/tests/fixtures/source.unicode.json new file mode 100644 index 0000000..8dacb9c --- /dev/null +++ b/tests/fixtures/source.unicode.json @@ -0,0 +1 @@ +[{"name" : "Käse", "price" : 5.5}, {"name" : "Müsli", "price" : 3.2}] diff --git a/tests/test_validate_entities.py b/tests/test_validate_entities.py index 633260b..76e34bf 100644 --- a/tests/test_validate_entities.py +++ b/tests/test_validate_entities.py @@ -25,8 +25,10 @@ class TestSetup: schema_dataset: str = "schema_dataset" valid_source_dataset_file: Path = FIXTURE_DIR / "source.valid.json" invalid_source_dataset_file: Path = FIXTURE_DIR / "source.invalid.json" + unicode_source_dataset_file: Path = FIXTURE_DIR / "source.unicode.json" valid_source_dataset: str = "valid_source_dataset" invalid_source_dataset: str = "invalid_source_dataset" + unicode_source_dataset: str = "unicode_source_dataset" target_dataset_file: str = "target.json" target_dataset: str = "target_dataset" project_name: str = "validate_entities_test_project" @@ -68,6 +70,7 @@ def project() -> Generator[TestSetup]: for dataset_name, dataset_file in ( (_.valid_source_dataset, _.valid_source_dataset_file), (_.invalid_source_dataset, _.invalid_source_dataset_file), + (_.unicode_source_dataset, _.unicode_source_dataset_file), (_.schema_dataset, _.schema_dataset_file), ): _make_dataset(client, _.project_name, dataset_name, dataset_file.name) @@ -138,7 +141,8 @@ def test_execute_with_source_dataset(project: TestSetup) -> None: assert len(list(entities.entities)) == 1 -def validate_test_source_target_dataset(project: TestSetup) -> None: +@needs_cmem +def test_source_and_target_dataset(project: TestSetup) -> None: """Test source and target dataset mode""" _ = project @@ -154,3 +158,27 @@ def validate_test_source_target_dataset(project: TestSetup) -> None: client = get_client(_.project_name) data = json.loads(client.files.read(f"{_.project_name}:{_.target_dataset_file}")) assert len(data) == _.valid_source_object_count + + +@needs_cmem +def test_target_dataset_keeps_unicode_characters(project: TestSetup) -> None: + """Test that non-ASCII characters in the source data are not escaped in the target dataset""" + _ = project + + ValidateEntity( + source_mode=SOURCE.dataset, + target_mode=TARGET.dataset, + json_schema_dataset=_.schema_dataset, + fail_on_violations=True, + source_dataset=_.unicode_source_dataset, + target_dataset=_.target_dataset, + ).execute([], TestExecutionContext(project_id=_.project_name)) + + client = get_client(_.project_name) + raw_content = client.files.read(f"{_.project_name}:{_.target_dataset_file}").decode("utf-8") + assert "\\u00e4" not in raw_content # ä + assert "\\u00fc" not in raw_content # ü + assert json.loads(raw_content) == [ + {"name": "Käse", "price": 5.5}, + {"name": "Müsli", "price": 3.2}, + ]