From 4931c4502ac47ddb503b022fd169b0a5a123da7a Mon Sep 17 00:00:00 2001 From: Alberto Date: Fri, 31 Jul 2026 13:27:03 +0200 Subject: [PATCH 1/4] Fix #937: count accepts any Measure type and counts Data Points --- src/vtlengine/Interpreter/__init__.py | 11 ++- src/vtlengine/Operators/Aggregation.py | 18 +++- .../duckdb_transpiler/Transpiler/__init__.py | 22 ++--- .../data/DataSet/output/GL_222_1-1.csv | 2 +- .../data/DataSet/output/GL_466_1-3.csv | 28 +++--- .../data/DataSet/output/GL_466_2-1.csv | 8 +- .../output/DEMO1-aggr.numDPCouYear.csv | 96 +++++++++---------- tests/Bugs/data/DataSet/input/GH_937_1-1.csv | 4 + tests/Bugs/data/DataSet/output/GH_937_1-1.csv | 2 + tests/Bugs/data/DataSet/output/GH_937_1-2.csv | 3 + tests/Bugs/data/DataSet/output/GL_270_2-1.csv | 4 +- .../data/DataStructure/input/GH_937_1-1.json | 33 +++++++ .../data/DataStructure/output/GH_937_1-1.json | 15 +++ .../data/DataStructure/output/GH_937_1-2.json | 21 ++++ tests/Bugs/data/vtl/GH_937_1.vtl | 2 + tests/Bugs/test_bugs.py | 18 ++++ tests/DWI/data/DataSet/output/GL_218_7-1.csv | 2 +- tests/DWI/data/DataSet/output/GL_218_8-1.csv | 2 +- .../data/DataSet/output/10-1-12-DS_r.csv | 2 +- .../data/DataSet/output/10-1-13-DS_r.csv | 2 +- .../data/DataSet/output/10-1-14-DS_r.csv | 2 +- .../data/DataSet/output/10-1-15-DS_r.csv | 2 +- .../data/DataSet/output/10-1-16-DS_r.csv | 2 +- .../data/DataSet/output/10-1-17-DS_r.csv | 2 +- .../data/DataSet/output/10-1-18-DS_r.csv | 2 +- .../data/DataSet/output/10-1-19-DS_r.csv | 4 +- .../test_aggregate_operators.py | 12 +-- tests/UDO/data/DataSet/output/GL_442_1-1.csv | 60 ++++++------ tests/UDO/data/DataSet/output/GL_442_3-1.csv | 60 ++++++------ tests/UDO/data/DataSet/output/GL_442_3-2.csv | 38 ++++---- tests/UDO/data/DataSet/output/GL_442_4-1.csv | 60 ++++++------ tests/UDO/data/DataSet/output/GL_442_4-2.csv | 38 ++++---- tests/UDO/data/DataSet/output/GL_442_5-3.csv | 38 ++++---- tests/UDO/data/DataSet/output/GL_442_5-4.csv | 60 ++++++------ 34 files changed, 388 insertions(+), 287 deletions(-) create mode 100644 tests/Bugs/data/DataSet/input/GH_937_1-1.csv create mode 100644 tests/Bugs/data/DataSet/output/GH_937_1-1.csv create mode 100644 tests/Bugs/data/DataSet/output/GH_937_1-2.csv create mode 100644 tests/Bugs/data/DataStructure/input/GH_937_1-1.json create mode 100644 tests/Bugs/data/DataStructure/output/GH_937_1-1.json create mode 100644 tests/Bugs/data/DataStructure/output/GH_937_1-2.json create mode 100644 tests/Bugs/data/vtl/GH_937_1.vtl diff --git a/src/vtlengine/Interpreter/__init__.py b/src/vtlengine/Interpreter/__init__.py index cd46054c2..b3ab06f23 100644 --- a/src/vtlengine/Interpreter/__init__.py +++ b/src/vtlengine/Interpreter/__init__.py @@ -666,7 +666,16 @@ def visit_Aggregation(self, node: AST.Aggregation) -> None: # Setting here group by as we have already selected the identifiers we need grouping_op = "group by" - result = AGGREGATION_MAPPING[node.op].analyze(operand, grouping_op, groupings, having) + # count over a Component counts that Component's non-null values, while count + # over a Data Set counts Data Points; the manual gives them separate syntaxes. + component_operand = ( + not self.is_from_having + and self.is_from_regular_aggregation + and node.operand is not None + ) + result = AGGREGATION_MAPPING[node.op].analyze( + operand, grouping_op, groupings, having, component_operand + ) if not self.is_from_regular_aggregation: result.name = VirtualCounter._new_ds_name() return result diff --git a/src/vtlengine/Operators/Aggregation.py b/src/vtlengine/Operators/Aggregation.py index 33178cd99..c45679afd 100644 --- a/src/vtlengine/Operators/Aggregation.py +++ b/src/vtlengine/Operators/Aggregation.py @@ -112,6 +112,7 @@ def validate( # type: ignore[override] group_op: Optional[str], grouping_columns: Any, having_data: Any, + component_operand: bool = False, ) -> Dataset: result_components = {k: copy(v) for k, v in operand.components.items()} if cls.op not in [COUNT, MIN, MAX] and len(operand.get_measures_names()) == 0: @@ -147,8 +148,7 @@ def validate( # type: ignore[override] for comp_name, comp in operand.components.items(): if comp.role == Role.ATTRIBUTE: del result_components[comp_name] - # TimeInterval is not supported as a measure in aggregate operations - if any( + if cls.op != COUNT and any( comp.role == Role.MEASURE and comp.data_type is TimeInterval for comp in result_components.values() ): @@ -190,6 +190,7 @@ def _agg_func( grouping_keys: Optional[List[str]], measure_names: Optional[List[str]], having_expression: Optional[str], + component_operand: bool = False, ) -> pd.DataFrame: grouping_names = ( [f'"{name}"' for name in grouping_keys] if grouping_keys is not None else None @@ -226,7 +227,11 @@ def _agg_func( f"{cls.py_op}(CAST({e} AS DOUBLE)) AS {e}, " # Count can only be one here ) elif cls.op == COUNT: - functions += f"{cls.py_op}({e}) AS int_var, " + functions += ( + f"{cls.py_op}({e}) AS int_var, " + if component_operand + else "COUNT(*) AS int_var, " + ) break else: functions += f"{cls.py_op}({e}) AS {e}, " @@ -263,6 +268,7 @@ def evaluate( # type: ignore[override] group_op: Optional[str], grouping_columns: Optional[List[str]], having_expr: Optional[str], + component_operand: bool = False, ) -> Dataset: result = cls.validate(operand, group_op, grouping_columns, having_expr) @@ -273,14 +279,16 @@ def evaluate( # type: ignore[override] # Keep a copy of viral attrs for post-aggregation propagation viral_df = result_df[grouping_keys + viral_attr_names].copy() if viral_attr_names else None result_df = result_df[grouping_keys + measure_names] - if cls.op == COUNT: + if cls.op == COUNT and component_operand: result_df = result_df.dropna(subset=measure_names, how="any") if cls.op in [MAX, MIN]: for measure in operand.get_measures(): if measure.data_type == TimeInterval: raise RunTimeError("2-1-19-18", op=cls.op) cls._handle_data_types(result_df, operand.get_measures(), "input") - result_df = cls._agg_func(result_df, grouping_keys, measure_names, having_expr) + result_df = cls._agg_func( + result_df, grouping_keys, measure_names, having_expr, component_operand + ) cls._handle_data_types(result_df, operand.get_measures(), "result") # Handle correct order on result diff --git a/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py b/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py index 1bee4b314..311bc6ca1 100644 --- a/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py +++ b/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py @@ -2219,11 +2219,8 @@ def visit_Aggregation(self, node: AST.Aggregation) -> str: # type: ignore[overr # count() without operand if node.operand is None: if op == tokens.COUNT: - if self._in_clause and self._current_dataset: - measures = self._current_dataset.get_measures_names() - if measures: - or_parts = " OR ".join(f"{quote_name(m)} IS NOT NULL" for m in measures) - return f"NULLIF(COUNT(CASE WHEN {or_parts} THEN 1 END), 0)" + # count() without an operand counts Data Points, so a Data Point whose + # Measures are null still contributes (issue #937). return "NULLIF(COUNT(*), 0)" return "" @@ -2240,18 +2237,11 @@ def visit_Aggregation(self, node: AST.Aggregation) -> str: # type: ignore[overr cols, group_by_cols = self._build_agg_group_cols(node, ds, group_cols) ds_tp_minmax_cols: List[tuple[str, str]] = [] - # count() produces a single int_var measure. + # count() produces a single int_var measure. It reports the number of Data + # Points, so a Data Point is counted even where one of its Measures is null + # (issue #937); a group that exists always holds at least one of them. if op == tokens.COUNT: - alias = "int_var" - source_measures = ds.get_measures_names() - if source_measures: - and_parts = " AND ".join(f"{quote_name(m)} IS NOT NULL" for m in source_measures) - count_expr = f"COUNT(CASE WHEN {and_parts} THEN 1 END)" - if group_cols: - count_expr = f"NULLIF({count_expr}, 0)" - cols.append(f"{count_expr} AS {quote_name(alias)}") - else: - cols.append(f"COUNT(*) AS {quote_name(alias)}") + cols.append(f"COUNT(*) AS {quote_name('int_var')}") else: measures = ds.get_measures_names() for measure in measures: diff --git a/tests/Additional/data/DataSet/output/GL_222_1-1.csv b/tests/Additional/data/DataSet/output/GL_222_1-1.csv index 65ebeeab8..f46417126 100644 --- a/tests/Additional/data/DataSet/output/GL_222_1-1.csv +++ b/tests/Additional/data/DataSet/output/GL_222_1-1.csv @@ -1,4 +1,4 @@ Id_1,Me_3,Me_4 -1,1,1 +1,2,2 2,3,3 diff --git a/tests/Aggregate/data/DataSet/output/GL_466_1-3.csv b/tests/Aggregate/data/DataSet/output/GL_466_1-3.csv index bf7f01d44..fe199020d 100644 --- a/tests/Aggregate/data/DataSet/output/GL_466_1-3.csv +++ b/tests/Aggregate/data/DataSet/output/GL_466_1-3.csv @@ -1,16 +1,16 @@ month,int_var -2023-01,1 -2023-02, -2023-03, -2023-04, -2023-05, -2023-06, -2023-07, -2023-08, -2023-09, -2023-10, -2023-11, -2023-12, -2024-01, -2024-02,2 +2023-01,12 +2023-02,28 +2023-03,31 +2023-04,30 +2023-05,31 +2023-06,30 +2023-07,31 +2023-08,31 +2023-09,30 +2023-10,31 +2023-11,30 +2023-12,31 +2024-01,31 +2024-02,29 2024-03,1 diff --git a/tests/Aggregate/data/DataSet/output/GL_466_2-1.csv b/tests/Aggregate/data/DataSet/output/GL_466_2-1.csv index 3a06ef915..4cfea2c1d 100644 --- a/tests/Aggregate/data/DataSet/output/GL_466_2-1.csv +++ b/tests/Aggregate/data/DataSet/output/GL_466_2-1.csv @@ -1,5 +1,5 @@ month,int_var -2023-01, -2024-02, -2024-03, -2024-04,1.0 \ No newline at end of file +2023-01,2 +2024-02,6 +2024-03,1 +2024-04,1 diff --git a/tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv b/tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv index 4da8d0a5f..ccca13765 100644 --- a/tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv +++ b/tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv @@ -1,97 +1,97 @@ REF_DATE,REP_COUNTRY,int_var -2018-12-31,CN,6 +2018-12-31,CN,14 2018-12-31,AU,14 -2018-12-31,FI,13 -2018-12-31,TR,12 -2018-12-31,US,12 -2018-12-31,BM,12 +2018-12-31,FI,14 +2018-12-31,TR,14 +2018-12-31,US,14 +2018-12-31,BM,14 2018-12-31,GG,14 -2018-12-31,CY,13 -2018-12-31,ID,12 +2018-12-31,CY,14 +2018-12-31,ID,14 2018-12-31,5A,14 2018-12-31,KY,14 -2018-12-31,BS,12 +2018-12-31,BS,14 2018-12-31,RU,14 2018-12-31,KR,14 -2018-12-31,GR,6 -2018-12-31,SE,12 +2018-12-31,GR,7 +2018-12-31,SE,14 2018-12-31,DE,14 -2018-12-31,SG,7 -2018-12-31,JP,13 +2018-12-31,SG,9 +2018-12-31,JP,14 2018-12-31,IT,14 2018-12-31,NL,14 -2018-12-31,CW,6 -2018-12-31,HK,8 +2018-12-31,CW,7 +2018-12-31,HK,14 2018-12-31,ZA,14 2018-12-31,IM,14 2018-12-31,BE,14 -2018-12-31,MY,4 +2018-12-31,MY,14 2018-12-31,MO,14 2018-12-31,AT,14 2018-12-31,LU,14 2018-12-31,PH,14 2018-12-31,CH,14 -2018-12-31,PT,11 -2018-12-31,DK,11 -2018-12-31,IN,6 -2018-12-31,JE,8 +2018-12-31,PT,14 +2018-12-31,DK,14 +2018-12-31,IN,14 +2018-12-31,JE,9 2018-12-31,NO,14 2018-12-31,GB,14 2018-12-31,TW,14 -2018-12-31,PA,7 -2018-12-31,CL,8 -2018-12-31,CA,13 +2018-12-31,PA,10 +2018-12-31,CL,10 +2018-12-31,CA,14 2018-12-31,ES,14 2018-12-31,IE,14 -2018-12-31,BR,8 +2018-12-31,BR,9 2018-12-31,MX,6 2018-12-31,BH,6 2018-12-31,FR,14 -2019-03-31,CN,7 +2019-03-31,CN,14 2019-03-31,AU,14 -2019-03-31,FI,13 -2019-03-31,TR,12 -2019-03-31,US,13 -2019-03-31,BM,13 +2019-03-31,FI,14 +2019-03-31,TR,14 +2019-03-31,US,14 +2019-03-31,BM,14 2019-03-31,GG,14 -2019-03-31,CY,13 -2019-03-31,ID,12 +2019-03-31,CY,14 +2019-03-31,ID,14 2019-03-31,5A,14 2019-03-31,KY,14 -2019-03-31,BS,12 +2019-03-31,BS,14 2019-03-31,RU,14 2019-03-31,KR,14 -2019-03-31,GR,6 -2019-03-31,SE,13 +2019-03-31,GR,7 +2019-03-31,SE,14 2019-03-31,DE,14 -2019-03-31,SG,7 -2019-03-31,JP,13 +2019-03-31,SG,9 +2019-03-31,JP,14 2019-03-31,IT,14 2019-03-31,NL,14 -2019-03-31,CW,6 -2019-03-31,HK,8 +2019-03-31,CW,7 +2019-03-31,HK,14 2019-03-31,ZA,14 -2019-03-31,IM,13 +2019-03-31,IM,14 2019-03-31,BE,14 -2019-03-31,MY,4 +2019-03-31,MY,14 2019-03-31,MO,14 -2019-03-31,AT,13 +2019-03-31,AT,14 2019-03-31,LU,14 2019-03-31,PH,14 2019-03-31,CH,14 -2019-03-31,PT,11 -2019-03-31,DK,11 -2019-03-31,IN,6 -2019-03-31,JE,8 +2019-03-31,PT,14 +2019-03-31,DK,14 +2019-03-31,IN,14 +2019-03-31,JE,9 2019-03-31,NO,14 2019-03-31,GB,14 2019-03-31,TW,14 -2019-03-31,PA,6 -2019-03-31,CL,8 -2019-03-31,CA,13 +2019-03-31,PA,10 +2019-03-31,CL,10 +2019-03-31,CA,14 2019-03-31,ES,14 2019-03-31,IE,14 -2019-03-31,BR,8 +2019-03-31,BR,9 2019-03-31,MX,6 2019-03-31,BH,6 2019-03-31,FR,14 diff --git a/tests/Bugs/data/DataSet/input/GH_937_1-1.csv b/tests/Bugs/data/DataSet/input/GH_937_1-1.csv new file mode 100644 index 000000000..77c12c6b0 --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_937_1-1.csv @@ -0,0 +1,4 @@ +Id_1,Id_2,Me_1,Me_2 +1,a,1.0,2020-01-01/2020-12-31 +1,b,2.0,2021-01-01/2021-12-31 +2,a,3.0,2022-01-01/2022-12-31 diff --git a/tests/Bugs/data/DataSet/output/GH_937_1-1.csv b/tests/Bugs/data/DataSet/output/GH_937_1-1.csv new file mode 100644 index 000000000..a3e52b27e --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_937_1-1.csv @@ -0,0 +1,2 @@ +int_var +3 diff --git a/tests/Bugs/data/DataSet/output/GH_937_1-2.csv b/tests/Bugs/data/DataSet/output/GH_937_1-2.csv new file mode 100644 index 000000000..bee4812da --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_937_1-2.csv @@ -0,0 +1,3 @@ +Id_1,int_var +1,2 +2,1 diff --git a/tests/Bugs/data/DataSet/output/GL_270_2-1.csv b/tests/Bugs/data/DataSet/output/GL_270_2-1.csv index 14a6f59d5..91ea776dc 100644 --- a/tests/Bugs/data/DataSet/output/GL_270_2-1.csv +++ b/tests/Bugs/data/DataSet/output/GL_270_2-1.csv @@ -1,5 +1,5 @@ CNTRCT_ID,DT_RFRNC,INSTRMNT_ID,OBSRVD_AGNT_CD,JNT_LBLTY_AMNT_SUM,NMBR_DBTR,IS_JNT_LBLTY_RPRTD_ALL,JNT_LBLTY_AMNT_MAX -AAA,2020-01-01,AAA,AAA,101.0,2,False,100.0 +AAA,2020-01-01,AAA,AAA,101.0,3,False,100.0 BBB,2020-01-01,BBB,BBB,12.0,2,True,11.0 -CCC,2020-01-01,BBB,BBB,,,False, +CCC,2020-01-01,BBB,BBB,,1,False, DDD,2020-01-01,DDD,BBB,0.0,1,True,0.0 diff --git a/tests/Bugs/data/DataStructure/input/GH_937_1-1.json b/tests/Bugs/data/DataStructure/input/GH_937_1-1.json new file mode 100644 index 000000000..2b542c03c --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_937_1-1.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Integer", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "Me_2", + "role": "Measure", + "type": "Time", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_937_1-1.json b/tests/Bugs/data/DataStructure/output/GH_937_1-1.json new file mode 100644 index 000000000..7e6215291 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_937_1-1.json @@ -0,0 +1,15 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "int_var", + "role": "Measure", + "type": "Integer", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_937_1-2.json b/tests/Bugs/data/DataStructure/output/GH_937_1-2.json new file mode 100644 index 000000000..60cd31967 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_937_1-2.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_r2", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Integer", + "nullable": false + }, + { + "name": "int_var", + "role": "Measure", + "type": "Integer", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/vtl/GH_937_1.vtl b/tests/Bugs/data/vtl/GH_937_1.vtl new file mode 100644 index 000000000..a029d3fd5 --- /dev/null +++ b/tests/Bugs/data/vtl/GH_937_1.vtl @@ -0,0 +1,2 @@ +DS_r1 <- count(DS_1); +DS_r2 <- count(DS_1 group by Id_1); diff --git a/tests/Bugs/test_bugs.py b/tests/Bugs/test_bugs.py index 6e4feefca..7a41da458 100644 --- a/tests/Bugs/test_bugs.py +++ b/tests/Bugs/test_bugs.py @@ -1945,6 +1945,24 @@ def test_GL_410(self): code=code, number_inputs=number_inputs, exception_code=message ) + def test_GH_937_1(self): + """ + Status: OK + Description: count only reports the number of Data Points, so it does not + aggregate the Measures it is given: the reference manual types its + operand as a plain dataset, states no Additional Constraints, and + counts a String Measure in its own example. A Time Measure was + rejected anyway, because the guard against aggregating one ran + before count replaced every Measure with int_var. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/937 + Goal: Check Result. + """ + code = "GH_937_1" + number_inputs = 1 + references_names = ["1", "2"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + class DataValidationBugs(BugHelper): """ """ diff --git a/tests/DWI/data/DataSet/output/GL_218_7-1.csv b/tests/DWI/data/DataSet/output/GL_218_7-1.csv index 354b08a35..54858b568 100644 --- a/tests/DWI/data/DataSet/output/GL_218_7-1.csv +++ b/tests/DWI/data/DataSet/output/GL_218_7-1.csv @@ -1,2 +1,2 @@ int_var -0 +1 diff --git a/tests/DWI/data/DataSet/output/GL_218_8-1.csv b/tests/DWI/data/DataSet/output/GL_218_8-1.csv index 231e26d86..54858b568 100644 --- a/tests/DWI/data/DataSet/output/GL_218_8-1.csv +++ b/tests/DWI/data/DataSet/output/GL_218_8-1.csv @@ -1,2 +1,2 @@ int_var -0 \ No newline at end of file +1 diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-12-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-12-DS_r.csv index a4a9a71e4..b0f4b613a 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-12-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-12-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,1 \ No newline at end of file +2,2 diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-13-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-13-DS_r.csv index a4a9a71e4..b0f4b613a 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-13-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-13-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,1 \ No newline at end of file +2,2 diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-14-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-14-DS_r.csv index a4a9a71e4..b0f4b613a 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-14-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-14-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,1 \ No newline at end of file +2,2 diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-15-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-15-DS_r.csv index a4a9a71e4..b0f4b613a 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-15-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-15-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,1 \ No newline at end of file +2,2 diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-16-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-16-DS_r.csv index a4a9a71e4..b0f4b613a 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-16-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-16-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,1 \ No newline at end of file +2,2 diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-17-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-17-DS_r.csv index a4a9a71e4..b0f4b613a 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-17-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-17-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,1 \ No newline at end of file +2,2 diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-18-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-18-DS_r.csv index cd3ee94f9..b15a6807a 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-18-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-18-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,2 \ No newline at end of file +2,3 diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-19-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-19-DS_r.csv index a92971305..b15a6807a 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-19-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-19-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var -1, -2, \ No newline at end of file +1,2 +2,3 diff --git a/tests/TypeChecking/AggregateOperators/test_aggregate_operators.py b/tests/TypeChecking/AggregateOperators/test_aggregate_operators.py index a70d67c1b..a52ceb7d9 100644 --- a/tests/TypeChecking/AggregateOperators/test_aggregate_operators.py +++ b/tests/TypeChecking/AggregateOperators/test_aggregate_operators.py @@ -229,11 +229,9 @@ def test_13(self): """ code = "10-1-13" number_inputs = 1 - exception_code = "1-1-19-12" + references_names = ["DS_r"] - self.NewSemanticExceptionTest( - code=code, number_inputs=number_inputs, exception_code=exception_code - ) + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) def test_14(self): """ @@ -1506,11 +1504,9 @@ def test_12(self): """ code = "10-2-12" number_inputs = 1 - exception_code = "1-1-19-12" + references_names = ["DS_r"] - self.NewSemanticExceptionTest( - code=code, number_inputs=number_inputs, exception_code=exception_code - ) + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) def test_13(self): """ diff --git a/tests/UDO/data/DataSet/output/GL_442_1-1.csv b/tests/UDO/data/DataSet/output/GL_442_1-1.csv index b0dab8d34..3c332b490 100644 --- a/tests/UDO/data/DataSet/output/GL_442_1-1.csv +++ b/tests/UDO/data/DataSet/output/GL_442_1-1.csv @@ -1,31 +1,31 @@ Entity.LegalAddress.PostalCode,int_var -VG1110, -KY1-1001, -0, -KY1-1108, -KY1-1002, -238891, -59817, -KY1-9008, -HM 12, -KY1-1104, -560042, -19801, -673310, -EC2N 1HQ, -19904, -89703-4934, -N/A for testing purposes, -KY1-9002, -KY1-1110, -M5X 1G5, -KY1-9005, -KY1-1100, -81677, -19808, -816, -1134, -MH 96960, -PO Box 3511, -P.O. Box 3511, -1087 HW, +VG1110,10 +KY1-1001,16 +0,1 +KY1-1108,1 +KY1-1002,2 +238891,1 +59817,2 +KY1-9008,4 +HM 12,1 +KY1-1104,16 +560042,1 +19801,1 +673310,1 +EC2N 1HQ,1 +19904,1 +89703-4934,1 +N/A for testing purposes,1 +KY1-9002,1 +KY1-1110,1 +M5X 1G5,1 +KY1-9005,3 +KY1-1100,1 +81677,1 +19808,1 +816,1 +1134,1 +MH 96960,1 +PO Box 3511,1 +P.O. Box 3511,1 +1087 HW,1 diff --git a/tests/UDO/data/DataSet/output/GL_442_3-1.csv b/tests/UDO/data/DataSet/output/GL_442_3-1.csv index 4a8a743a6..5e431c921 100644 --- a/tests/UDO/data/DataSet/output/GL_442_3-1.csv +++ b/tests/UDO/data/DataSet/output/GL_442_3-1.csv @@ -1,31 +1,31 @@ Entity.LegalAddress.PostalCode,int_var -VG1110, -KY1-1001, -0, -KY1-1108, -KY1-1002, -238891, -59817, -KY1-9008, -HM 12, -KY1-1104, -560042, -19801, -673310, -EC2N 1HQ, -19904, -89703-4934, -N/A for tests, -KY1-9002, -KY1-1110, -M5X 1G5, -KY1-9005, -KY1-1100, -81677, -19808, -816, -1134, -MH 96960, -PO Box 3511, -P.O. Box 3511, -1087 HW, +VG1110,10 +KY1-1001,16 +0,1 +KY1-1108,1 +KY1-1002,2 +238891,1 +59817,2 +KY1-9008,4 +HM 12,1 +KY1-1104,16 +560042,1 +19801,1 +673310,1 +EC2N 1HQ,1 +19904,1 +89703-4934,1 +N/A for tests,1 +KY1-9002,1 +KY1-1110,1 +M5X 1G5,1 +KY1-9005,3 +KY1-1100,1 +81677,1 +19808,1 +816,1 +1134,1 +MH 96960,1 +PO Box 3511,1 +P.O. Box 3511,1 +1087 HW,1 diff --git a/tests/UDO/data/DataSet/output/GL_442_3-2.csv b/tests/UDO/data/DataSet/output/GL_442_3-2.csv index 078368f85..8850350c8 100644 --- a/tests/UDO/data/DataSet/output/GL_442_3-2.csv +++ b/tests/UDO/data/DataSet/output/GL_442_3-2.csv @@ -1,20 +1,20 @@ Entity.LegalForm.EntityLegalFormCode,int_var -6EH6, -8888, -6TPA, -XAQA, -6XB7, -LWXI, -ASR5, -OSBR, -YSP9, -XTIQ, -7QEH, -9999, -HZEH, -UDLA, -2HBR, -T91T, -P9F2, -EZL8, -CODH, +6EH6,8 +8888,18 +6TPA,2 +XAQA,11 +6XB7,19 +LWXI,2 +ASR5,1 +OSBR,3 +YSP9,1 +XTIQ,1 +7QEH,2 +9999,1 +HZEH,1 +UDLA,1 +2HBR,1 +T91T,1 +P9F2,1 +EZL8,1 +CODH,1 diff --git a/tests/UDO/data/DataSet/output/GL_442_4-1.csv b/tests/UDO/data/DataSet/output/GL_442_4-1.csv index 6e563f8c4..01a2f439f 100644 --- a/tests/UDO/data/DataSet/output/GL_442_4-1.csv +++ b/tests/UDO/data/DataSet/output/GL_442_4-1.csv @@ -1,31 +1,31 @@ Entity.LegalAddress.PostalCode,int_var -VG1110, -KY1-1001, -0, -KY1-1108, -KY1-1002, -238891, -59817, -KY1-9008, -HM 12, -KY1-1104, -560042, -19801, -673310, -EC2N 1HQ, -19904, -89703-4934, -N/A for test purposes, -KY1-9002, -KY1-1110, -M5X 1G5, -KY1-9005, -KY1-1100, -81677, -19808, -816, -1134, -MH 96960, -PO Box 3511, -P.O. Box 3511, -1087 HW, +VG1110,10 +KY1-1001,16 +0,1 +KY1-1108,1 +KY1-1002,2 +238891,1 +59817,2 +KY1-9008,4 +HM 12,1 +KY1-1104,16 +560042,1 +19801,1 +673310,1 +EC2N 1HQ,1 +19904,1 +89703-4934,1 +N/A for test purposes,1 +KY1-9002,1 +KY1-1110,1 +M5X 1G5,1 +KY1-9005,3 +KY1-1100,1 +81677,1 +19808,1 +816,1 +1134,1 +MH 96960,1 +PO Box 3511,1 +P.O. Box 3511,1 +1087 HW,1 diff --git a/tests/UDO/data/DataSet/output/GL_442_4-2.csv b/tests/UDO/data/DataSet/output/GL_442_4-2.csv index 078368f85..8850350c8 100644 --- a/tests/UDO/data/DataSet/output/GL_442_4-2.csv +++ b/tests/UDO/data/DataSet/output/GL_442_4-2.csv @@ -1,20 +1,20 @@ Entity.LegalForm.EntityLegalFormCode,int_var -6EH6, -8888, -6TPA, -XAQA, -6XB7, -LWXI, -ASR5, -OSBR, -YSP9, -XTIQ, -7QEH, -9999, -HZEH, -UDLA, -2HBR, -T91T, -P9F2, -EZL8, -CODH, +6EH6,8 +8888,18 +6TPA,2 +XAQA,11 +6XB7,19 +LWXI,2 +ASR5,1 +OSBR,3 +YSP9,1 +XTIQ,1 +7QEH,2 +9999,1 +HZEH,1 +UDLA,1 +2HBR,1 +T91T,1 +P9F2,1 +EZL8,1 +CODH,1 diff --git a/tests/UDO/data/DataSet/output/GL_442_5-3.csv b/tests/UDO/data/DataSet/output/GL_442_5-3.csv index 078368f85..8850350c8 100644 --- a/tests/UDO/data/DataSet/output/GL_442_5-3.csv +++ b/tests/UDO/data/DataSet/output/GL_442_5-3.csv @@ -1,20 +1,20 @@ Entity.LegalForm.EntityLegalFormCode,int_var -6EH6, -8888, -6TPA, -XAQA, -6XB7, -LWXI, -ASR5, -OSBR, -YSP9, -XTIQ, -7QEH, -9999, -HZEH, -UDLA, -2HBR, -T91T, -P9F2, -EZL8, -CODH, +6EH6,8 +8888,18 +6TPA,2 +XAQA,11 +6XB7,19 +LWXI,2 +ASR5,1 +OSBR,3 +YSP9,1 +XTIQ,1 +7QEH,2 +9999,1 +HZEH,1 +UDLA,1 +2HBR,1 +T91T,1 +P9F2,1 +EZL8,1 +CODH,1 diff --git a/tests/UDO/data/DataSet/output/GL_442_5-4.csv b/tests/UDO/data/DataSet/output/GL_442_5-4.csv index 6e563f8c4..01a2f439f 100644 --- a/tests/UDO/data/DataSet/output/GL_442_5-4.csv +++ b/tests/UDO/data/DataSet/output/GL_442_5-4.csv @@ -1,31 +1,31 @@ Entity.LegalAddress.PostalCode,int_var -VG1110, -KY1-1001, -0, -KY1-1108, -KY1-1002, -238891, -59817, -KY1-9008, -HM 12, -KY1-1104, -560042, -19801, -673310, -EC2N 1HQ, -19904, -89703-4934, -N/A for test purposes, -KY1-9002, -KY1-1110, -M5X 1G5, -KY1-9005, -KY1-1100, -81677, -19808, -816, -1134, -MH 96960, -PO Box 3511, -P.O. Box 3511, -1087 HW, +VG1110,10 +KY1-1001,16 +0,1 +KY1-1108,1 +KY1-1002,2 +238891,1 +59817,2 +KY1-9008,4 +HM 12,1 +KY1-1104,16 +560042,1 +19801,1 +673310,1 +EC2N 1HQ,1 +19904,1 +89703-4934,1 +N/A for test purposes,1 +KY1-9002,1 +KY1-1110,1 +M5X 1G5,1 +KY1-9005,3 +KY1-1100,1 +81677,1 +19808,1 +816,1 +1134,1 +MH 96960,1 +PO Box 3511,1 +P.O. Box 3511,1 +1087 HW,1 From a46612c5112fd7a9bd3c765067e6e4eb712dd7d6 Mon Sep 17 00:00:00 2001 From: Alberto Date: Fri, 31 Jul 2026 13:27:19 +0200 Subject: [PATCH 2/4] Fix #937: count accepts any Measure type and counts Data Points --- src/vtlengine/duckdb_transpiler/Transpiler/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py b/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py index 311bc6ca1..6c8526b73 100644 --- a/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py +++ b/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py @@ -2219,8 +2219,6 @@ def visit_Aggregation(self, node: AST.Aggregation) -> str: # type: ignore[overr # count() without operand if node.operand is None: if op == tokens.COUNT: - # count() without an operand counts Data Points, so a Data Point whose - # Measures are null still contributes (issue #937). return "NULLIF(COUNT(*), 0)" return "" From cb76a11a326312f3e00d22bbd82d7184ba573f73 Mon Sep 17 00:00:00 2001 From: Alberto Date: Mon, 3 Aug 2026 13:05:00 +0200 Subject: [PATCH 3/4] Drop the count Data Point semantics from #937 count no longer rejecting a Time Measure is the fix #937 asked for. Changing what count returns for a Data Point with a null Measure is a separate, breaking change and is tracked on its own; it is reverted here together with the reference Data Sets it required. What remains is the guard exemption, its regression test, and the two type-checking tests that asserted the error and now assert a result against their original references. --- src/vtlengine/Interpreter/__init__.py | 11 +-- src/vtlengine/Operators/Aggregation.py | 18 +--- .../duckdb_transpiler/Transpiler/__init__.py | 20 +++- .../data/DataSet/output/GL_222_1-1.csv | 2 +- .../data/DataSet/output/GL_466_1-3.csv | 28 +++--- .../data/DataSet/output/GL_466_2-1.csv | 8 +- .../output/DEMO1-aggr.numDPCouYear.csv | 96 +++++++++---------- tests/Bugs/data/DataSet/output/GL_270_2-1.csv | 4 +- tests/DWI/data/DataSet/output/GL_218_7-1.csv | 2 +- tests/DWI/data/DataSet/output/GL_218_8-1.csv | 2 +- .../data/DataSet/output/10-1-12-DS_r.csv | 2 +- .../data/DataSet/output/10-1-13-DS_r.csv | 2 +- .../data/DataSet/output/10-1-14-DS_r.csv | 2 +- .../data/DataSet/output/10-1-15-DS_r.csv | 2 +- .../data/DataSet/output/10-1-16-DS_r.csv | 2 +- .../data/DataSet/output/10-1-17-DS_r.csv | 2 +- .../data/DataSet/output/10-1-18-DS_r.csv | 2 +- .../data/DataSet/output/10-1-19-DS_r.csv | 4 +- tests/UDO/data/DataSet/output/GL_442_1-1.csv | 60 ++++++------ tests/UDO/data/DataSet/output/GL_442_3-1.csv | 60 ++++++------ tests/UDO/data/DataSet/output/GL_442_3-2.csv | 38 ++++---- tests/UDO/data/DataSet/output/GL_442_4-1.csv | 60 ++++++------ tests/UDO/data/DataSet/output/GL_442_4-2.csv | 38 ++++---- tests/UDO/data/DataSet/output/GL_442_5-3.csv | 38 ++++---- tests/UDO/data/DataSet/output/GL_442_5-4.csv | 60 ++++++------ 25 files changed, 279 insertions(+), 284 deletions(-) diff --git a/src/vtlengine/Interpreter/__init__.py b/src/vtlengine/Interpreter/__init__.py index b3ab06f23..cd46054c2 100644 --- a/src/vtlengine/Interpreter/__init__.py +++ b/src/vtlengine/Interpreter/__init__.py @@ -666,16 +666,7 @@ def visit_Aggregation(self, node: AST.Aggregation) -> None: # Setting here group by as we have already selected the identifiers we need grouping_op = "group by" - # count over a Component counts that Component's non-null values, while count - # over a Data Set counts Data Points; the manual gives them separate syntaxes. - component_operand = ( - not self.is_from_having - and self.is_from_regular_aggregation - and node.operand is not None - ) - result = AGGREGATION_MAPPING[node.op].analyze( - operand, grouping_op, groupings, having, component_operand - ) + result = AGGREGATION_MAPPING[node.op].analyze(operand, grouping_op, groupings, having) if not self.is_from_regular_aggregation: result.name = VirtualCounter._new_ds_name() return result diff --git a/src/vtlengine/Operators/Aggregation.py b/src/vtlengine/Operators/Aggregation.py index c45679afd..33178cd99 100644 --- a/src/vtlengine/Operators/Aggregation.py +++ b/src/vtlengine/Operators/Aggregation.py @@ -112,7 +112,6 @@ def validate( # type: ignore[override] group_op: Optional[str], grouping_columns: Any, having_data: Any, - component_operand: bool = False, ) -> Dataset: result_components = {k: copy(v) for k, v in operand.components.items()} if cls.op not in [COUNT, MIN, MAX] and len(operand.get_measures_names()) == 0: @@ -148,7 +147,8 @@ def validate( # type: ignore[override] for comp_name, comp in operand.components.items(): if comp.role == Role.ATTRIBUTE: del result_components[comp_name] - if cls.op != COUNT and any( + # TimeInterval is not supported as a measure in aggregate operations + if any( comp.role == Role.MEASURE and comp.data_type is TimeInterval for comp in result_components.values() ): @@ -190,7 +190,6 @@ def _agg_func( grouping_keys: Optional[List[str]], measure_names: Optional[List[str]], having_expression: Optional[str], - component_operand: bool = False, ) -> pd.DataFrame: grouping_names = ( [f'"{name}"' for name in grouping_keys] if grouping_keys is not None else None @@ -227,11 +226,7 @@ def _agg_func( f"{cls.py_op}(CAST({e} AS DOUBLE)) AS {e}, " # Count can only be one here ) elif cls.op == COUNT: - functions += ( - f"{cls.py_op}({e}) AS int_var, " - if component_operand - else "COUNT(*) AS int_var, " - ) + functions += f"{cls.py_op}({e}) AS int_var, " break else: functions += f"{cls.py_op}({e}) AS {e}, " @@ -268,7 +263,6 @@ def evaluate( # type: ignore[override] group_op: Optional[str], grouping_columns: Optional[List[str]], having_expr: Optional[str], - component_operand: bool = False, ) -> Dataset: result = cls.validate(operand, group_op, grouping_columns, having_expr) @@ -279,16 +273,14 @@ def evaluate( # type: ignore[override] # Keep a copy of viral attrs for post-aggregation propagation viral_df = result_df[grouping_keys + viral_attr_names].copy() if viral_attr_names else None result_df = result_df[grouping_keys + measure_names] - if cls.op == COUNT and component_operand: + if cls.op == COUNT: result_df = result_df.dropna(subset=measure_names, how="any") if cls.op in [MAX, MIN]: for measure in operand.get_measures(): if measure.data_type == TimeInterval: raise RunTimeError("2-1-19-18", op=cls.op) cls._handle_data_types(result_df, operand.get_measures(), "input") - result_df = cls._agg_func( - result_df, grouping_keys, measure_names, having_expr, component_operand - ) + result_df = cls._agg_func(result_df, grouping_keys, measure_names, having_expr) cls._handle_data_types(result_df, operand.get_measures(), "result") # Handle correct order on result diff --git a/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py b/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py index 6c8526b73..1bee4b314 100644 --- a/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py +++ b/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py @@ -2219,6 +2219,11 @@ def visit_Aggregation(self, node: AST.Aggregation) -> str: # type: ignore[overr # count() without operand if node.operand is None: if op == tokens.COUNT: + if self._in_clause and self._current_dataset: + measures = self._current_dataset.get_measures_names() + if measures: + or_parts = " OR ".join(f"{quote_name(m)} IS NOT NULL" for m in measures) + return f"NULLIF(COUNT(CASE WHEN {or_parts} THEN 1 END), 0)" return "NULLIF(COUNT(*), 0)" return "" @@ -2235,11 +2240,18 @@ def visit_Aggregation(self, node: AST.Aggregation) -> str: # type: ignore[overr cols, group_by_cols = self._build_agg_group_cols(node, ds, group_cols) ds_tp_minmax_cols: List[tuple[str, str]] = [] - # count() produces a single int_var measure. It reports the number of Data - # Points, so a Data Point is counted even where one of its Measures is null - # (issue #937); a group that exists always holds at least one of them. + # count() produces a single int_var measure. if op == tokens.COUNT: - cols.append(f"COUNT(*) AS {quote_name('int_var')}") + alias = "int_var" + source_measures = ds.get_measures_names() + if source_measures: + and_parts = " AND ".join(f"{quote_name(m)} IS NOT NULL" for m in source_measures) + count_expr = f"COUNT(CASE WHEN {and_parts} THEN 1 END)" + if group_cols: + count_expr = f"NULLIF({count_expr}, 0)" + cols.append(f"{count_expr} AS {quote_name(alias)}") + else: + cols.append(f"COUNT(*) AS {quote_name(alias)}") else: measures = ds.get_measures_names() for measure in measures: diff --git a/tests/Additional/data/DataSet/output/GL_222_1-1.csv b/tests/Additional/data/DataSet/output/GL_222_1-1.csv index f46417126..65ebeeab8 100644 --- a/tests/Additional/data/DataSet/output/GL_222_1-1.csv +++ b/tests/Additional/data/DataSet/output/GL_222_1-1.csv @@ -1,4 +1,4 @@ Id_1,Me_3,Me_4 -1,2,2 +1,1,1 2,3,3 diff --git a/tests/Aggregate/data/DataSet/output/GL_466_1-3.csv b/tests/Aggregate/data/DataSet/output/GL_466_1-3.csv index fe199020d..bf7f01d44 100644 --- a/tests/Aggregate/data/DataSet/output/GL_466_1-3.csv +++ b/tests/Aggregate/data/DataSet/output/GL_466_1-3.csv @@ -1,16 +1,16 @@ month,int_var -2023-01,12 -2023-02,28 -2023-03,31 -2023-04,30 -2023-05,31 -2023-06,30 -2023-07,31 -2023-08,31 -2023-09,30 -2023-10,31 -2023-11,30 -2023-12,31 -2024-01,31 -2024-02,29 +2023-01,1 +2023-02, +2023-03, +2023-04, +2023-05, +2023-06, +2023-07, +2023-08, +2023-09, +2023-10, +2023-11, +2023-12, +2024-01, +2024-02,2 2024-03,1 diff --git a/tests/Aggregate/data/DataSet/output/GL_466_2-1.csv b/tests/Aggregate/data/DataSet/output/GL_466_2-1.csv index 4cfea2c1d..3a06ef915 100644 --- a/tests/Aggregate/data/DataSet/output/GL_466_2-1.csv +++ b/tests/Aggregate/data/DataSet/output/GL_466_2-1.csv @@ -1,5 +1,5 @@ month,int_var -2023-01,2 -2024-02,6 -2024-03,1 -2024-04,1 +2023-01, +2024-02, +2024-03, +2024-04,1.0 \ No newline at end of file diff --git a/tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv b/tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv index ccca13765..4da8d0a5f 100644 --- a/tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv +++ b/tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv @@ -1,97 +1,97 @@ REF_DATE,REP_COUNTRY,int_var -2018-12-31,CN,14 +2018-12-31,CN,6 2018-12-31,AU,14 -2018-12-31,FI,14 -2018-12-31,TR,14 -2018-12-31,US,14 -2018-12-31,BM,14 +2018-12-31,FI,13 +2018-12-31,TR,12 +2018-12-31,US,12 +2018-12-31,BM,12 2018-12-31,GG,14 -2018-12-31,CY,14 -2018-12-31,ID,14 +2018-12-31,CY,13 +2018-12-31,ID,12 2018-12-31,5A,14 2018-12-31,KY,14 -2018-12-31,BS,14 +2018-12-31,BS,12 2018-12-31,RU,14 2018-12-31,KR,14 -2018-12-31,GR,7 -2018-12-31,SE,14 +2018-12-31,GR,6 +2018-12-31,SE,12 2018-12-31,DE,14 -2018-12-31,SG,9 -2018-12-31,JP,14 +2018-12-31,SG,7 +2018-12-31,JP,13 2018-12-31,IT,14 2018-12-31,NL,14 -2018-12-31,CW,7 -2018-12-31,HK,14 +2018-12-31,CW,6 +2018-12-31,HK,8 2018-12-31,ZA,14 2018-12-31,IM,14 2018-12-31,BE,14 -2018-12-31,MY,14 +2018-12-31,MY,4 2018-12-31,MO,14 2018-12-31,AT,14 2018-12-31,LU,14 2018-12-31,PH,14 2018-12-31,CH,14 -2018-12-31,PT,14 -2018-12-31,DK,14 -2018-12-31,IN,14 -2018-12-31,JE,9 +2018-12-31,PT,11 +2018-12-31,DK,11 +2018-12-31,IN,6 +2018-12-31,JE,8 2018-12-31,NO,14 2018-12-31,GB,14 2018-12-31,TW,14 -2018-12-31,PA,10 -2018-12-31,CL,10 -2018-12-31,CA,14 +2018-12-31,PA,7 +2018-12-31,CL,8 +2018-12-31,CA,13 2018-12-31,ES,14 2018-12-31,IE,14 -2018-12-31,BR,9 +2018-12-31,BR,8 2018-12-31,MX,6 2018-12-31,BH,6 2018-12-31,FR,14 -2019-03-31,CN,14 +2019-03-31,CN,7 2019-03-31,AU,14 -2019-03-31,FI,14 -2019-03-31,TR,14 -2019-03-31,US,14 -2019-03-31,BM,14 +2019-03-31,FI,13 +2019-03-31,TR,12 +2019-03-31,US,13 +2019-03-31,BM,13 2019-03-31,GG,14 -2019-03-31,CY,14 -2019-03-31,ID,14 +2019-03-31,CY,13 +2019-03-31,ID,12 2019-03-31,5A,14 2019-03-31,KY,14 -2019-03-31,BS,14 +2019-03-31,BS,12 2019-03-31,RU,14 2019-03-31,KR,14 -2019-03-31,GR,7 -2019-03-31,SE,14 +2019-03-31,GR,6 +2019-03-31,SE,13 2019-03-31,DE,14 -2019-03-31,SG,9 -2019-03-31,JP,14 +2019-03-31,SG,7 +2019-03-31,JP,13 2019-03-31,IT,14 2019-03-31,NL,14 -2019-03-31,CW,7 -2019-03-31,HK,14 +2019-03-31,CW,6 +2019-03-31,HK,8 2019-03-31,ZA,14 -2019-03-31,IM,14 +2019-03-31,IM,13 2019-03-31,BE,14 -2019-03-31,MY,14 +2019-03-31,MY,4 2019-03-31,MO,14 -2019-03-31,AT,14 +2019-03-31,AT,13 2019-03-31,LU,14 2019-03-31,PH,14 2019-03-31,CH,14 -2019-03-31,PT,14 -2019-03-31,DK,14 -2019-03-31,IN,14 -2019-03-31,JE,9 +2019-03-31,PT,11 +2019-03-31,DK,11 +2019-03-31,IN,6 +2019-03-31,JE,8 2019-03-31,NO,14 2019-03-31,GB,14 2019-03-31,TW,14 -2019-03-31,PA,10 -2019-03-31,CL,10 -2019-03-31,CA,14 +2019-03-31,PA,6 +2019-03-31,CL,8 +2019-03-31,CA,13 2019-03-31,ES,14 2019-03-31,IE,14 -2019-03-31,BR,9 +2019-03-31,BR,8 2019-03-31,MX,6 2019-03-31,BH,6 2019-03-31,FR,14 diff --git a/tests/Bugs/data/DataSet/output/GL_270_2-1.csv b/tests/Bugs/data/DataSet/output/GL_270_2-1.csv index 91ea776dc..14a6f59d5 100644 --- a/tests/Bugs/data/DataSet/output/GL_270_2-1.csv +++ b/tests/Bugs/data/DataSet/output/GL_270_2-1.csv @@ -1,5 +1,5 @@ CNTRCT_ID,DT_RFRNC,INSTRMNT_ID,OBSRVD_AGNT_CD,JNT_LBLTY_AMNT_SUM,NMBR_DBTR,IS_JNT_LBLTY_RPRTD_ALL,JNT_LBLTY_AMNT_MAX -AAA,2020-01-01,AAA,AAA,101.0,3,False,100.0 +AAA,2020-01-01,AAA,AAA,101.0,2,False,100.0 BBB,2020-01-01,BBB,BBB,12.0,2,True,11.0 -CCC,2020-01-01,BBB,BBB,,1,False, +CCC,2020-01-01,BBB,BBB,,,False, DDD,2020-01-01,DDD,BBB,0.0,1,True,0.0 diff --git a/tests/DWI/data/DataSet/output/GL_218_7-1.csv b/tests/DWI/data/DataSet/output/GL_218_7-1.csv index 54858b568..354b08a35 100644 --- a/tests/DWI/data/DataSet/output/GL_218_7-1.csv +++ b/tests/DWI/data/DataSet/output/GL_218_7-1.csv @@ -1,2 +1,2 @@ int_var -1 +0 diff --git a/tests/DWI/data/DataSet/output/GL_218_8-1.csv b/tests/DWI/data/DataSet/output/GL_218_8-1.csv index 54858b568..231e26d86 100644 --- a/tests/DWI/data/DataSet/output/GL_218_8-1.csv +++ b/tests/DWI/data/DataSet/output/GL_218_8-1.csv @@ -1,2 +1,2 @@ int_var -1 +0 \ No newline at end of file diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-12-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-12-DS_r.csv index b0f4b613a..a4a9a71e4 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-12-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-12-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,2 +2,1 \ No newline at end of file diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-13-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-13-DS_r.csv index b0f4b613a..a4a9a71e4 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-13-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-13-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,2 +2,1 \ No newline at end of file diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-14-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-14-DS_r.csv index b0f4b613a..a4a9a71e4 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-14-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-14-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,2 +2,1 \ No newline at end of file diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-15-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-15-DS_r.csv index b0f4b613a..a4a9a71e4 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-15-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-15-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,2 +2,1 \ No newline at end of file diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-16-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-16-DS_r.csv index b0f4b613a..a4a9a71e4 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-16-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-16-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,2 +2,1 \ No newline at end of file diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-17-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-17-DS_r.csv index b0f4b613a..a4a9a71e4 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-17-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-17-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,2 +2,1 \ No newline at end of file diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-18-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-18-DS_r.csv index b15a6807a..cd3ee94f9 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-18-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-18-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var 1,2 -2,3 +2,2 \ No newline at end of file diff --git a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-19-DS_r.csv b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-19-DS_r.csv index b15a6807a..a92971305 100644 --- a/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-19-DS_r.csv +++ b/tests/TypeChecking/AggregateOperators/data/DataSet/output/10-1-19-DS_r.csv @@ -1,3 +1,3 @@ Id_1,int_var -1,2 -2,3 +1, +2, \ No newline at end of file diff --git a/tests/UDO/data/DataSet/output/GL_442_1-1.csv b/tests/UDO/data/DataSet/output/GL_442_1-1.csv index 3c332b490..b0dab8d34 100644 --- a/tests/UDO/data/DataSet/output/GL_442_1-1.csv +++ b/tests/UDO/data/DataSet/output/GL_442_1-1.csv @@ -1,31 +1,31 @@ Entity.LegalAddress.PostalCode,int_var -VG1110,10 -KY1-1001,16 -0,1 -KY1-1108,1 -KY1-1002,2 -238891,1 -59817,2 -KY1-9008,4 -HM 12,1 -KY1-1104,16 -560042,1 -19801,1 -673310,1 -EC2N 1HQ,1 -19904,1 -89703-4934,1 -N/A for testing purposes,1 -KY1-9002,1 -KY1-1110,1 -M5X 1G5,1 -KY1-9005,3 -KY1-1100,1 -81677,1 -19808,1 -816,1 -1134,1 -MH 96960,1 -PO Box 3511,1 -P.O. Box 3511,1 -1087 HW,1 +VG1110, +KY1-1001, +0, +KY1-1108, +KY1-1002, +238891, +59817, +KY1-9008, +HM 12, +KY1-1104, +560042, +19801, +673310, +EC2N 1HQ, +19904, +89703-4934, +N/A for testing purposes, +KY1-9002, +KY1-1110, +M5X 1G5, +KY1-9005, +KY1-1100, +81677, +19808, +816, +1134, +MH 96960, +PO Box 3511, +P.O. Box 3511, +1087 HW, diff --git a/tests/UDO/data/DataSet/output/GL_442_3-1.csv b/tests/UDO/data/DataSet/output/GL_442_3-1.csv index 5e431c921..4a8a743a6 100644 --- a/tests/UDO/data/DataSet/output/GL_442_3-1.csv +++ b/tests/UDO/data/DataSet/output/GL_442_3-1.csv @@ -1,31 +1,31 @@ Entity.LegalAddress.PostalCode,int_var -VG1110,10 -KY1-1001,16 -0,1 -KY1-1108,1 -KY1-1002,2 -238891,1 -59817,2 -KY1-9008,4 -HM 12,1 -KY1-1104,16 -560042,1 -19801,1 -673310,1 -EC2N 1HQ,1 -19904,1 -89703-4934,1 -N/A for tests,1 -KY1-9002,1 -KY1-1110,1 -M5X 1G5,1 -KY1-9005,3 -KY1-1100,1 -81677,1 -19808,1 -816,1 -1134,1 -MH 96960,1 -PO Box 3511,1 -P.O. Box 3511,1 -1087 HW,1 +VG1110, +KY1-1001, +0, +KY1-1108, +KY1-1002, +238891, +59817, +KY1-9008, +HM 12, +KY1-1104, +560042, +19801, +673310, +EC2N 1HQ, +19904, +89703-4934, +N/A for tests, +KY1-9002, +KY1-1110, +M5X 1G5, +KY1-9005, +KY1-1100, +81677, +19808, +816, +1134, +MH 96960, +PO Box 3511, +P.O. Box 3511, +1087 HW, diff --git a/tests/UDO/data/DataSet/output/GL_442_3-2.csv b/tests/UDO/data/DataSet/output/GL_442_3-2.csv index 8850350c8..078368f85 100644 --- a/tests/UDO/data/DataSet/output/GL_442_3-2.csv +++ b/tests/UDO/data/DataSet/output/GL_442_3-2.csv @@ -1,20 +1,20 @@ Entity.LegalForm.EntityLegalFormCode,int_var -6EH6,8 -8888,18 -6TPA,2 -XAQA,11 -6XB7,19 -LWXI,2 -ASR5,1 -OSBR,3 -YSP9,1 -XTIQ,1 -7QEH,2 -9999,1 -HZEH,1 -UDLA,1 -2HBR,1 -T91T,1 -P9F2,1 -EZL8,1 -CODH,1 +6EH6, +8888, +6TPA, +XAQA, +6XB7, +LWXI, +ASR5, +OSBR, +YSP9, +XTIQ, +7QEH, +9999, +HZEH, +UDLA, +2HBR, +T91T, +P9F2, +EZL8, +CODH, diff --git a/tests/UDO/data/DataSet/output/GL_442_4-1.csv b/tests/UDO/data/DataSet/output/GL_442_4-1.csv index 01a2f439f..6e563f8c4 100644 --- a/tests/UDO/data/DataSet/output/GL_442_4-1.csv +++ b/tests/UDO/data/DataSet/output/GL_442_4-1.csv @@ -1,31 +1,31 @@ Entity.LegalAddress.PostalCode,int_var -VG1110,10 -KY1-1001,16 -0,1 -KY1-1108,1 -KY1-1002,2 -238891,1 -59817,2 -KY1-9008,4 -HM 12,1 -KY1-1104,16 -560042,1 -19801,1 -673310,1 -EC2N 1HQ,1 -19904,1 -89703-4934,1 -N/A for test purposes,1 -KY1-9002,1 -KY1-1110,1 -M5X 1G5,1 -KY1-9005,3 -KY1-1100,1 -81677,1 -19808,1 -816,1 -1134,1 -MH 96960,1 -PO Box 3511,1 -P.O. Box 3511,1 -1087 HW,1 +VG1110, +KY1-1001, +0, +KY1-1108, +KY1-1002, +238891, +59817, +KY1-9008, +HM 12, +KY1-1104, +560042, +19801, +673310, +EC2N 1HQ, +19904, +89703-4934, +N/A for test purposes, +KY1-9002, +KY1-1110, +M5X 1G5, +KY1-9005, +KY1-1100, +81677, +19808, +816, +1134, +MH 96960, +PO Box 3511, +P.O. Box 3511, +1087 HW, diff --git a/tests/UDO/data/DataSet/output/GL_442_4-2.csv b/tests/UDO/data/DataSet/output/GL_442_4-2.csv index 8850350c8..078368f85 100644 --- a/tests/UDO/data/DataSet/output/GL_442_4-2.csv +++ b/tests/UDO/data/DataSet/output/GL_442_4-2.csv @@ -1,20 +1,20 @@ Entity.LegalForm.EntityLegalFormCode,int_var -6EH6,8 -8888,18 -6TPA,2 -XAQA,11 -6XB7,19 -LWXI,2 -ASR5,1 -OSBR,3 -YSP9,1 -XTIQ,1 -7QEH,2 -9999,1 -HZEH,1 -UDLA,1 -2HBR,1 -T91T,1 -P9F2,1 -EZL8,1 -CODH,1 +6EH6, +8888, +6TPA, +XAQA, +6XB7, +LWXI, +ASR5, +OSBR, +YSP9, +XTIQ, +7QEH, +9999, +HZEH, +UDLA, +2HBR, +T91T, +P9F2, +EZL8, +CODH, diff --git a/tests/UDO/data/DataSet/output/GL_442_5-3.csv b/tests/UDO/data/DataSet/output/GL_442_5-3.csv index 8850350c8..078368f85 100644 --- a/tests/UDO/data/DataSet/output/GL_442_5-3.csv +++ b/tests/UDO/data/DataSet/output/GL_442_5-3.csv @@ -1,20 +1,20 @@ Entity.LegalForm.EntityLegalFormCode,int_var -6EH6,8 -8888,18 -6TPA,2 -XAQA,11 -6XB7,19 -LWXI,2 -ASR5,1 -OSBR,3 -YSP9,1 -XTIQ,1 -7QEH,2 -9999,1 -HZEH,1 -UDLA,1 -2HBR,1 -T91T,1 -P9F2,1 -EZL8,1 -CODH,1 +6EH6, +8888, +6TPA, +XAQA, +6XB7, +LWXI, +ASR5, +OSBR, +YSP9, +XTIQ, +7QEH, +9999, +HZEH, +UDLA, +2HBR, +T91T, +P9F2, +EZL8, +CODH, diff --git a/tests/UDO/data/DataSet/output/GL_442_5-4.csv b/tests/UDO/data/DataSet/output/GL_442_5-4.csv index 01a2f439f..6e563f8c4 100644 --- a/tests/UDO/data/DataSet/output/GL_442_5-4.csv +++ b/tests/UDO/data/DataSet/output/GL_442_5-4.csv @@ -1,31 +1,31 @@ Entity.LegalAddress.PostalCode,int_var -VG1110,10 -KY1-1001,16 -0,1 -KY1-1108,1 -KY1-1002,2 -238891,1 -59817,2 -KY1-9008,4 -HM 12,1 -KY1-1104,16 -560042,1 -19801,1 -673310,1 -EC2N 1HQ,1 -19904,1 -89703-4934,1 -N/A for test purposes,1 -KY1-9002,1 -KY1-1110,1 -M5X 1G5,1 -KY1-9005,3 -KY1-1100,1 -81677,1 -19808,1 -816,1 -1134,1 -MH 96960,1 -PO Box 3511,1 -P.O. Box 3511,1 -1087 HW,1 +VG1110, +KY1-1001, +0, +KY1-1108, +KY1-1002, +238891, +59817, +KY1-9008, +HM 12, +KY1-1104, +560042, +19801, +673310, +EC2N 1HQ, +19904, +89703-4934, +N/A for test purposes, +KY1-9002, +KY1-1110, +M5X 1G5, +KY1-9005, +KY1-1100, +81677, +19808, +816, +1134, +MH 96960, +PO Box 3511, +P.O. Box 3511, +1087 HW, From 959126a90b05a34605df64edfb95c4f99e7ef47c Mon Sep 17 00:00:00 2001 From: Alberto Date: Mon, 3 Aug 2026 13:05:33 +0200 Subject: [PATCH 4/4] Drop the count Data Point semantics from #937 count no longer rejecting a Time Measure is the fix #937 asked for. Changing what count returns for a Data Point with a null Measure is a separate, breaking change and is tracked on its own; it is reverted here together with the reference Data Sets it required. What remains is the guard exemption, its regression test, and the two type-checking tests that asserted the error and now assert a result against their original references. --- src/vtlengine/Operators/Aggregation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vtlengine/Operators/Aggregation.py b/src/vtlengine/Operators/Aggregation.py index 33178cd99..095927356 100644 --- a/src/vtlengine/Operators/Aggregation.py +++ b/src/vtlengine/Operators/Aggregation.py @@ -147,8 +147,10 @@ def validate( # type: ignore[override] for comp_name, comp in operand.components.items(): if comp.role == Role.ATTRIBUTE: del result_components[comp_name] - # TimeInterval is not supported as a measure in aggregate operations - if any( + # TimeInterval is not supported as a measure in aggregate operations. + # count is exempt: it only reports the number of Data Points, so the Measures it + # is given are replaced by int_var below rather than aggregated (issue #937). + if cls.op != COUNT and any( comp.role == Role.MEASURE and comp.data_type is TimeInterval for comp in result_components.values() ):