Fix #937: count accepts any Measure type and counts Data Points

src/vtlengine/Interpreter/__init__.py

@@ -666,7 +666,16 @@ def visit_Aggregation(self, node: AST.Aggregation) -> None:
             # Setting here group by as we have already selected the identifiers we need
             grouping_op = "group by"
 
-        result = AGGREGATION_MAPPING[node.op].analyze(operand, grouping_op, groupings, having)
+        # count over a Component counts that Component's non-null values, while count
+        # over a Data Set counts Data Points; the manual gives them separate syntaxes.
+        component_operand = (
+            not self.is_from_having
+            and self.is_from_regular_aggregation
+            and node.operand is not None
+        )
+        result = AGGREGATION_MAPPING[node.op].analyze(
+            operand, grouping_op, groupings, having, component_operand
+        )
         if not self.is_from_regular_aggregation:
             result.name = VirtualCounter._new_ds_name()
         return result

src/vtlengine/Operators/Aggregation.py

@@ -112,6 +112,7 @@ def validate(  # type: ignore[override]
         group_op: Optional[str],
         grouping_columns: Any,
         having_data: Any,
+        component_operand: bool = False,
     ) -> Dataset:
         result_components = {k: copy(v) for k, v in operand.components.items()}
         if cls.op not in [COUNT, MIN, MAX] and len(operand.get_measures_names()) == 0:
@@ -147,8 +148,7 @@ def validate(  # type: ignore[override]
         for comp_name, comp in operand.components.items():
             if comp.role == Role.ATTRIBUTE:
                 del result_components[comp_name]
-        # TimeInterval is not supported as a measure in aggregate operations
-        if any(
+        if cls.op != COUNT and any(
             comp.role == Role.MEASURE and comp.data_type is TimeInterval
             for comp in result_components.values()
         ):
@@ -190,6 +190,7 @@ def _agg_func(
         grouping_keys: Optional[List[str]],
         measure_names: Optional[List[str]],
         having_expression: Optional[str],
+        component_operand: bool = False,
     ) -> pd.DataFrame:
         grouping_names = (
             [f'"{name}"' for name in grouping_keys] if grouping_keys is not None else None
@@ -226,7 +227,11 @@ def _agg_func(
                         f"{cls.py_op}(CAST({e} AS DOUBLE)) AS {e}, "  # Count can only be one here
                     )
                 elif cls.op == COUNT:
-                    functions += f"{cls.py_op}({e}) AS int_var, "
+                    functions += (
+                        f"{cls.py_op}({e}) AS int_var, "
+                        if component_operand
+                        else "COUNT(*) AS int_var, "
+                    )
                     break
                 else:
                     functions += f"{cls.py_op}({e}) AS {e}, "
@@ -263,6 +268,7 @@ def evaluate(  # type: ignore[override]
         group_op: Optional[str],
         grouping_columns: Optional[List[str]],
         having_expr: Optional[str],
+        component_operand: bool = False,
     ) -> Dataset:
         result = cls.validate(operand, group_op, grouping_columns, having_expr)
 
@@ -273,14 +279,16 @@ def evaluate(  # type: ignore[override]
         # Keep a copy of viral attrs for post-aggregation propagation
         viral_df = result_df[grouping_keys + viral_attr_names].copy() if viral_attr_names else None
         result_df = result_df[grouping_keys + measure_names]
-        if cls.op == COUNT:
+        if cls.op == COUNT and component_operand:
             result_df = result_df.dropna(subset=measure_names, how="any")
         if cls.op in [MAX, MIN]:
             for measure in operand.get_measures():
                 if measure.data_type == TimeInterval:
                     raise RunTimeError("2-1-19-18", op=cls.op)
         cls._handle_data_types(result_df, operand.get_measures(), "input")
-        result_df = cls._agg_func(result_df, grouping_keys, measure_names, having_expr)
+        result_df = cls._agg_func(
+            result_df, grouping_keys, measure_names, having_expr, component_operand
+        )
 
         cls._handle_data_types(result_df, operand.get_measures(), "result")
         # Handle correct order on result

src/vtlengine/duckdb_transpiler/Transpiler/__init__.py

@@ -2219,11 +2219,6 @@ def visit_Aggregation(self, node: AST.Aggregation) -> str:  # type: ignore[overr
         # count() without operand
         if node.operand is None:
             if op == tokens.COUNT:
-                if self._in_clause and self._current_dataset:
-                    measures = self._current_dataset.get_measures_names()
-                    if measures:
-                        or_parts = " OR ".join(f"{quote_name(m)} IS NOT NULL" for m in measures)
-                        return f"NULLIF(COUNT(CASE WHEN {or_parts} THEN 1 END), 0)"
                 return "NULLIF(COUNT(*), 0)"
             return ""
 
@@ -2240,18 +2235,11 @@ def visit_Aggregation(self, node: AST.Aggregation) -> str:  # type: ignore[overr
         cols, group_by_cols = self._build_agg_group_cols(node, ds, group_cols)
         ds_tp_minmax_cols: List[tuple[str, str]] = []
 
-        # count() produces a single int_var measure.
+        # count() produces a single int_var measure. It reports the number of Data
+        # Points, so a Data Point is counted even where one of its Measures is null
+        # (issue #937); a group that exists always holds at least one of them.
         if op == tokens.COUNT:
-            alias = "int_var"
-            source_measures = ds.get_measures_names()
-            if source_measures:
-                and_parts = " AND ".join(f"{quote_name(m)} IS NOT NULL" for m in source_measures)
-                count_expr = f"COUNT(CASE WHEN {and_parts} THEN 1 END)"
-                if group_cols:
-                    count_expr = f"NULLIF({count_expr}, 0)"
-                cols.append(f"{count_expr} AS {quote_name(alias)}")
-            else:
-                cols.append(f"COUNT(*) AS {quote_name(alias)}")
+            cols.append(f"COUNT(*) AS {quote_name('int_var')}")
         else:
             measures = ds.get_measures_names()
             for measure in measures:

tests/Additional/data/DataSet/output/GL_222_1-1.csv

@@ -1,4 +1,4 @@
 Id_1,Me_3,Me_4
-1,1,1
+1,2,2
 2,3,3
 

tests/Aggregate/data/DataSet/output/GL_466_1-3.csv

@@ -1,16 +1,16 @@
 month,int_var
-2023-01,1
-2023-02,
-2023-03,
-2023-04,
-2023-05,
-2023-06,
-2023-07,
-2023-08,
-2023-09,
-2023-10,
-2023-11,
-2023-12,
-2024-01,
-2024-02,2
+2023-01,12
+2023-02,28
+2023-03,31
+2023-04,30
+2023-05,31
+2023-06,30
+2023-07,31
+2023-08,31
+2023-09,30
+2023-10,31
+2023-11,30
+2023-12,31
+2024-01,31
+2024-02,29
 2024-03,1

tests/Aggregate/data/DataSet/output/GL_466_2-1.csv

@@ -1,5 +1,5 @@
 month,int_var
-2023-01,
-2024-02,
-2024-03,
-2024-04,1.0
+2023-01,2
+2024-02,6
+2024-03,1
+2024-04,1

tests/BigProjects/MD_DEMO/data/DataSet/output/DEMO1-aggr.numDPCouYear.csv

@@ -1,97 +1,97 @@
 REF_DATE,REP_COUNTRY,int_var
-2018-12-31,CN,6
+2018-12-31,CN,14
 2018-12-31,AU,14
-2018-12-31,FI,13
-2018-12-31,TR,12
-2018-12-31,US,12
-2018-12-31,BM,12
+2018-12-31,FI,14
+2018-12-31,TR,14
+2018-12-31,US,14
+2018-12-31,BM,14
 2018-12-31,GG,14
-2018-12-31,CY,13
-2018-12-31,ID,12
+2018-12-31,CY,14
+2018-12-31,ID,14
 2018-12-31,5A,14
 2018-12-31,KY,14
-2018-12-31,BS,12
+2018-12-31,BS,14
 2018-12-31,RU,14
 2018-12-31,KR,14
-2018-12-31,GR,6
-2018-12-31,SE,12
+2018-12-31,GR,7
+2018-12-31,SE,14
 2018-12-31,DE,14
-2018-12-31,SG,7
-2018-12-31,JP,13
+2018-12-31,SG,9
+2018-12-31,JP,14
 2018-12-31,IT,14
 2018-12-31,NL,14
-2018-12-31,CW,6
-2018-12-31,HK,8
+2018-12-31,CW,7
+2018-12-31,HK,14
 2018-12-31,ZA,14
 2018-12-31,IM,14
 2018-12-31,BE,14
-2018-12-31,MY,4
+2018-12-31,MY,14
 2018-12-31,MO,14
 2018-12-31,AT,14
 2018-12-31,LU,14
 2018-12-31,PH,14
 2018-12-31,CH,14
-2018-12-31,PT,11
-2018-12-31,DK,11
-2018-12-31,IN,6
-2018-12-31,JE,8
+2018-12-31,PT,14
+2018-12-31,DK,14
+2018-12-31,IN,14
+2018-12-31,JE,9
 2018-12-31,NO,14
 2018-12-31,GB,14
 2018-12-31,TW,14
-2018-12-31,PA,7
-2018-12-31,CL,8
-2018-12-31,CA,13
+2018-12-31,PA,10
+2018-12-31,CL,10
+2018-12-31,CA,14
 2018-12-31,ES,14
 2018-12-31,IE,14
-2018-12-31,BR,8
+2018-12-31,BR,9
 2018-12-31,MX,6
 2018-12-31,BH,6
 2018-12-31,FR,14
-2019-03-31,CN,7
+2019-03-31,CN,14
 2019-03-31,AU,14
-2019-03-31,FI,13
-2019-03-31,TR,12
-2019-03-31,US,13
-2019-03-31,BM,13
+2019-03-31,FI,14
+2019-03-31,TR,14
+2019-03-31,US,14
+2019-03-31,BM,14
 2019-03-31,GG,14
-2019-03-31,CY,13
-2019-03-31,ID,12
+2019-03-31,CY,14
+2019-03-31,ID,14
 2019-03-31,5A,14
 2019-03-31,KY,14
-2019-03-31,BS,12
+2019-03-31,BS,14
 2019-03-31,RU,14
 2019-03-31,KR,14
-2019-03-31,GR,6
-2019-03-31,SE,13
+2019-03-31,GR,7
+2019-03-31,SE,14
 2019-03-31,DE,14
-2019-03-31,SG,7
-2019-03-31,JP,13
+2019-03-31,SG,9
+2019-03-31,JP,14
 2019-03-31,IT,14
 2019-03-31,NL,14
-2019-03-31,CW,6
-2019-03-31,HK,8
+2019-03-31,CW,7
+2019-03-31,HK,14
 2019-03-31,ZA,14
-2019-03-31,IM,13
+2019-03-31,IM,14
 2019-03-31,BE,14
-2019-03-31,MY,4
+2019-03-31,MY,14
 2019-03-31,MO,14
-2019-03-31,AT,13
+2019-03-31,AT,14
 2019-03-31,LU,14
 2019-03-31,PH,14
 2019-03-31,CH,14
-2019-03-31,PT,11
-2019-03-31,DK,11
-2019-03-31,IN,6
-2019-03-31,JE,8
+2019-03-31,PT,14
+2019-03-31,DK,14
+2019-03-31,IN,14
+2019-03-31,JE,9
 2019-03-31,NO,14
 2019-03-31,GB,14
 2019-03-31,TW,14
-2019-03-31,PA,6
-2019-03-31,CL,8
-2019-03-31,CA,13
+2019-03-31,PA,10
+2019-03-31,CL,10
+2019-03-31,CA,14
 2019-03-31,ES,14
 2019-03-31,IE,14
-2019-03-31,BR,8
+2019-03-31,BR,9
 2019-03-31,MX,6
 2019-03-31,BH,6
 2019-03-31,FR,14

tests/Bugs/data/DataSet/input/GH_937_1-1.csv

@@ -0,0 +1,4 @@
+Id_1,Id_2,Me_1,Me_2
+1,a,1.0,2020-01-01/2020-12-31
+1,b,2.0,2021-01-01/2021-12-31
+2,a,3.0,2022-01-01/2022-12-31

tests/Bugs/data/DataSet/output/GH_937_1-1.csv

@@ -0,0 +1,2 @@
+int_var
+3

tests/Bugs/data/DataSet/output/GH_937_1-2.csv

@@ -0,0 +1,3 @@
+Id_1,int_var
+1,2
+2,1

tests/Bugs/data/DataSet/output/GL_270_2-1.csv

@@ -1,5 +1,5 @@
 CNTRCT_ID,DT_RFRNC,INSTRMNT_ID,OBSRVD_AGNT_CD,JNT_LBLTY_AMNT_SUM,NMBR_DBTR,IS_JNT_LBLTY_RPRTD_ALL,JNT_LBLTY_AMNT_MAX
-AAA,2020-01-01,AAA,AAA,101.0,2,False,100.0
+AAA,2020-01-01,AAA,AAA,101.0,3,False,100.0
 BBB,2020-01-01,BBB,BBB,12.0,2,True,11.0
-CCC,2020-01-01,BBB,BBB,,,False,
+CCC,2020-01-01,BBB,BBB,,1,False,
 DDD,2020-01-01,DDD,BBB,0.0,1,True,0.0

tests/Bugs/data/DataStructure/input/GH_937_1-1.json

@@ -0,0 +1,33 @@
+{
+  "datasets": [
+    {
+      "name": "DS_1",
+      "DataStructure": [
+        {
+          "name": "Id_1",
+          "role": "Identifier",
+          "type": "Integer",
+          "nullable": false
+        },
+        {
+          "name": "Id_2",
+          "role": "Identifier",
+          "type": "String",
+          "nullable": false
+        },
+        {
+          "name": "Me_1",
+          "role": "Measure",
+          "type": "Number",
+          "nullable": true
+        },
+        {
+          "name": "Me_2",
+          "role": "Measure",
+          "type": "Time",
+          "nullable": true
+        }
+      ]
+    }
+  ]
+}

tests/Bugs/data/DataStructure/output/GH_937_1-1.json

@@ -0,0 +1,15 @@
+{
+  "datasets": [
+    {
+      "name": "DS_r1",
+      "DataStructure": [
+        {
+          "name": "int_var",
+          "role": "Measure",
+          "type": "Integer",
+          "nullable": true
+        }
+      ]
+    }
+  ]
+}