diff --git a/src/vtlengine/Operators/Time.py b/src/vtlengine/Operators/Time.py index 7977833a7..88717f55d 100644 --- a/src/vtlengine/Operators/Time.py +++ b/src/vtlengine/Operators/Time.py @@ -92,6 +92,28 @@ def _get_period(cls, value: str) -> str: tp_value = TimePeriodHandler(value) return tp_value.period_indicator + @classmethod + def _iter_groups(cls, data: pd.DataFrame) -> Any: + """Iterate over the series the operand holds, one per combination of the + non-time identifiers. + + A Data Set may have the time identifier as its only identifier, in which + case it holds a single series and Pandas cannot group it. + """ + if not cls.other_ids: + return [((), data)] + return data.groupby(cls.other_ids) + + @staticmethod + def _group_key(values: Any) -> Any: + """The dictionary key a group of non-time identifier values maps to. + + Pandas hands over a one-element tuple when grouping by a single column, so + the values have to be unwrapped the same way on both sides of a lookup. + """ + values = tuple(values) + return values[0] if len(values) == 1 else values + @classmethod def parse_date(cls, date_str: str) -> date: return parse_date_value(date_str) @@ -142,16 +164,24 @@ def find_min_frequency(cls, dates: Any) -> str: (0, 0, 1): "D", } + @classmethod + def _interval_endpoints(cls, interval: str) -> Any: + """The two endpoint dates of a TimeInterval. + + The endpoints may carry a time component, which the input format allows, + so only the date part is read. + """ + start_str, end_str = interval.split("/") + return date.fromisoformat(start_str[:10]), date.fromisoformat(end_str[:10]) + @classmethod def get_frequency_from_time(cls, interval: str) -> Any: - start_date, end_date = interval.split("/") - return date.fromisoformat(end_date) - date.fromisoformat(start_date) + start, end = cls._interval_endpoints(interval) + return end - start @classmethod def _classify_interval_period(cls, interval: str) -> str: - start_str, end_str = interval.split("/") - start = date.fromisoformat(start_str) - end = date.fromisoformat(end_str) + start, end = cls._interval_endpoints(interval) candidates = [relativedelta(endpoint, start) for endpoint in (end, end + timedelta(days=1))] candidates = [ rd @@ -264,11 +294,15 @@ def evaluate(cls, operand: Any) -> Any: if data_type == TimePeriod: result.data = cls._period_accumulation(result.data, measure_names) elif data_type in (Date, TimeInterval): - result.data[measure_names] = ( - result.data.groupby(cls.other_ids)[measure_names] - .apply(cls.py_op) - .reset_index(drop=True) - ) + if cls.other_ids: + accumulated = ( + result.data.groupby(cls.other_ids)[measure_names] + .apply(cls.py_op) + .reset_index(drop=True) + ) + else: + accumulated = cls.py_op(result.data[measure_names].copy()).reset_index(drop=True) + result.data[measure_names] = accumulated else: raise SemanticError("1-1-19-8", op=cls.op, comp_type="dataset", param="date type") return result @@ -401,6 +435,9 @@ def evaluate(cls, operand: Dataset, fill_type: str) -> Dataset: result.data[cls.time_id] = result.data[cls.time_id].astype("string[pyarrow]") if len(result.data) < 2: return result + if not cls.other_ids: + # The operand holds a single series, so both limits span the same range. + fill_type = "all" data_type = result.components[cls.time_id].data_type if data_type == TimePeriod: result.data = cls.fill_periods(result.data, fill_type) @@ -516,12 +553,10 @@ def compute_min_max(group: Any) -> Dict[str, Any]: if fill_type == "all": return compute_min_max(data[cls.time_id]) - grouped = data.groupby(cls.other_ids) - result_dict = { - name if len(name) > 1 else name[0]: compute_min_max(group[cls.time_id]) - for name, group in grouped + return { + cls._group_key(name): compute_min_max(group[cls.time_id]) + for name, group in cls._iter_groups(data) } - return result_dict @classmethod def fill_dates(cls, data: pd.DataFrame, fill_type: str, min_frequency: str) -> pd.DataFrame: @@ -539,12 +574,13 @@ def date_filler(cls, data: pd.DataFrame, fill_type: str, min_frequency: str) -> def create_filled_dates(group: Any, min_max: Dict[str, Any]) -> (pd.DataFrame, str): # type: ignore[syntax] date_range = cls._period_range(min_max["min"], min_max["max"], min_frequency) date_df = pd.DataFrame(date_range, columns=[cls.time_id]) - date_df[cls.other_ids] = group.iloc[0][cls.other_ids] + if cls.other_ids: + date_df[cls.other_ids] = group.iloc[0][cls.other_ids] date_df[cls.measures] = None return date_df, min_max["date_format"] - for name, group in data.groupby(cls.other_ids): - min_max = MAX_MIN if fill_type == "all" else MAX_MIN[name if len(name) > 1 else name[0]] + for name, group in cls._iter_groups(data): + min_max = MAX_MIN if fill_type == "all" else MAX_MIN[cls._group_key(name)] filled_dates, date_format = create_filled_dates(group, min_max) filled_data.append(filled_dates) @@ -556,13 +592,12 @@ def create_filled_dates(group: Any, min_max: Dict[str, Any]) -> (pd.DataFrame, s @classmethod def max_min_from_time(cls, data: pd.DataFrame, fill_type: str = "all") -> Dict[str, Any]: - data = data.applymap(str).sort_values( # type: ignore[operator] - by=cls.other_ids + [cls.time_id] - ) + data = data.sort_values(by=cls.other_ids + [cls.time_id]) def extract_max_min(group: Any) -> Dict[str, Any]: - start_dates = group.str.split("/").str[0] - end_dates = group.str.split("/").str[1] + intervals = group.astype(str) + start_dates = intervals.str.split("/").str[0] + end_dates = intervals.str.split("/").str[1] return { "start": {"min": start_dates.min(), "max": start_dates.max()}, "end": {"min": end_dates.min(), "max": end_dates.max()}, @@ -570,58 +605,48 @@ def extract_max_min(group: Any) -> Dict[str, Any]: if fill_type == "all": return extract_max_min(data[cls.time_id]) - else: - return { - (name if len(name) > 1 else name[0]): extract_max_min(group[cls.time_id]) - for name, group in data.groupby(cls.other_ids) - } + return { + cls._group_key(name): extract_max_min(group[cls.time_id]) + for name, group in cls._iter_groups(data) + } @classmethod def fill_time_intervals( cls, data: pd.DataFrame, fill_type: str, frequency: str ) -> pd.DataFrame: - result_data = cls.time_filler(data, fill_type, frequency) - not_na = result_data[cls.measures].notna().any(axis=1) - duplicated = result_data.duplicated(subset=(cls.other_ids + [cls.time_id]), keep=False) - return result_data[~duplicated | not_na] + """Add the Data Points the interval grid expects and keep the operand's own. - @classmethod - def time_filler(cls, data: pd.DataFrame, fill_type: str, frequency: str) -> pd.DataFrame: + Both endpoints are stepped by one frequency from their own lower limit, and + the k-th start is then paired with the k-th end. Only Data Points whose key + is missing are added, so the operand's own intervals always survive, even + when they overlap and the two endpoint grids come out different lengths. + """ MAX_MIN = cls.max_min_from_time(data, fill_type) + non_ids = [c for c in data.columns if c not in cls.other_ids and c != cls.time_id] - def fill_group(group_df: pd.DataFrame) -> pd.DataFrame: - group_key = group_df.iloc[0][cls.other_ids].values - if fill_type != "all": - group_key = group_key[0] if len(group_key) == 1 else tuple(group_key) - group_dict = MAX_MIN if fill_type == "all" else MAX_MIN[group_key] + def grid_intervals(limits: Dict[str, Any], sample: str) -> List[str]: + # The limits stay ISO strings so that a time of day survives the range. + starts = cls._period_range(limits["start"]["min"], limits["start"]["max"], frequency) + ends = cls._period_range(limits["end"]["min"], limits["end"]["max"], frequency) + fmt = "%Y-%m-%dT%H:%M:%S" if _has_time_component(sample.split("/")[0]) else "%Y-%m-%d" + return [f"{s.strftime(fmt)}/{e.strftime(fmt)}" for s, e in zip(starts, ends)] - intervals = [ - f"{group_dict['start']['min']}/{group_dict['end']['min']}", - f"{group_dict['start']['max']}/{group_dict['end']['max']}", - ] - for interval in intervals: - if interval not in group_df[cls.time_id].values: - empty_row = group_df.iloc[0].copy() - empty_row[cls.time_id] = interval - empty_row[cls.measures] = None - group_df = pd.concat([group_df, pd.DataFrame([empty_row])], ignore_index=True) - start_group_df = group_df.copy() - start_group_df[cls.time_id] = start_group_df[cls.time_id].str.split("/").str[0] - end_group_df = group_df.copy() - end_group_df[cls.time_id] = end_group_df[cls.time_id].str.split("/").str[1] - start_filled = cls.date_filler(start_group_df, fill_type, frequency) - end_filled = cls.date_filler(end_group_df, fill_type, frequency) - start_filled[cls.time_id] = start_filled[cls.time_id].str.cat( - end_filled[cls.time_id], sep="/" - ) - return start_filled + filled_rows: List[Dict[str, Any]] = [] + for name, group_df in cls._iter_groups(data): + limits = MAX_MIN if fill_type == "all" else MAX_MIN[cls._group_key(name)] + existing = set(group_df[cls.time_id].astype(str)) + other_vals = dict(zip(cls.other_ids, tuple(name))) + for interval in grid_intervals(limits, str(group_df[cls.time_id].iloc[0])): + if interval not in existing: + filled_rows.append( + {**other_vals, cls.time_id: interval, **dict.fromkeys(non_ids)} + ) - filled_data = [fill_group(group_df) for _, group_df in data.groupby(cls.other_ids)] - return ( - pd.concat(filled_data, ignore_index=True) - .sort_values(by=cls.other_ids + [cls.time_id]) - .drop_duplicates() - ) + result = data + if filled_rows: + result = pd.concat([data, pd.DataFrame(filled_rows)], ignore_index=True) + result[cls.time_id] = result[cls.time_id].astype("string[pyarrow]") + return result.sort_values(by=cls.other_ids + [cls.time_id]).reset_index(drop=True) class Time_Shift(Binary): diff --git a/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py b/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py index 1bee4b314..066fda387 100644 --- a/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py +++ b/src/vtlengine/duckdb_transpiler/Transpiler/__init__.py @@ -1264,7 +1264,7 @@ def _visit_params(self, params: List[Any]) -> List[Optional[str]]: def _resolve_time_identifier(self, ds: Dataset, op_name: str) -> Any: """Return the time identifier name and type for time-based operators.""" for comp in ds.components.values(): - if comp.data_type in (TimePeriod, Date) and comp.role == Role.IDENTIFIER: + if comp.data_type in (TimePeriod, Date, TimeInterval) and comp.role == Role.IDENTIFIER: return comp.name, comp.data_type def _build_time_grid_parts( @@ -1321,6 +1321,8 @@ def visit_ParamOp_fill_time_series(self, node: AST.ParamOp) -> str: if time_type == Date: return self._fill_time_series_date(ds, src, time_id, fill_mode) + if time_type == TimeInterval: + return self._fill_time_series_interval(ds, src, time_id, fill_mode) return self._fill_time_series_period(ds, src, time_id, fill_mode) def _fill_time_series_period(self, ds: Dataset, src: str, time_id: str, fill_mode: str) -> str: @@ -1461,6 +1463,88 @@ def _fill_time_series_date(self, ds: Dataset, src: str, time_id: str, fill_mode: ) return cte.select(final) + def _fill_time_series_interval( + self, ds: Dataset, src: str, time_id: str, fill_mode: str + ) -> str: + """Fill time series for TimeInterval identifiers. + + The start dates and the end dates are each stepped by one frequency from + their own lower bound, and then the k-th start is paired with the k-th + end. That mirrors Fill_time_series.fill_time_intervals, and like it only + adds the Data Points whose key the operand is missing. + """ + time_col, other_id_cols, _, join_on, final_select, order_by = self._build_time_grid_parts( + ds, time_id + ) + per_group = fill_mode == "single" and bool(other_id_cols) + freq_step = "(SELECT step FROM freq)" + + cte = CTEBuilder() + cte.cte("source", f"SELECT * FROM {src}") + # One frequency for the whole operand, as Fill_time_series.evaluate requires. + # sample carries the operand's own representation over to the added intervals. + cte.cte( + "freq", + "SELECT CASE WHEN COUNT(DISTINCT f) > 1 THEN error(" + "'VTL 1-1-19-9: fill_time_series needs a single time interval frequency') " + "ELSE vtl_interval_freq_to_step(MIN(f)) END AS step, MIN(iv) AS sample " + f"FROM (SELECT vtl_interval_freq({time_col}) AS f, {time_col} AS iv " + f"FROM source WHERE {time_col} IS NOT NULL)", + ) + + bounds_cols = ( + f"MIN(vtl_interval_start_ts({time_col})) AS min_s, " + f"MAX(vtl_interval_start_ts({time_col})) AS max_s, " + f"MIN(vtl_interval_end_ts({time_col})) AS min_e, " + f"MAX(vtl_interval_end_ts({time_col})) AS max_e" + ) + if per_group: + oid_csv = ", ".join(other_id_cols) + cte.cte("bounds", f"SELECT {oid_csv}, {bounds_cols} FROM source GROUP BY {oid_csv}") + b_cols = ", ".join(f"b.{oc}" for oc in other_id_cols) + ", " + partition = "PARTITION BY " + ", ".join(f"b.{oc}" for oc in other_id_cols) + " " + else: + cte.cte("bounds", f"SELECT {bounds_cols} FROM source") + b_cols = partition = "" + + for name, lo, hi, alias in ( + ("starts", "min_s", "max_s", "d1"), + ("ends", "min_e", "max_e", "d2"), + ): + cte.cte( + name, + f"SELECT {b_cols}ROW_NUMBER() OVER ({partition}ORDER BY d) AS k, " + f"CAST(d AS TIMESTAMP) AS {alias} " + f"FROM bounds b, generate_series(b.{lo}, b.{hi}, {freq_step}) AS t(d)", + ) + + grid = f"vtl_interval_build(st.d1, en.d2, (SELECT sample FROM freq)) AS {time_col}" + if per_group: + on_ids = "".join(f" AND st.{oc} = en.{oc}" for oc in other_id_cols) + st_cols = ", ".join(f"st.{oc}" for oc in other_id_cols) + cte.cte( + "full_grid", + f"SELECT {st_cols}, {grid} FROM starts st JOIN ends en ON st.k = en.k{on_ids}", + ) + elif other_id_cols: + oid_csv = ", ".join(other_id_cols) + cte.cte("interval_grid", f"SELECT {grid} FROM starts st JOIN ends en ON st.k = en.k") + cte.cte("group_keys", f"SELECT DISTINCT {oid_csv} FROM source") + gk_cols = ", ".join(f"gk.{oc}" for oc in other_id_cols) + cte.cte( + "full_grid", + f"SELECT {gk_cols}, ig.{time_col} FROM group_keys gk, interval_grid ig", + ) + else: + cte.cte("full_grid", f"SELECT {grid} FROM starts st JOIN ends en ON st.k = en.k") + + keys = self._grid_with_source_keys(other_id_cols, time_col) + final = ( + f"SELECT {final_select} FROM ({keys}) g " + f"LEFT JOIN source s ON {join_on} ORDER BY {order_by}" + ) + return cte.select(final) + def _visit_flow_stock(self, node: AST.UnaryOp, op: str) -> str: """Visit FLOW_TO_STOCK or STOCK_TO_FLOW: window functions over time series.""" ds = self._get_dataset_structure(node.operand) @@ -1482,7 +1566,9 @@ def _visit_flow_stock(self, node: AST.UnaryOp, op: str) -> str: col = quote_name(comp.name) if comp.role == Role.IDENTIFIER: cols.append(col) - elif comp.data_type in (Integer, Number): + # Only number measures accumulate: the reference manual types the operand + # as measure, so attributes pass through untouched (issue #931). + elif comp.role == Role.MEASURE and comp.data_type in (Integer, Number): if op == tokens.FLOW_TO_STOCK: cols.append( f"CASE WHEN {col} IS NULL THEN NULL ELSE " @@ -1513,19 +1599,34 @@ def visit_BinOp_timeshift(self, node: AST.BinOp) -> str: col = quote_name(comp.name) cols.append(shifted if comp.name == time_id else col) return SQLBuilder().select(*cols).from_table(src).build() + elif time_type == TimeInterval: + # A TimeInterval series has a single frequency, taken from the duration of + # one interval. Time_Shift.evaluate reads it off row 0 of the unsorted + # operand; MIN() is the deterministic equivalent in SQL, and it agrees + # with row 0 for any operand in chronological order. + cols = [ + ( + f"vtl_interval_shift({quote_name(comp.name)}, {shift_sql}, freq.step) " + f"AS {quote_name(comp.name)}" + ) + if comp.name == time_id + else quote_name(comp.name) + for comp in ds.components.values() + ] + freq_sql = ( + f"SELECT vtl_interval_step(MIN({time_col})) AS step " + f"FROM {src} WHERE {time_col} IS NOT NULL" + ) + return f"""SELECT {", ".join(cols)} +FROM {src}, ( + {freq_sql} +) AS freq""" else: cols = [] for comp in ds.components.values(): col = quote_name(comp.name) if comp.name == time_id: - shifted_expr = ( - f"CASE WHEN freq.period_ind IN ('M','Q','S','A') " - f"AND CAST({col} AS DATE) = LAST_DAY(CAST({col} AS DATE)) " - f"THEN LAST_DAY(CAST(" - f"vtl_dateadd({col}, {shift_sql}, freq.period_ind) AS DATE)) " - f"ELSE vtl_dateadd({col}, {shift_sql}, freq.period_ind) END" - ) - cols.append(f"{shifted_expr} AS {col}") + cols.append(f"vtl_dateadd({col}, {shift_sql}, freq.period_ind) AS {col}") else: cols.append(col) diff --git a/src/vtlengine/duckdb_transpiler/io/_execution.py b/src/vtlengine/duckdb_transpiler/io/_execution.py index a620e8ce6..0389dd878 100644 --- a/src/vtlengine/duckdb_transpiler/io/_execution.py +++ b/src/vtlengine/duckdb_transpiler/io/_execution.py @@ -130,6 +130,15 @@ def _map_query_error(error: duckdb.Error, sql_query: str) -> Exception: len1, len2 = (m.group(1), m.group(2)) if m else ("?", "?") return SemanticError("1-1-18-11", op="string_distance", len1=len1, len2=len2) + # fill_time_series over TimeIntervals of more than one frequency (mirrors 1-1-19-9) + if "vtl 1-1-19-9" in msg_lower: + return SemanticError( + "1-1-19-9", + op="fill_time_series", + comp_type="dataset", + param="single time interval frequency", + ) + # Division by zero (explicit DuckDB error or VTL error from ratio_to_report) if "division by zero" in msg_lower or "divide by zero" in msg_lower: return RunTimeError("2-1-3-1", op="division") diff --git a/src/vtlengine/duckdb_transpiler/sql/time_operators.sql b/src/vtlengine/duckdb_transpiler/sql/time_operators.sql index 1aaebe1f9..4a583ce24 100644 --- a/src/vtlengine/duckdb_transpiler/sql/time_operators.sql +++ b/src/vtlengine/duckdb_transpiler/sql/time_operators.sql @@ -313,3 +313,109 @@ CREATE OR REPLACE MACRO vtl_tp_shift(p vtl_time_period, n INTEGER) AS ( }::vtl_time_period) END ); + + +-- ============================================================================ +-- TIMEINTERVAL FREQUENCY AND SHIFT +-- ============================================================================ +-- The frequency of a TimeInterval series is the DURATION of a single interval, +-- not a calendar anchor, so vtl_interval_to_period is not an equivalent here. +-- Reference: Time._classify_interval_period in Operators/Time.py. + +-- The endpoints. vtl_interval_parse cannot be reused: it assumes the date part +-- is exactly 10 characters, which breaks on the 2020-01-01T00:00:00/... form. +CREATE OR REPLACE MACRO vtl_interval_start_date(s) AS ( + CAST(SUBSTR(SPLIT_PART(s, '/', 1), 1, 10) AS DATE) +); + +CREATE OR REPLACE MACRO vtl_interval_end_date(s) AS ( + CAST(SUBSTR(SPLIT_PART(s, '/', 2), 1, 10) AS DATE) +); + +-- Whole calendar months from d1 to c: the largest m with d1 + m months <= c. +CREATE OR REPLACE MACRO vtl_interval_months(d1, c) AS ( + date_diff('month', d1, c) + - CASE WHEN d1 + INTERVAL (date_diff('month', d1, c)) MONTH > c THEN 1 ELSE 0 END +); + +-- The days left over once those whole months are taken out. Together with +-- vtl_interval_months this reproduces relativedelta's normalisation. +CREATE OR REPLACE MACRO vtl_interval_days(d1, c) AS ( + date_diff('day', d1 + INTERVAL (vtl_interval_months(d1, c)) MONTH, c) +); + +-- The six canonical VTL frequencies as (months, days): Y S Q M W D. +CREATE OR REPLACE MACRO vtl_interval_is_canonical(m, d) AS ( + (d = 0 AND m IN (12, 6, 3, 1)) OR (m = 0 AND d IN (7, 1)) +); + +-- Non-zero component count, over relativedelta's (years, months, days) split. +CREATE OR REPLACE MACRO vtl_interval_nonzero(m, d) AS ( + CASE WHEN m // 12 <> 0 THEN 1 ELSE 0 END + + CASE WHEN m % 12 <> 0 THEN 1 ELSE 0 END + + CASE WHEN d <> 0 THEN 1 ELSE 0 END +); + +-- The frequency of one interval as a (months, days) STRUCT. Both the interval +-- end and end + 1 day are candidates, the first canonical one wins, and +-- otherwise the one with fewer non-zero components does, ties going to the end +-- itself. A STRUCT rather than an INTERVAL because DuckDB normalises a month to +-- 30 days when comparing intervals, which would equate 30 days with one month. +CREATE OR REPLACE MACRO vtl_interval_freq(s) AS (( + SELECT CASE + WHEN (m1 > 0 OR d1 > 0) AND vtl_interval_is_canonical(m1, d1) + THEN {'months': m1, 'days': d1} + WHEN vtl_interval_is_canonical(m2, d2) THEN {'months': m2, 'days': d2} + -- A zero-length candidate is dropped, as relativedelta's filter does. + WHEN m1 = 0 AND d1 = 0 THEN {'months': m2, 'days': d2} + WHEN vtl_interval_nonzero(m1, d1) <= vtl_interval_nonzero(m2, d2) + THEN {'months': m1, 'days': d1} + ELSE {'months': m2, 'days': d2} + END + FROM (SELECT vtl_interval_months(a, b) AS m1, + vtl_interval_days(a, b) AS d1, + vtl_interval_months(a, b + INTERVAL 1 DAY) AS m2, + vtl_interval_days(a, b + INTERVAL 1 DAY) AS d2 + FROM (SELECT vtl_interval_start_date(s) AS a, + vtl_interval_end_date(s) AS b) AS _iv) AS _cand +)); + +-- The same frequency as a native step, for generate_series and date arithmetic. +CREATE OR REPLACE MACRO vtl_interval_freq_to_step(f) AS ( + INTERVAL (f.months) MONTH + INTERVAL (f.days) DAY +); + +CREATE OR REPLACE MACRO vtl_interval_step(s) AS ( + vtl_interval_freq_to_step(vtl_interval_freq(s)) +); + +-- The endpoints again, keeping any time component this time, for the operators +-- that have to reproduce the operand's own representation. +CREATE OR REPLACE MACRO vtl_interval_start_ts(s) AS ( + CAST(SPLIT_PART(s, '/', 1) AS TIMESTAMP) +); + +CREATE OR REPLACE MACRO vtl_interval_end_ts(s) AS ( + CAST(SPLIT_PART(s, '/', 2) AS TIMESTAMP) +); + +-- Assemble an interval out of two endpoints, in the representation sample uses. +-- STRFTIME needs a literal format, hence the duplicated arms. +CREATE OR REPLACE MACRO vtl_interval_build(a, b, sample) AS ( + CASE + WHEN LENGTH(SPLIT_PART(sample, '/', 1)) > 10 + THEN STRFTIME(a, '%Y-%m-%dT%H:%M:%S') || '/' || STRFTIME(b, '%Y-%m-%dT%H:%M:%S') + ELSE STRFTIME(a, '%Y-%m-%d') || '/' || STRFTIME(b, '%Y-%m-%d') + END +); + +-- Shift both endpoints by n periods, mirroring Time_Shift.shift_interval: plain +-- calendar addition, since pd.DateOffset clamps but never snaps to a month end. +CREATE OR REPLACE MACRO vtl_interval_shift(s, n, step) AS ( + CASE + WHEN s IS NULL THEN NULL + ELSE vtl_interval_build( + vtl_interval_start_ts(s) + step * n, vtl_interval_end_ts(s) + step * n, s + ) + END +); diff --git a/tests/Bugs/data/DataSet/input/GH_918_1-1.csv b/tests/Bugs/data/DataSet/input/GH_918_1-1.csv new file mode 100644 index 000000000..3f7cab170 --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_918_1-1.csv @@ -0,0 +1,5 @@ +Id_1,Id_2,Me_1,Me_2 +A,2001-01-01/2001-12-31,1.0,1 +A,2003-01-01/2003-12-31,2.0,2 +B,2002-01-01/2002-12-31,3.0,3 +B,2004-01-01/2004-12-31,4.0,4 diff --git a/tests/Bugs/data/DataSet/input/GH_918_2-1.csv b/tests/Bugs/data/DataSet/input/GH_918_2-1.csv new file mode 100644 index 000000000..654997f66 --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_918_2-1.csv @@ -0,0 +1,4 @@ +Id_1,Id_2,Me_1,At_1 +A,2020-01-31,1.0,10.0 +A,2020-02-29,2.0,20.0 +A,2020-03-31,3.0,30.0 diff --git a/tests/Bugs/data/DataSet/input/GH_918_3-1.csv b/tests/Bugs/data/DataSet/input/GH_918_3-1.csv new file mode 100644 index 000000000..01e82ec2f --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_918_3-1.csv @@ -0,0 +1,5 @@ +Id_1,Id_2,Me_1,At_1 +A,2001-01-01/2001-12-31,1.0,a +A,2003-01-01/2003-12-31,2.0,b +B,2002-01-01/2002-12-31,3.0,c +B,2004-01-01/2004-12-31,4.0,d diff --git a/tests/Bugs/data/DataSet/input/GH_918_4-1.csv b/tests/Bugs/data/DataSet/input/GH_918_4-1.csv new file mode 100644 index 000000000..1f38b7632 --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_918_4-1.csv @@ -0,0 +1,5 @@ +Id_1,Id_2,Me_1 +1,2020-01-01/2020-12-31,1.0 +1,2022-01-01/2022-12-31,2.0 +2,2021-01-01/2021-12-31,3.0 +2,2023-01-01/2023-12-31,4.0 diff --git a/tests/Bugs/data/DataSet/input/GH_918_5-1.csv b/tests/Bugs/data/DataSet/input/GH_918_5-1.csv new file mode 100644 index 000000000..d2abc984c --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_918_5-1.csv @@ -0,0 +1,3 @@ +Id_1,Me_1 +2020-01-01/2020-12-31,1.0 +2022-01-01/2022-12-31,2.0 diff --git a/tests/Bugs/data/DataSet/input/GH_918_6-1.csv b/tests/Bugs/data/DataSet/input/GH_918_6-1.csv new file mode 100644 index 000000000..e0e37230c --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_918_6-1.csv @@ -0,0 +1,3 @@ +Id_1,Me_1 +2020-01-01,1.0 +2022-01-01,2.0 diff --git a/tests/Bugs/data/DataSet/input/GH_918_7-1.csv b/tests/Bugs/data/DataSet/input/GH_918_7-1.csv new file mode 100644 index 000000000..d2e29047c --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_918_7-1.csv @@ -0,0 +1,3 @@ +Id_1,Id_2,Me_1 +A,2020-01-01/2020-01-31,1.0 +A,2020-01-31/2020-02-29,2.0 diff --git a/tests/Bugs/data/DataSet/input/GH_918_8-1.csv b/tests/Bugs/data/DataSet/input/GH_918_8-1.csv new file mode 100644 index 000000000..f14bcf43d --- /dev/null +++ b/tests/Bugs/data/DataSet/input/GH_918_8-1.csv @@ -0,0 +1,3 @@ +Id_1,Id_2,Me_1 +A,2020-01-01T06:30:00/2020-12-31T06:30:00,1.0 +A,2022-01-01T06:30:00/2022-12-31T06:30:00,2.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_1-1.csv b/tests/Bugs/data/DataSet/output/GH_918_1-1.csv new file mode 100644 index 000000000..3df1e8b75 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_1-1.csv @@ -0,0 +1,5 @@ +Id_1,Id_2,Me_1,Me_2 +A,2001-01-01/2001-12-31,1.0,1 +A,2003-01-01/2003-12-31,3.0,3 +B,2002-01-01/2002-12-31,3.0,3 +B,2004-01-01/2004-12-31,7.0,7 diff --git a/tests/Bugs/data/DataSet/output/GH_918_1-2.csv b/tests/Bugs/data/DataSet/output/GH_918_1-2.csv new file mode 100644 index 000000000..1c4799e73 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_1-2.csv @@ -0,0 +1,5 @@ +Id_1,Id_2,Me_1,Me_2 +A,2001-01-01/2001-12-31,1.0,1 +A,2003-01-01/2003-12-31,1.0,1 +B,2002-01-01/2002-12-31,3.0,3 +B,2004-01-01/2004-12-31,1.0,1 diff --git a/tests/Bugs/data/DataSet/output/GH_918_1-3.csv b/tests/Bugs/data/DataSet/output/GH_918_1-3.csv new file mode 100644 index 000000000..4948ddc22 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_1-3.csv @@ -0,0 +1,5 @@ +Id_1,Id_2,Me_1,Me_2 +A,2002-01-01/2002-12-31,1.0,1 +A,2004-01-01/2004-12-31,2.0,2 +B,2003-01-01/2003-12-31,3.0,3 +B,2005-01-01/2005-12-31,4.0,4 diff --git a/tests/Bugs/data/DataSet/output/GH_918_1-4.csv b/tests/Bugs/data/DataSet/output/GH_918_1-4.csv new file mode 100644 index 000000000..e1824dd92 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_1-4.csv @@ -0,0 +1,9 @@ +Id_2,Id_1,Me_1,Me_2 +2001-01-01/2001-12-31,A,1.0,1 +2002-01-01/2002-12-31,A,, +2003-01-01/2003-12-31,A,2.0,2 +2004-01-01/2004-12-31,A,, +2001-01-01/2001-12-31,B,, +2002-01-01/2002-12-31,B,3.0,3 +2003-01-01/2003-12-31,B,, +2004-01-01/2004-12-31,B,4.0,4 diff --git a/tests/Bugs/data/DataSet/output/GH_918_1-5.csv b/tests/Bugs/data/DataSet/output/GH_918_1-5.csv new file mode 100644 index 000000000..bd74d0610 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_1-5.csv @@ -0,0 +1,7 @@ +Id_2,Id_1,Me_1,Me_2 +2001-01-01/2001-12-31,A,1.0,1 +2002-01-01/2002-12-31,A,, +2003-01-01/2003-12-31,A,2.0,2 +2002-01-01/2002-12-31,B,3.0,3 +2003-01-01/2003-12-31,B,, +2004-01-01/2004-12-31,B,4.0,4 diff --git a/tests/Bugs/data/DataSet/output/GH_918_2-1.csv b/tests/Bugs/data/DataSet/output/GH_918_2-1.csv new file mode 100644 index 000000000..180999126 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_2-1.csv @@ -0,0 +1,4 @@ +Id_1,Id_2,Me_1,At_1 +A,2020-02-29,1.0,10.0 +A,2020-03-29,2.0,20.0 +A,2020-04-30,3.0,30.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_2-2.csv b/tests/Bugs/data/DataSet/output/GH_918_2-2.csv new file mode 100644 index 000000000..fc48640a6 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_2-2.csv @@ -0,0 +1,4 @@ +Id_1,Id_2,Me_1,At_1 +A,2020-01-31,1.0,10.0 +A,2020-02-29,3.0,20.0 +A,2020-03-31,6.0,30.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_2-3.csv b/tests/Bugs/data/DataSet/output/GH_918_2-3.csv new file mode 100644 index 000000000..13a320c58 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_2-3.csv @@ -0,0 +1,4 @@ +Id_1,Id_2,Me_1,At_1 +A,2020-01-31,1.0,10.0 +A,2020-02-29,1.0,20.0 +A,2020-03-31,1.0,30.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_3-1.csv b/tests/Bugs/data/DataSet/output/GH_918_3-1.csv new file mode 100644 index 000000000..a1f3888be --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_3-1.csv @@ -0,0 +1,9 @@ +Id_1,Id_2,Me_1,At_1 +A,2001-01-01/2001-12-31,1.0,a +A,2002-01-01/2002-12-31,, +A,2003-01-01/2003-12-31,2.0,b +A,2004-01-01/2004-12-31,, +B,2001-01-01/2001-12-31,, +B,2002-01-01/2002-12-31,3.0,c +B,2003-01-01/2003-12-31,, +B,2004-01-01/2004-12-31,4.0,d diff --git a/tests/Bugs/data/DataSet/output/GH_918_3-2.csv b/tests/Bugs/data/DataSet/output/GH_918_3-2.csv new file mode 100644 index 000000000..c6a5f9160 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_3-2.csv @@ -0,0 +1,7 @@ +Id_1,Id_2,Me_1,At_1 +A,2001-01-01/2001-12-31,1.0,a +A,2002-01-01/2002-12-31,, +A,2003-01-01/2003-12-31,2.0,b +B,2002-01-01/2002-12-31,3.0,c +B,2003-01-01/2003-12-31,, +B,2004-01-01/2004-12-31,4.0,d diff --git a/tests/Bugs/data/DataSet/output/GH_918_4-1.csv b/tests/Bugs/data/DataSet/output/GH_918_4-1.csv new file mode 100644 index 000000000..fe98e826b --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_4-1.csv @@ -0,0 +1,7 @@ +Id_1,Id_2,Me_1 +1,2020-01-01/2020-12-31,1.0 +1,2021-01-01/2021-12-31, +1,2022-01-01/2022-12-31,2.0 +2,2021-01-01/2021-12-31,3.0 +2,2022-01-01/2022-12-31, +2,2023-01-01/2023-12-31,4.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_5-1.csv b/tests/Bugs/data/DataSet/output/GH_918_5-1.csv new file mode 100644 index 000000000..0e7828752 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_5-1.csv @@ -0,0 +1,3 @@ +Id_1,Me_1 +2020-01-01/2020-12-31,1.0 +2022-01-01/2022-12-31,3.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_5-2.csv b/tests/Bugs/data/DataSet/output/GH_918_5-2.csv new file mode 100644 index 000000000..3e2f55028 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_5-2.csv @@ -0,0 +1,3 @@ +Id_1,Me_1 +2020-01-01/2020-12-31,1.0 +2022-01-01/2022-12-31,1.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_5-3.csv b/tests/Bugs/data/DataSet/output/GH_918_5-3.csv new file mode 100644 index 000000000..0236c7da0 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_5-3.csv @@ -0,0 +1,4 @@ +Id_1,Me_1 +2020-01-01/2020-12-31,1.0 +2021-01-01/2021-12-31, +2022-01-01/2022-12-31,2.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_5-4.csv b/tests/Bugs/data/DataSet/output/GH_918_5-4.csv new file mode 100644 index 000000000..0236c7da0 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_5-4.csv @@ -0,0 +1,4 @@ +Id_1,Me_1 +2020-01-01/2020-12-31,1.0 +2021-01-01/2021-12-31, +2022-01-01/2022-12-31,2.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_6-1.csv b/tests/Bugs/data/DataSet/output/GH_918_6-1.csv new file mode 100644 index 000000000..4d9a1e1ee --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_6-1.csv @@ -0,0 +1,3 @@ +Id_1,Me_1 +2020-01-01,1.0 +2022-01-01,3.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_6-2.csv b/tests/Bugs/data/DataSet/output/GH_918_6-2.csv new file mode 100644 index 000000000..18d49196c --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_6-2.csv @@ -0,0 +1,4 @@ +Id_1,Me_1 +2020-01-01,1.0 +2021-01-01, +2022-01-01,2.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_7-1.csv b/tests/Bugs/data/DataSet/output/GH_918_7-1.csv new file mode 100644 index 000000000..d2e29047c --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_7-1.csv @@ -0,0 +1,3 @@ +Id_1,Id_2,Me_1 +A,2020-01-01/2020-01-31,1.0 +A,2020-01-31/2020-02-29,2.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_7-2.csv b/tests/Bugs/data/DataSet/output/GH_918_7-2.csv new file mode 100644 index 000000000..35b30bbea --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_7-2.csv @@ -0,0 +1,3 @@ +Id_1,Id_2,Me_1 +A,2020-02-01/2020-02-29,1.0 +A,2020-02-29/2020-03-29,2.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_8-1.csv b/tests/Bugs/data/DataSet/output/GH_918_8-1.csv new file mode 100644 index 000000000..f8755b57f --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_8-1.csv @@ -0,0 +1,4 @@ +Id_1,Id_2,Me_1 +A,2020-01-01T06:30:00/2020-12-31T06:30:00,1.0 +A,2021-01-01T06:30:00/2021-12-31T06:30:00, +A,2022-01-01T06:30:00/2022-12-31T06:30:00,2.0 diff --git a/tests/Bugs/data/DataSet/output/GH_918_8-2.csv b/tests/Bugs/data/DataSet/output/GH_918_8-2.csv new file mode 100644 index 000000000..2c7a4e777 --- /dev/null +++ b/tests/Bugs/data/DataSet/output/GH_918_8-2.csv @@ -0,0 +1,3 @@ +Id_1,Id_2,Me_1 +A,2021-01-01T06:30:00/2021-12-31T06:30:00,1.0 +A,2023-01-01T06:30:00/2023-12-31T06:30:00,2.0 diff --git a/tests/Bugs/data/DataStructure/input/GH_918_1-1.json b/tests/Bugs/data/DataStructure/input/GH_918_1-1.json new file mode 100644 index 000000000..4caa80e81 --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_918_1-1.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "Me_2", + "role": "Measure", + "type": "Integer", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/input/GH_918_2-1.json b/tests/Bugs/data/DataStructure/input/GH_918_2-1.json new file mode 100644 index 000000000..e1fa1a411 --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_918_2-1.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Date", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "At_1", + "role": "Attribute", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/input/GH_918_3-1.json b/tests/Bugs/data/DataStructure/input/GH_918_3-1.json new file mode 100644 index 000000000..47b635427 --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_918_3-1.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "At_1", + "role": "Attribute", + "type": "String", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/input/GH_918_4-1.json b/tests/Bugs/data/DataStructure/input/GH_918_4-1.json new file mode 100644 index 000000000..5fe85fa8c --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_918_4-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Integer", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/input/GH_918_5-1.json b/tests/Bugs/data/DataStructure/input/GH_918_5-1.json new file mode 100644 index 000000000..4216fdde8 --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_918_5-1.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/input/GH_918_6-1.json b/tests/Bugs/data/DataStructure/input/GH_918_6-1.json new file mode 100644 index 000000000..121ee6d8f --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_918_6-1.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Date", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/input/GH_918_7-1.json b/tests/Bugs/data/DataStructure/input/GH_918_7-1.json new file mode 100644 index 000000000..bc33e9552 --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_918_7-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/input/GH_918_8-1.json b/tests/Bugs/data/DataStructure/input/GH_918_8-1.json new file mode 100644 index 000000000..bc33e9552 --- /dev/null +++ b/tests/Bugs/data/DataStructure/input/GH_918_8-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_1-1.json b/tests/Bugs/data/DataStructure/output/GH_918_1-1.json new file mode 100644 index 000000000..f9101bc83 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_1-1.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "Me_2", + "role": "Measure", + "type": "Integer", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_1-2.json b/tests/Bugs/data/DataStructure/output/GH_918_1-2.json new file mode 100644 index 000000000..93db08a4a --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_1-2.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r2", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "Me_2", + "role": "Measure", + "type": "Integer", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_1-3.json b/tests/Bugs/data/DataStructure/output/GH_918_1-3.json new file mode 100644 index 000000000..7accd9e76 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_1-3.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r3", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "Me_2", + "role": "Measure", + "type": "Integer", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_1-4.json b/tests/Bugs/data/DataStructure/output/GH_918_1-4.json new file mode 100644 index 000000000..f7cfa1ac8 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_1-4.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r4", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "Me_2", + "role": "Measure", + "type": "Integer", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_1-5.json b/tests/Bugs/data/DataStructure/output/GH_918_1-5.json new file mode 100644 index 000000000..5f3d944cf --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_1-5.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r5", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "Me_2", + "role": "Measure", + "type": "Integer", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_2-1.json b/tests/Bugs/data/DataStructure/output/GH_918_2-1.json new file mode 100644 index 000000000..488d4e245 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_2-1.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Date", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "At_1", + "role": "Attribute", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_2-2.json b/tests/Bugs/data/DataStructure/output/GH_918_2-2.json new file mode 100644 index 000000000..44e5fbf27 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_2-2.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r2", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Date", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "At_1", + "role": "Attribute", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_2-3.json b/tests/Bugs/data/DataStructure/output/GH_918_2-3.json new file mode 100644 index 000000000..2eac48afc --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_2-3.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r3", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Date", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "At_1", + "role": "Attribute", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_3-1.json b/tests/Bugs/data/DataStructure/output/GH_918_3-1.json new file mode 100644 index 000000000..8ac6a213e --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_3-1.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "At_1", + "role": "Attribute", + "type": "String", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_3-2.json b/tests/Bugs/data/DataStructure/output/GH_918_3-2.json new file mode 100644 index 000000000..4ad837aea --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_3-2.json @@ -0,0 +1,33 @@ +{ + "datasets": [ + { + "name": "DS_r2", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + }, + { + "name": "At_1", + "role": "Attribute", + "type": "String", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_4-1.json b/tests/Bugs/data/DataStructure/output/GH_918_4-1.json new file mode 100644 index 000000000..c754717e6 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_4-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Integer", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_5-1.json b/tests/Bugs/data/DataStructure/output/GH_918_5-1.json new file mode 100644 index 000000000..406f7e486 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_5-1.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_5-2.json b/tests/Bugs/data/DataStructure/output/GH_918_5-2.json new file mode 100644 index 000000000..196855d7d --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_5-2.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_r2", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_5-3.json b/tests/Bugs/data/DataStructure/output/GH_918_5-3.json new file mode 100644 index 000000000..5ee6d4a15 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_5-3.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_r3", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_5-4.json b/tests/Bugs/data/DataStructure/output/GH_918_5-4.json new file mode 100644 index 000000000..db265ff24 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_5-4.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_r4", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_6-1.json b/tests/Bugs/data/DataStructure/output/GH_918_6-1.json new file mode 100644 index 000000000..49e1b72d0 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_6-1.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Date", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_6-2.json b/tests/Bugs/data/DataStructure/output/GH_918_6-2.json new file mode 100644 index 000000000..60f98d013 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_6-2.json @@ -0,0 +1,21 @@ +{ + "datasets": [ + { + "name": "DS_r2", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "Date", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_7-1.json b/tests/Bugs/data/DataStructure/output/GH_918_7-1.json new file mode 100644 index 000000000..48930caef --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_7-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_7-2.json b/tests/Bugs/data/DataStructure/output/GH_918_7-2.json new file mode 100644 index 000000000..ae4de6270 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_7-2.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_r2", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_8-1.json b/tests/Bugs/data/DataStructure/output/GH_918_8-1.json new file mode 100644 index 000000000..48930caef --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_8-1.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_r1", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/DataStructure/output/GH_918_8-2.json b/tests/Bugs/data/DataStructure/output/GH_918_8-2.json new file mode 100644 index 000000000..ae4de6270 --- /dev/null +++ b/tests/Bugs/data/DataStructure/output/GH_918_8-2.json @@ -0,0 +1,27 @@ +{ + "datasets": [ + { + "name": "DS_r2", + "DataStructure": [ + { + "name": "Id_1", + "role": "Identifier", + "type": "String", + "nullable": false + }, + { + "name": "Id_2", + "role": "Identifier", + "type": "Time", + "nullable": false + }, + { + "name": "Me_1", + "role": "Measure", + "type": "Number", + "nullable": true + } + ] + } + ] +} diff --git a/tests/Bugs/data/vtl/GH_918_1.vtl b/tests/Bugs/data/vtl/GH_918_1.vtl new file mode 100644 index 000000000..bf3d814b0 --- /dev/null +++ b/tests/Bugs/data/vtl/GH_918_1.vtl @@ -0,0 +1,5 @@ +DS_r1 <- flow_to_stock(DS_1); +DS_r2 <- stock_to_flow(DS_1); +DS_r3 <- timeshift(DS_1, 1); +DS_r4 <- fill_time_series(DS_1, all); +DS_r5 <- fill_time_series(DS_1, single); diff --git a/tests/Bugs/data/vtl/GH_918_2.vtl b/tests/Bugs/data/vtl/GH_918_2.vtl new file mode 100644 index 000000000..7922350ee --- /dev/null +++ b/tests/Bugs/data/vtl/GH_918_2.vtl @@ -0,0 +1,3 @@ +DS_r1 <- timeshift(DS_1, 1); +DS_r2 <- flow_to_stock(DS_1); +DS_r3 <- stock_to_flow(DS_1); diff --git a/tests/Bugs/data/vtl/GH_918_3.vtl b/tests/Bugs/data/vtl/GH_918_3.vtl new file mode 100644 index 000000000..8b1b53c75 --- /dev/null +++ b/tests/Bugs/data/vtl/GH_918_3.vtl @@ -0,0 +1,2 @@ +DS_r1 <- fill_time_series(DS_1, all); +DS_r2 <- fill_time_series(DS_1, single); diff --git a/tests/Bugs/data/vtl/GH_918_4.vtl b/tests/Bugs/data/vtl/GH_918_4.vtl new file mode 100644 index 000000000..9e32f9234 --- /dev/null +++ b/tests/Bugs/data/vtl/GH_918_4.vtl @@ -0,0 +1 @@ +DS_r1 <- fill_time_series(DS_1, single); diff --git a/tests/Bugs/data/vtl/GH_918_5.vtl b/tests/Bugs/data/vtl/GH_918_5.vtl new file mode 100644 index 000000000..0a22824dc --- /dev/null +++ b/tests/Bugs/data/vtl/GH_918_5.vtl @@ -0,0 +1,4 @@ +DS_r1 <- flow_to_stock(DS_1); +DS_r2 <- stock_to_flow(DS_1); +DS_r3 <- fill_time_series(DS_1, all); +DS_r4 <- fill_time_series(DS_1, single); diff --git a/tests/Bugs/data/vtl/GH_918_6.vtl b/tests/Bugs/data/vtl/GH_918_6.vtl new file mode 100644 index 000000000..f45b8c6f4 --- /dev/null +++ b/tests/Bugs/data/vtl/GH_918_6.vtl @@ -0,0 +1,2 @@ +DS_r1 <- flow_to_stock(DS_1); +DS_r2 <- fill_time_series(DS_1, all); diff --git a/tests/Bugs/data/vtl/GH_918_7.vtl b/tests/Bugs/data/vtl/GH_918_7.vtl new file mode 100644 index 000000000..b827bbee0 --- /dev/null +++ b/tests/Bugs/data/vtl/GH_918_7.vtl @@ -0,0 +1,2 @@ +DS_r1 <- fill_time_series(DS_1, single); +DS_r2 <- timeshift(DS_1, 1); diff --git a/tests/Bugs/data/vtl/GH_918_8.vtl b/tests/Bugs/data/vtl/GH_918_8.vtl new file mode 100644 index 000000000..b827bbee0 --- /dev/null +++ b/tests/Bugs/data/vtl/GH_918_8.vtl @@ -0,0 +1,2 @@ +DS_r1 <- fill_time_series(DS_1, single); +DS_r2 <- timeshift(DS_1, 1); diff --git a/tests/Bugs/test_bugs.py b/tests/Bugs/test_bugs.py index 6e4feefca..8dc1858e9 100644 --- a/tests/Bugs/test_bugs.py +++ b/tests/Bugs/test_bugs.py @@ -1476,6 +1476,128 @@ def test_GH_949_1(self): self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + def test_GH_918_1(self): + """ + Status: OK + Description: the time operators accept an identifier of type Time + (TimeInterval), not only Date and Time_Period. The DuckDB + backend used to reject those Data Sets before generating any + SQL, so flow_to_stock, stock_to_flow, timeshift and + fill_time_series all raised a TypeError. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/918 + Goal: Check Result. + """ + code = "GH_918_1" + number_inputs = 1 + references_names = ["1", "2", "3", "4", "5"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_918_2(self): + """ + Status: OK + Description: on a Date identifier, timeshift adds calendar periods without + snapping a month end to the month end of the target month, and + only number measures accumulate, so a number Attribute passes + through flow_to_stock and stock_to_flow untouched. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/918 + Goal: Check Result. + """ + code = "GH_918_2" + number_inputs = 1 + references_names = ["1", "2", "3"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_918_3(self): + """ + Status: OK + Description: fill_time_series adds every Data Point the grid expects even + when the Data Set has Attribute components. The added Data + Points carry no values, so their Attributes are null too. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/918 + Goal: Check Result. + """ + code = "GH_918_3" + number_inputs = 1 + references_names = ["1", "2"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_918_4(self): + """ + Status: OK + Description: fill_time_series with the single limits method works whatever + the type of the other identifiers, which used to be looked up + by their string representation. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/918 + Goal: Check Result. + """ + code = "GH_918_4" + number_inputs = 1 + references_names = ["1"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_918_5(self): + """ + Status: OK + Description: a Data Set may hold a single series, with the time identifier + as its only identifier. The time operators then have nothing + to group by, which used to raise a Pandas ValueError. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/918 + Goal: Check Result. + """ + code = "GH_918_5" + number_inputs = 1 + references_names = ["1", "2", "3", "4"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_918_6(self): + """ + Status: OK + Description: same single series as GH_918_5, on a Date identifier. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/918 + Goal: Check Result. + """ + code = "GH_918_6" + number_inputs = 1 + references_names = ["1", "2"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_918_7(self): + """ + Status: OK + Description: fill_time_series only adds Data Points, so intervals that + overlap keep their own values. Their two endpoint grids come + out different lengths, which used to rewrite the operand's + intervals into ones it never held. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/918 + Goal: Check Result. + """ + code = "GH_918_7" + number_inputs = 1 + references_names = ["1", "2"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + + def test_GH_918_8(self): + """ + Status: OK + Description: a Time value may carry a time component, which the operators + read past when they measure an interval's duration and keep in + the intervals they add. + Git Issue: https://github.com/Meaningful-Data/vtlengine/issues/918 + Goal: Check Result. + """ + code = "GH_918_8" + number_inputs = 1 + references_names = ["1", "2"] + + self.BaseTest(code=code, number_inputs=number_inputs, references_names=references_names) + class SetBugs(BugHelper): """ """ diff --git a/tests/DateTime/test_datetime.py b/tests/DateTime/test_datetime.py index ffd8da6b7..c8ce3c901 100644 --- a/tests/DateTime/test_datetime.py +++ b/tests/DateTime/test_datetime.py @@ -9,7 +9,7 @@ from vtlengine.DataTypes import Date, Integer from vtlengine.DataTypes._time_checking import check_date from vtlengine.DataTypes.TimeHandling import check_max_date -from vtlengine.Exceptions import InputValidationException, RunTimeError +from vtlengine.Exceptions import InputValidationException, RunTimeError, SemanticError def _run_scalar(expression): @@ -818,10 +818,101 @@ def test_fill_time_series_interval_uniform_frequency(intervals): script="DS_r <- fill_time_series(DS_1, single);", data_structures=structure, datapoints={"DS_1": data_df}, + use_duckdb=_use_duckdb_backend(), ) assert set(result["DS_r"].data["Id_2"].tolist()) >= set(intervals) +@pytest.mark.parametrize( + "intervals, shift, expected", + [ + pytest.param( + ["2001-01-01/2001-12-31", "2002-01-01/2002-12-31"], + 1, + ["2002-01-01/2002-12-31", "2003-01-01/2003-12-31"], + id="yearly", + ), + pytest.param( + ["2020-01-31/2020-02-29", "2020-02-29/2020-03-29"], + 1, + ["2020-02-29/2020-03-29", "2020-03-29/2020-04-29"], + id="monthly_month_end_clamps_without_snapping", + ), + pytest.param( + ["2020-01-01/2022-01-01", "2022-01-01/2024-01-01"], + -1, + ["2018-01-01/2020-01-01", "2020-01-01/2022-01-01"], + id="biennial_negative_shift", + ), + pytest.param( + ["2020-01-01/2020-08-01", "2020-08-01/2021-03-01"], + 1, + ["2020-08-01/2021-03-01", "2021-03-01/2021-10-01"], + id="seven_month_span", + ), + ], +) +def test_timeshift_interval(intervals, shift, expected): + """Timeshift moves both endpoints of a TimeInterval by the frequency the + interval's own duration implies, canonical or not.""" + structure = { + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + {"name": "Id_1", "type": "String", "role": "Identifier", "nullable": False}, + {"name": "Id_2", "type": "Time", "role": "Identifier", "nullable": False}, + {"name": "Me_1", "type": "Integer", "role": "Measure", "nullable": True}, + ], + } + ] + } + data_df = pd.DataFrame( + {"Id_1": ["A"] * len(intervals), "Id_2": intervals, "Me_1": list(range(len(intervals)))} + ) + result = run( + script=f"DS_r <- timeshift(DS_1, {shift});", + data_structures=structure, + datapoints={"DS_1": data_df}, + use_duckdb=_use_duckdb_backend(), + ) + assert sorted(_to_pylist(result["DS_r"].data["Id_2"])) == sorted(expected) + + +def test_fill_time_series_interval_mixed_frequency(): + """Intervals of more than one frequency cannot define a grid (1-1-19-9).""" + structure = { + "datasets": [ + { + "name": "DS_1", + "DataStructure": [ + {"name": "Id_1", "type": "String", "role": "Identifier", "nullable": False}, + {"name": "Id_2", "type": "Time", "role": "Identifier", "nullable": False}, + {"name": "Me_1", "type": "Integer", "role": "Measure", "nullable": True}, + ], + } + ] + } + data_df = pd.DataFrame( + { + "Id_1": ["A"] * 3, + "Id_2": [ + "2020-01-01/2020-01-31", + "2020-02-01/2020-02-29", + "2020-03-01/2021-03-01", + ], + "Me_1": [1, 2, 3], + } + ) + with pytest.raises(SemanticError, match="1-1-19-9"): + run( + script="DS_r <- fill_time_series(DS_1, single);", + data_structures=structure, + datapoints={"DS_1": data_df}, + use_duckdb=_use_duckdb_backend(), + ) + + @pytest.mark.parametrize( "lim_method, Id_1, Id_2, Me_1, exp_Id_1, exp_Id_2, exp_Me_1", fill_time_series_period_params, diff --git a/tests/duckdb_transpiler/test_time_types.py b/tests/duckdb_transpiler/test_time_types.py index 592a85669..63edf99e8 100644 --- a/tests/duckdb_transpiler/test_time_types.py +++ b/tests/duckdb_transpiler/test_time_types.py @@ -352,3 +352,103 @@ def test_interval_varchar_equality(self, conn): "SELECT '2021-01-01/2022-01-01' = '2021-01-01/2022-06-30'" ).fetchone()[0] assert result is False + + +# ========================================================================= +# vtl_interval_freq / vtl_interval_shift: TimeInterval frequency and shifting +# ========================================================================= + + +class TestIntervalFrequency: + """The frequency of a TimeInterval must match Time._classify_interval_period.""" + + # (months, days) for every case of test_classify_interval_period, with the + # PYMD codes flattened into the offset they stand for. + @pytest.mark.parametrize( + "interval,months,days", + [ + # Daily and weekly + ("2020-01-01/2020-01-02", 0, 1), + ("2020-01-01/2020-01-01", 0, 1), + ("2020-01-01/2020-01-08", 0, 7), + ("2020-01-01/2020-01-07", 0, 7), + # Monthly, both the "start of next" and "end of current" conventions + ("2020-01-01/2020-02-01", 1, 0), + ("2020-01-01/2020-01-31", 1, 0), + ("2020-02-01/2020-03-01", 1, 0), + ("2020-02-01/2020-02-29", 1, 0), # leap year February + ("2021-02-01/2021-02-28", 1, 0), # non-leap year February + # Quarterly and semesterly + ("2020-01-01/2020-04-01", 3, 0), + ("2020-01-01/2020-03-31", 3, 0), + ("2020-01-01/2020-07-01", 6, 0), + ("2020-01-01/2020-06-30", 6, 0), + # Yearly, not anchored to January + ("2020-01-01/2021-01-01", 12, 0), + ("2020-01-01/2020-12-31", 12, 0), + ("2001-06-15/2002-06-14", 12, 0), + # Multi-period spans, which have no canonical period indicator + ("2020-01-01/2022-01-01", 24, 0), # P2Y + ("2020-01-01/2021-12-31", 24, 0), # P2Y, "end of period" + ("2020-01-01/2027-01-01", 84, 0), # P7Y + ("2020-01-01/2020-08-01", 7, 0), # P7M + ("2020-01-01/2020-07-31", 7, 0), # P7M, "end of period" + ("2020-01-01/2021-03-01", 14, 0), # P1Y2M + ("2020-01-01/2020-01-15", 0, 14), # P14D + ("2020-01-01/2020-01-16", 0, 15), # P15D + ], + ) + def test_interval_freq(self, conn, interval, months, days): + result = conn.execute(f"SELECT vtl_interval_freq('{interval}')").fetchone()[0] + assert (result["months"], result["days"]) == (months, days) + + def test_interval_freq_matches_pandas(self, conn): + """Cross-check against the pandas classifier the macro mirrors.""" + from vtlengine.Operators.Time import Time + + for interval in ("2020-01-01/2020-12-31", "2020-01-01/2022-01-01", "2020-01-01/2020-01-15"): + code = Time._classify_interval_period(interval) + offset = Time._period_offset(code).kwds + expected = ( + offset.get("years", 0) * 12 + offset.get("months", 0), + offset.get("days", 0), + ) + result = conn.execute(f"SELECT vtl_interval_freq('{interval}')").fetchone()[0] + assert (result["months"], result["days"]) == expected, interval + + def test_interval_step(self, conn): + """The step drives generate_series, so it must stay a native INTERVAL.""" + result = conn.execute( + "SELECT CAST(DATE '2020-01-31' + vtl_interval_step('2020-01-01/2020-01-31') AS DATE)" + ).fetchone()[0] + assert result.isoformat() == "2020-02-29" + + @pytest.mark.parametrize( + "interval,shift,expected", + [ + ("2001-01-01/2001-12-31", 1, "2002-01-01/2002-12-31"), + ("2001-01-01/2001-12-31", -2, "1999-01-01/1999-12-31"), + ("2001-01-01/2001-12-31", 0, "2001-01-01/2001-12-31"), + # A month end clamps but is not snapped to the target month end + ("2020-01-31/2020-02-29", 1, "2020-02-29/2020-03-29"), + # Non-canonical durations shift by their own length + ("2020-01-01/2022-01-01", 2, "2024-01-01/2026-01-01"), + ("2020-01-01/2020-01-15", 3, "2020-02-12/2020-02-26"), + ], + ) + def test_interval_shift(self, conn, interval, shift, expected): + result = conn.execute( + f"SELECT vtl_interval_shift('{interval}', {shift}, vtl_interval_step('{interval}'))" + ).fetchone()[0] + assert result == expected + + def test_interval_shift_keeps_time_component(self, conn): + result = conn.execute( + "SELECT vtl_interval_shift('2020-01-01T00:00:00/2020-12-31T00:00:00', 1, " + "vtl_interval_step('2020-01-01T00:00:00/2020-12-31T00:00:00'))" + ).fetchone()[0] + assert result == "2021-01-01T00:00:00/2021-12-31T00:00:00" + + def test_interval_shift_null(self, conn): + result = conn.execute("SELECT vtl_interval_shift(NULL, 1, INTERVAL 1 YEAR)").fetchone()[0] + assert result is None