diff --git a/sdv/metadata/metadata.py b/sdv/metadata/metadata.py index 383394761..bd3fc7e6f 100644 --- a/sdv/metadata/metadata.py +++ b/sdv/metadata/metadata.py @@ -4,7 +4,7 @@ import pandas as pd -from sdv._utils import _validate_boolean_parameter +from sdv._utils import _is_datetime_type, _is_numerical, _validate_boolean_parameter from sdv.metadata.errors import InvalidMetadataError from sdv.metadata.multi_table import MultiTableMetadata from sdv.metadata.single_table import SingleTableMetadata @@ -308,6 +308,35 @@ def set_sequence_key(self, column_name, table_name=None): self._validate_table_exists(table_name) self.tables[table_name].set_sequence_key(column_name) + def _validate_data(self, data, table_name=None): + """Validate data and warn about datetime formats that cannot be verified.""" + super()._validate_data(data, table_name) + for current_table_name, table_data in data.items(): + table_metadata = self.tables.get(current_table_name) + if table_metadata is None: + continue + + for column_name, column_metadata in table_metadata.columns.items(): + datetime_format = column_metadata.get('datetime_format') + if not datetime_format: + continue + + column_data = table_data[column_name] + has_datetime_objects = any( + not isinstance(value, str) + and not _is_numerical(value) + and _is_datetime_type(value) + for value in column_data.dropna().head(1000) + ) + if has_datetime_objects: + warnings.warn( + f"The datetime format for column '{column_name}' " + f"(table '{current_table_name}') could not be verified because the data " + f"is represented as dtype '{column_data.dtype}'.\n" + 'Please omit the datetime format string from the metadata or cast the ' + 'data to strings with the right format.' + ) + def validate_table(self, data, table_name=None): """Validate a table against the metadata. diff --git a/sdv/single_table/base.py b/sdv/single_table/base.py index f7c0254f7..42b545556 100644 --- a/sdv/single_table/base.py +++ b/sdv/single_table/base.py @@ -651,6 +651,11 @@ def validate(self, data): message=r"No 'datetime_format' is present.*", category=UserWarning, ) + warnings.filterwarnings( + 'ignore', + message=r'The datetime format for column .* could not be verified.*', + category=UserWarning, + ) self._original_metadata.validate_data({self._table_name: data}) else: self._original_metadata.validate_data({self._table_name: data}) diff --git a/sdv/utils/utils.py b/sdv/utils/utils.py index 8ffb2a239..0b018cd5a 100644 --- a/sdv/utils/utils.py +++ b/sdv/utils/utils.py @@ -64,6 +64,11 @@ def drop_unknown_references(data, metadata, drop_missing_values=False, verbose=T message=r"No 'datetime_format' is present.*", category=UserWarning, ) + warnings.filterwarnings( + 'ignore', + message=r'The datetime format for column .* could not be verified.*', + category=UserWarning, + ) metadata.validate_data(data) if drop_missing_values: _validate_foreign_keys_not_null(metadata, data) diff --git a/tests/unit/metadata/test_metadata.py b/tests/unit/metadata/test_metadata.py index 0a186aedc..7ead04367 100644 --- a/tests/unit/metadata/test_metadata.py +++ b/tests/unit/metadata/test_metadata.py @@ -1,4 +1,5 @@ import re +from datetime import date, datetime from unittest.mock import Mock, call, mock_open, patch import pandas as pd @@ -574,6 +575,74 @@ def test_validate_data_no_relationships(self): metadata.validate_data(data) assert metadata.METADATA_SPEC_VERSION == 'V1' + def test_validate_data_warns_when_datetime_format_cannot_be_verified(self): + """Test warnings for formatted datetime columns without verifiable formats.""" + # Setup + metadata = Metadata.load_from_dict({ + 'tables': { + 'guests': { + 'columns': { + 'datetime64_date': { + 'sdtype': 'datetime', + 'datetime_format': '%d %b %Y', + }, + 'timestamp_date': { + 'sdtype': 'datetime', + 'datetime_format': '%Y-%m-%d', + }, + 'datetime_date': { + 'sdtype': 'datetime', + 'datetime_format': '%Y-%m-%d', + }, + 'date_date': { + 'sdtype': 'datetime', + 'datetime_format': '%Y-%m-%d', + }, + 'string_date': { + 'sdtype': 'datetime', + 'datetime_format': '%d %b %Y', + }, + 'numerical_date': { + 'sdtype': 'datetime', + 'datetime_format': '%Y%m%d', + }, + }, + }, + }, + }) + data = { + 'guests': pd.DataFrame({ + 'datetime64_date': pd.to_datetime(['2026-07-21']), + 'timestamp_date': pd.Series([pd.Timestamp('2026-07-21')], dtype='object'), + 'datetime_date': pd.Series([datetime(2026, 7, 21)], dtype='object'), + 'date_date': pd.Series([date(2026, 7, 21)], dtype='object'), + 'string_date': ['21 Jul 2026'], + 'numerical_date': [20260721], + }), + } + + # Run + with pytest.warns(UserWarning) as raised_warnings: + metadata.validate_data(data) + + # Assert + expected_warning_columns = { + 'datetime64_date': 'datetime64[ns]', + 'timestamp_date': 'object', + 'datetime_date': 'object', + 'date_date': 'object', + } + expected_messages = [ + ( + f"The datetime format for column '{column_name}' (table 'guests') could not be " + f"verified because the data is represented as dtype '{dtype}'.\n" + 'Please omit the datetime format string from the metadata or cast the data to ' + 'strings with the right format.' + ) + for column_name, dtype in expected_warning_columns.items() + ] + assert [str(warning.message) for warning in raised_warnings] == expected_messages + def test_validate_table(self): """Test the ``validate_table``method.""" # Setup