Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 30 additions & 1 deletion sdv/metadata/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import pandas as pd

from sdv._utils import _validate_boolean_parameter
from sdv._utils import _is_datetime_type, _is_numerical, _validate_boolean_parameter
from sdv.metadata.errors import InvalidMetadataError
from sdv.metadata.multi_table import MultiTableMetadata
from sdv.metadata.single_table import SingleTableMetadata
Expand Down Expand Up @@ -308,6 +308,35 @@ def set_sequence_key(self, column_name, table_name=None):
self._validate_table_exists(table_name)
self.tables[table_name].set_sequence_key(column_name)

def _validate_data(self, data, table_name=None):
"""Validate data and warn about datetime formats that cannot be verified."""
super()._validate_data(data, table_name)
for current_table_name, table_data in data.items():
table_metadata = self.tables.get(current_table_name)
if table_metadata is None:
continue

for column_name, column_metadata in table_metadata.columns.items():
datetime_format = column_metadata.get('datetime_format')
if not datetime_format:
continue

column_data = table_data[column_name]
has_datetime_objects = any(
not isinstance(value, str)
and not _is_numerical(value)
and _is_datetime_type(value)
for value in column_data.dropna().head(1000)
)
if has_datetime_objects:
warnings.warn(
f"The datetime format for column '{column_name}' "
f"(table '{current_table_name}') could not be verified because the data "
f"is represented as dtype '{column_data.dtype}'.\n"
'Please omit the datetime format string from the metadata or cast the '
'data to strings with the right format.'
)

def validate_table(self, data, table_name=None):
"""Validate a table against the metadata.

Expand Down
5 changes: 5 additions & 0 deletions sdv/single_table/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,11 @@ def validate(self, data):
message=r"No 'datetime_format' is present.*",
category=UserWarning,
)
warnings.filterwarnings(
'ignore',
message=r'The datetime format for column .* could not be verified.*',
category=UserWarning,
)
self._original_metadata.validate_data({self._table_name: data})
else:
self._original_metadata.validate_data({self._table_name: data})
Expand Down
5 changes: 5 additions & 0 deletions sdv/utils/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ def drop_unknown_references(data, metadata, drop_missing_values=False, verbose=T
message=r"No 'datetime_format' is present.*",
category=UserWarning,
)
warnings.filterwarnings(
'ignore',
message=r'The datetime format for column .* could not be verified.*',
category=UserWarning,
)
metadata.validate_data(data)
if drop_missing_values:
_validate_foreign_keys_not_null(metadata, data)
Expand Down
69 changes: 69 additions & 0 deletions tests/unit/metadata/test_metadata.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import re
from datetime import date, datetime
from unittest.mock import Mock, call, mock_open, patch

import pandas as pd
Expand Down Expand Up @@ -574,6 +575,74 @@ def test_validate_data_no_relationships(self):
metadata.validate_data(data)
assert metadata.METADATA_SPEC_VERSION == 'V1'

def test_validate_data_warns_when_datetime_format_cannot_be_verified(self):
"""Test warnings for formatted datetime columns without verifiable formats."""
# Setup
metadata = Metadata.load_from_dict({
'tables': {
'guests': {
'columns': {
'datetime64_date': {
'sdtype': 'datetime',
'datetime_format': '%d %b %Y',
},
'timestamp_date': {
'sdtype': 'datetime',
'datetime_format': '%Y-%m-%d',
},
'datetime_date': {
'sdtype': 'datetime',
'datetime_format': '%Y-%m-%d',
},
'date_date': {
'sdtype': 'datetime',
'datetime_format': '%Y-%m-%d',
},
'string_date': {
'sdtype': 'datetime',
'datetime_format': '%d %b %Y',
},
'numerical_date': {
'sdtype': 'datetime',
'datetime_format': '%Y%m%d',
},
},
},
},
})
data = {
'guests': pd.DataFrame({
'datetime64_date': pd.to_datetime(['2026-07-21']),
'timestamp_date': pd.Series([pd.Timestamp('2026-07-21')], dtype='object'),
'datetime_date': pd.Series([datetime(2026, 7, 21)], dtype='object'),
'date_date': pd.Series([date(2026, 7, 21)], dtype='object'),
'string_date': ['21 Jul 2026'],
'numerical_date': [20260721],
}),
}

# Run
with pytest.warns(UserWarning) as raised_warnings:
metadata.validate_data(data)

# Assert
expected_warning_columns = {
'datetime64_date': 'datetime64[ns]',
'timestamp_date': 'object',
'datetime_date': 'object',
'date_date': 'object',
}
expected_messages = [
(
f"The datetime format for column '{column_name}' (table 'guests') could not be "
f"verified because the data is represented as dtype '{dtype}'.\n"
'Please omit the datetime format string from the metadata or cast the data to '
'strings with the right format.'
)
for column_name, dtype in expected_warning_columns.items()
]
assert [str(warning.message) for warning in raised_warnings] == expected_messages

def test_validate_table(self):
"""Test the ``validate_table``method."""
# Setup
Expand Down