Issue 2919 add a manual check for referential integrity

sdv/_utils.py

@@ -562,3 +562,15 @@ def _get_unreferenced_keys(parent_columns, child_columns):
 def _validate_boolean_parameter(parameter, parameter_name):
     if not isinstance(parameter, bool):
         raise ValueError(f"'{parameter_name}' must be a boolean value.")
+
+
+def _check_is_dict_of_dataframes(data, arg_name='data'):
+    error_message_data = (
+        f"'{arg_name}' must be a dictionary that maps table names to pandas DataFrames."
+    )
+    if not isinstance(data, dict):
+        raise ValueError(error_message_data)
+
+    for table_name, table in data.items():
+        if not isinstance(table, pd.DataFrame):
+            raise ValueError(error_message_data)

sdv/datasets/local.py

@@ -3,9 +3,7 @@
 import os
 import warnings
 
-import pandas as pd
-
-from sdv._utils import _load_data_from_csv
+from sdv._utils import _check_is_dict_of_dataframes, _load_data_from_csv
 
 
 def load_csvs(folder_name, read_csv_parameters=None):
@@ -59,13 +57,7 @@ def save_csvs(data, folder_name, suffix=None, to_csv_parameters=None):
             A python dictionary of with string and value accepted by ``pandas.DataFrame.to_csv``
             function. Defaults to ``None``.
     """
-    error_message_data = "'data' must be a dictionary that maps table names to pandas DataFrames."
-    if not isinstance(data, dict):
-        raise ValueError(error_message_data)
-
-    for table_name, table in data.items():
-        if not isinstance(table, pd.DataFrame):
-            raise ValueError(error_message_data)
+    _check_is_dict_of_dataframes(data)
 
     if not os.path.exists(folder_name):
         os.makedirs(folder_name)

sdv/evaluation/utils.py

@@ -0,0 +1,154 @@
+"""Utility methods to compare the real and synthetic data."""
+
+import sys
+import warnings
+
+import pandas as pd
+
+from sdv._utils import _cast_to_iterable, _check_is_dict_of_dataframes
+from sdv.metadata import Metadata
+
+
+def _validate_referential_integrity_inputs(
+    metadata, synthetic_data, table_name, foreign_key_name, num_rows
+):
+    """Validate the inputs of the ``print_referential_integrity`` method."""
+    if not isinstance(metadata, Metadata):
+        raise TypeError('metadata must be of Metadata type.')
+
+    _check_is_dict_of_dataframes(synthetic_data, 'synthetic_data')
+
+    if not isinstance(table_name, str):
+        raise TypeError('table_name must be a string.')
+
+    foreign_key_names = _cast_to_iterable(foreign_key_name)
+    if not all(isinstance(name, str) for name in foreign_key_names):
+        raise TypeError('foreign_key_name must be a string or a tuple of strings.')
+
+    if isinstance(num_rows, bool) or not isinstance(num_rows, int):
+        raise TypeError("'num_rows' must be an integer greater than 0.")
+
+    if num_rows <= 0:
+        raise ValueError("'num_rows' must be an integer greater than 0.")
+
+    if table_name not in metadata.tables:
+        raise ValueError(f"table_name: '{table_name}' not found in metadata.")
+
+    if table_name not in synthetic_data:
+        raise ValueError(f"table_name: '{table_name}' not found in synthetic_data.")
+
+    for name in foreign_key_names:
+        if name not in metadata.tables[table_name].columns:
+            raise ValueError(
+                f"foreign_key_name: '{name}' not in Metadata for table_name: '{table_name}'."
+            )
+
+        if name not in synthetic_data[table_name].columns:
+            raise ValueError(f"foreign_key_name: '{name}' not found in synthetic_data.")
+
+    return foreign_key_names
+
+
+def _get_parent_relationship(metadata, table_name, foreign_key_names):
+    """Get the parent table and primary key linked to the given foreign key.
+
+    Args:
+        metadata (Metadata):
+            The metadata object describing the synthetic data.
+        table_name (str):
+            The name of the table that contains the foreign key.
+        foreign_key_names (list[str]):
+            The columns making up the foreign key to look up.
+
+    Returns:
+        tuple[str, list[str], list[str]]:
+            The parent table name, the columns making up its primary key, and the foreign key
+            columns in the order the relationship defines them.
+    """
+    for relationship in metadata.relationships:
+        child_foreign_key = _cast_to_iterable(relationship['child_foreign_key'])
+        if table_name == relationship['child_table_name'] and set(child_foreign_key) == set(
+            foreign_key_names
+        ):
+            return (
+                relationship['parent_table_name'],
+                _cast_to_iterable(relationship['parent_primary_key']),
+                child_foreign_key,
+            )
+
+    foreign_key = "', '".join(foreign_key_names)
+    raise ValueError(
+        f"Unable to find a relationship in metadata given table_name: '{table_name}' "
+        f"and foreign_key_name: '{foreign_key}'."
+    )
+
+
+def _format_key(key_names, key_values):
+    """Format a set of key columns and their values as ``name: value`` pairs."""
+    return ', '.join(f'{name}: {value}' for name, value in zip(key_names, key_values))
+
+
+def print_referential_integrity(
+    metadata, synthetic_data, table_name, foreign_key_name, num_rows=10
+):
+    """Check that referential integrity is met by looking up a few rows.
+
+    A random selection of rows is taken from the table containing the foreign key. For each
+    one, the linked row is looked up in the parent table and the outcome is printed.
+
+    Args:
+        metadata (Metadata):
+            The metadata object describing the synthetic data.
+        synthetic_data (dict):
+            A dictionary mapping each table name to a pandas DataFrame containing the
+            synthetic data for it.
+        table_name (str):
+            The name of the table that contains the foreign key to check.
+        foreign_key_name (str or tuple[str]):
+            The column of the foreign key to check. For composite keys, this is a tuple of
+            strings.
+        num_rows (int):
+            The number of rows to check. Defaults to 10.
+
+    Raises:
+        TypeError:
+            If any of the inputs is not of the expected type.
+        ValueError:
+            If the table, the columns or the relationship is missing, or if ``num_rows`` is
+            not greater than 0.
+    """
+    foreign_key_names = _validate_referential_integrity_inputs(
+        metadata, synthetic_data, table_name, foreign_key_name, num_rows
+    )
+    parent_table_name, parent_primary_keys, foreign_key_names = _get_parent_relationship(
+        metadata, table_name, foreign_key_names
+    )
+
+    child_data = synthetic_data[table_name]
+    if len(child_data) < num_rows:
+        warnings.warn(
+            f"The synthetic data contains '{len(child_data)}' rows which is less than "
+            f"num_rows: '{num_rows}'. Changing num_rows to '{len(child_data)}'."
+        )
+        num_rows = len(child_data)
+
+    parent_data = synthetic_data[parent_table_name]
+    parent_keys = set(parent_data[parent_primary_keys].itertuples(index=False, name=None))
+    child_primary_keys = _cast_to_iterable(metadata.tables[table_name].primary_key or [])
+
+    for _, child_row in child_data.sample(n=num_rows, replace=False).iterrows():
+        heading = f'Picking random {table_name} row'
+        if child_primary_keys:
+            key_values = ', '.join(str(child_row[name]) for name in child_primary_keys)
+            heading += f': {key_values}'
+
+        foreign_key_values = tuple(child_row[name] for name in foreign_key_names)
+        if any(pd.isna(value) for value in foreign_key_values):
+            result = '✅ Foreign key is null; no linked parent row expected'
+        elif foreign_key_values in parent_keys:
+            found = _format_key(parent_primary_keys, foreign_key_values)
+            result = f'✅ Found {parent_table_name} row! {found}'
+        else:
+            result = f'❌ Unable to find the linked {parent_table_name} row'
+
+        sys.stdout.write(f'{heading}\n{result}\n\n')

tests/integration/evaluation/test_utils.py

@@ -0,0 +1,139 @@
+import pandas as pd
+import pytest
+
+from sdv.evaluation.utils import print_referential_integrity
+from sdv.metadata import Metadata
+
+
+@pytest.fixture()
+def data_metadata_parent_child():
+    """A parent-child dataset with a primary to foreign key relationship between 1 column."""
+    data = {
+        'parent': pd.DataFrame({'parent_id': [0, 1, 2, 3], 'col': [1, 2, 3, 4]}),
+        'child': pd.DataFrame({'child_id': ['A', 'B', 'C', 'D'], 'parent_id': [0, 1, 1, 4]}),
+    }
+    metadata = Metadata().load_from_dict({
+        'tables': {
+            'parent': {
+                'columns': {
+                    'parent_id': {'sdtype': 'id'},
+                    'col': {'sdtype': 'numerical'},
+                },
+                'primary_key': 'parent_id',
+            },
+            'child': {
+                'columns': {
+                    'child_id': {'sdtype': 'id'},
+                    'parent_id': {'sdtype': 'id'},
+                },
+                'primary_key': 'child_id',
+            },
+        },
+        'relationships': [
+            {
+                'parent_table_name': 'parent',
+                'parent_primary_key': 'parent_id',
+                'child_table_name': 'child',
+                'child_foreign_key': 'parent_id',
+            }
+        ],
+    })
+    return data, metadata
+
+
+@pytest.fixture()
+def data_metadata_parent_child_composite_keys():
+    """A parent-child dataset whose primary and foreign keys span two columns."""
+    data = {
+        'parent': pd.DataFrame({
+            'A': [1, 1, 2, 3],
+            'B': ['X', 'Y', 'X', 'Z'],
+        }),
+        'child': pd.DataFrame({
+            'A': [1, 1, 2, 3, 1, 1, 2, 3, 4, 2],
+            'B': ['X', 'Y', 'X', 'Z', 'X', 'Z', 'Y', 'X', 'W', 'Z'],
+        }),
+    }
+    metadata = Metadata().load_from_dict({
+        'tables': {
+            'parent': {
+                'columns': {
+                    'A': {'sdtype': 'id'},
+                    'B': {'sdtype': 'id'},
+                },
+                'primary_key': ['A', 'B'],
+            },
+            'child': {
+                'columns': {
+                    'A': {'sdtype': 'id'},
+                    'B': {'sdtype': 'id'},
+                },
+            },
+        },
+        'relationships': [
+            {
+                'parent_table_name': 'parent',
+                'parent_primary_key': ['A', 'B'],
+                'child_table_name': 'child',
+                'child_foreign_key': ['A', 'B'],
+            },
+        ],
+    })
+    return data, metadata
+
+
+def test_print_referential_integrity(data_metadata_parent_child, capsys):
+    """Test ``print_referential_integrity`` with a simple parent-child dataset."""
+    # Setup
+    synthetic_data, metadata = data_metadata_parent_child
+
+    # Run
+    print_referential_integrity(metadata, synthetic_data, 'child', 'parent_id', num_rows=4)
+
+    # Assert
+    captured = capsys.readouterr().out
+    assert 'Picking random child row: A' in captured
+    assert 'Picking random child row: B' in captured
+    assert 'Picking random child row: C' in captured
+    assert 'Picking random child row: D' in captured
+    assert captured.count('✅ Found parent row! parent_id: 0') == 1
+    assert captured.count('✅ Found parent row! parent_id: 1') == 2
+    assert captured.count('❌ Unable to find the linked parent row') == 1
+
+
+def test_print_referential_integrity_composite_keys(
+    data_metadata_parent_child_composite_keys, capsys
+):
+    """Test ``print_referential_integrity`` with a parent-child dataset with composite keys.
+
+    The child table has no primary key, so no key value is printed in the heading. Two of its
+    ten rows reference a combination that is missing from the parent table.
+    """
+    # Setup
+    synthetic_data, metadata = data_metadata_parent_child_composite_keys
+
+    # Run
+    print_referential_integrity(metadata, synthetic_data, 'child', ('A', 'B'), num_rows=10)
+
+    # Assert
+    captured = capsys.readouterr().out
+    assert captured.count('Picking random child row\n') == 10
+    assert captured.count('✅ Found parent row! A: 1, B: X') == 2
+    assert captured.count('✅ Found parent row! A: 1, B: Y') == 1
+    assert captured.count('✅ Found parent row! A: 2, B: X') == 1
+    assert captured.count('✅ Found parent row! A: 3, B: Z') == 1
+    assert captured.count('❌ Unable to find the linked parent row') == 5
+
+
+def test_print_referential_integrity_with_null_foreign_key(data_metadata_parent_child, capsys):
+    """Test that a null foreign key is reported as expected rather than as a broken link."""
+    # Setup
+    synthetic_data, metadata = data_metadata_parent_child
+    synthetic_data['child'].loc[0, 'parent_id'] = None
+
+    # Run
+    print_referential_integrity(metadata, synthetic_data, 'child', 'parent_id', num_rows=4)
+
+    # Assert
+    captured = capsys.readouterr().out
+    assert captured.count('✅ Foreign key is null; no linked parent row expected') == 1

tests/unit/evaluation/test_single_table.py

@@ -3,7 +3,6 @@
 
 import pandas as pd
 import pytest
-from tests.utils import DataFrameDictMatcher
 
 from sdv.errors import VisualizationUnavailableError
 from sdv.evaluation.single_table import (
@@ -15,6 +14,7 @@
     run_diagnostic,
 )
 from sdv.metadata.metadata import Metadata
+from tests.utils import DataFrameDictMatcher
 
 
 def test_evaluate_quality():