Normalize timestamps to millisecond (3-digit) precision across all databases

.gitignore

@@ -144,3 +144,4 @@ benchmark_*.png
 
 # VSCode
 .vscode
+.cecli*

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "reladiff"
-version = "0.6.0"
+version = "0.6.6"
 description = "Command-line tool and Python library to efficiently diff rows across two different databases."
 authors = ["Erez Shinan <erezshin@gmail.com>"]
 license = "MIT"
@@ -21,7 +21,10 @@ classifiers = [
     "Topic :: Database :: Database Engines/Servers",
     "Typing :: Typed"
 ]
-packages = [{ include = "reladiff" }]
+packages = [
+    { include = "reladiff" },
+    { include = "sqeleton" }
+    ]
 
 [tool.poetry.dependencies]
 python = "^3.8"
@@ -30,15 +33,33 @@ dsnparse = "*"
 click = ">=8.1"
 rich = "*"
 toml = ">=0.10.2"
-sqeleton = "^0.1.7"
 mysql-connector-python = {version=">=8.0.29", optional=true}
 psycopg2-binary = {version="*", optional=true}
 snowflake-connector-python = {version=">=2.7.2", optional=true}
 cryptography = {version="*", optional=true}
 trino = {version=">=0.314.0", optional=true}
 presto-python-client = {version="*", optional=true}
 clickhouse-driver = {version="*", optional=true}
-duckdb = {version=">=0.6.0", optional=true}
+duckdb = {version=">=0.7.0", optional=true}
+textual = {version=">=0.9.1", optional=true}
+textual-select = {version="*", optional=true}
+pygments = {version=">=2.13.0", optional=true}
+prompt-toolkit = {version=">=3.0.36", optional=true}
+pyarrow = [
+    {version=">=18.0.0", markers="python_version >= '3.9'", optional=true},
+    {version="<18.0.0", markers="python_version < '3.9'", optional=true},
+]
+pandas = [
+    {version=">=2.1", markers="python_version >= '3.9'", optional=true},
+    {version="*", markers="python_version < '3.9'", optional=true},
+]
+numpy = [
+    {version=">=1.26.4", markers="python_version >= '3.9'", optional=true},
+    {version=">=1.24.4", markers="python_version < '3.9'", optional=true},
+]
+# sqlalchemy-dremio 3.0.4 is missing a type conversion for datetime[ms] and is not compatible with Python 3.8. Created a
+# fork with a fix and opening a PR.
+sqlalchemy-dremio = {git = "https://github.com/scottkwalker/sqlalchemy_dremio", rev="308e686b166787941180850ce5813c313d3a9985", optional=true}
 
 [tool.poetry.dev-dependencies]
 parameterized = "*"
@@ -52,9 +73,24 @@ trino = ">=0.314.0"
 presto-python-client = "*"
 clickhouse-driver = "*"
 vertica-python = "*"
-duckdb = ">=0.6.0"
+duckdb = "<1.2"
 # google-cloud-bigquery = "*"
 # databricks-sql-connector = "*"
+pyarrow = [
+    {version=">=18.0.0", markers="python_version >= '3.9'"},
+    {version="<18.0.0", markers="python_version < '3.9'"},
+]
+pandas = [
+    {version=">=2.1", markers="python_version >= '3.9'"},
+    {version="*", markers="python_version < '3.9'"},
+]
+numpy = [
+    {version=">=1.26.4", markers="python_version >= '3.9'"},
+    {version=">=1.24.4", markers="python_version < '3.9'"},
+]
+# sqlalchemy-dremio 3.0.4 is missing a type conversion for datetime[ms] and is not compatible with Python 3.8. Created a
+# fork with a fix and opening a PR.
+sqlalchemy-dremio = {git = "https://github.com/scottkwalker/sqlalchemy_dremio", rev="308e686b166787941180850ce5813c313d3a9985"}
 
 [tool.poetry.extras]
 # When adding, update also: README + dev deps just above
@@ -69,6 +105,8 @@ trino = ["trino"]
 clickhouse = ["clickhouse-driver"]
 vertica = ["vertica-python"]
 duckdb = ["duckdb"]
+tui = ["textual", "textual-select", "pygments", "prompt-toolkit"]
+dremio = ["sqlalchemy-dremio", "pandas", "pyarrow"]
 
 all = ["mysql-connector-python", "psycopg2-binary", "snowflake-connector-python", "cryptography", "presto-python-client", "cx_Oracle", "trino", "clickhouse-driver", "vertica-python", "duckdb"]
 
@@ -84,6 +122,7 @@ no_implicit_optional=false
 
 [tool.ruff]
 line-length = 120
+target-version = "py38"
 
 [tool.black]
 line-length = 120

reladiff/hashdiff_tables.py

@@ -1,4 +1,5 @@
 import os
+import re
 from functools import cmp_to_key
 from numbers import Number
 import logging
@@ -27,6 +28,18 @@
 
 logger = logging.getLogger("hashdiff_tables")
 
+# Patterns for concatenation-too-long errors from various databases
+_CONCAT_TOO_LONG_PATTERNS = [
+    re.compile(r"ORA-01489", re.IGNORECASE),  # Oracle: result of string concatenation is too long
+    re.compile(r"too long and would be truncated", re.IGNORECASE),  # Snowflake
+]
+
+
+def _is_concat_too_long_error(e: Exception) -> bool:
+    """Check if the exception is a concatenation-too-long error from the database."""
+    msg = str(e)
+    return any(p.search(msg) for p in _CONCAT_TOO_LONG_PATTERNS)
+
 
 def compare_element(a, b):
     """Compare a and b, treat None as the smallest value.
@@ -189,7 +202,20 @@ def _diff_segments(
             count1, count2 = self._threaded_call("count", [table1, table2])
             checksum1 = checksum2 = None
         else:
-            (count1, checksum1), (count2, checksum2) = self._threaded_call("count_and_checksum", [table1, table2])
+            try:
+                (count1, checksum1), (count2, checksum2) = self._threaded_call("count_and_checksum", [table1, table2])
+            except Exception as e:
+                if _is_concat_too_long_error(e):
+                    logger.warning(
+                        "Concatenation too long error detected. Retrying with MD5-hashed text columns."
+                    )
+                    table1 = table1.with_md5_text_columns()
+                    table2 = table2.with_md5_text_columns()
+                    (count1, checksum1), (count2, checksum2) = self._threaded_call(
+                        "count_and_checksum", [table1, table2]
+                    )
+                else:
+                    raise
 
         assert not info_tree.info.rowcounts
         info_tree.info.rowcounts = {1: count1, 2: count2}
@@ -232,7 +258,11 @@ def _bisect_and_diff_segments(
         # If count is below the threshold, just download and compare the columns locally
         # This saves time, as bisection speed is limited by ping and query performance.
         if max_rows < self.bisection_threshold or max_space_size < self.bisection_factor * 2:
-            rows1, rows2 = self._threaded_call("get_values", [table1, table2])
+            # get_values() downloads each column individually (no concatenation),
+            # so always use non-MD5 mode to get actual values for comparison.
+            plain_table1 = table1.new(_md5_text_columns=False) if table1._md5_text_columns else table1
+            plain_table2 = table2.new(_md5_text_columns=False) if table2._md5_text_columns else table2
+            rows1, rows2 = self._threaded_call("get_values", [plain_table1, plain_table2])
             diff = list(diff_sets(rows1, rows2, self.skip_sort_results, self.duplicate_rows_support))
 
             info_tree.info.set_diff(diff)

reladiff/table_segment.py

@@ -8,10 +8,10 @@
 from .utils import safezip, Vector
 from sqeleton.utils import ArithString, split_space
 from sqeleton.databases import Database, DbPath, DbKey, DbTime
-from sqeleton.abcs.database_types import String_UUID
+from sqeleton.abcs.database_types import String_UUID, Text
 from sqeleton.schema import Schema, create_schema
 from sqeleton.queries import Count, Checksum, SKIP, table, this, Expr, min_, max_, Code
-from sqeleton.queries.extras import ApplyFuncAndNormalizeAsString, NormalizeAsString
+from sqeleton.queries.extras import ApplyFuncAndNormalizeAsString, NormalizeAsString, Md5AsString
 
 logger = logging.getLogger("table_segment")
 
@@ -126,6 +126,7 @@ class TableSegment:
 
     case_sensitive: bool = True
     _schema: Schema = None
+    _md5_text_columns: bool = False
 
     def __post_init__(self):
         if not self.update_column and (self.min_update or self.max_update):
@@ -206,7 +207,10 @@ def make_select(self):
 
     def get_values(self) -> list:
         "Download all the relevant values of the segment from the database"
-        select = self.make_select().select(*self._relevant_columns_repr)
+        # get_values() downloads each column individually (no concatenation),
+        # so always use plain NormalizeAsString to get actual values.
+        plain_repr = [NormalizeAsString(this[c]) for c in self.relevant_columns]
+        select = self.make_select().select(*plain_repr)
         return self.database.query(select, List[Tuple])
 
     def choose_checkpoints(self, count: int) -> List[List[DbKey]]:
@@ -250,8 +254,18 @@ def relevant_columns(self) -> List[str]:
 
     @property
     def _relevant_columns_repr(self) -> List[Expr]:
+        if self._md5_text_columns and self._schema:
+            return [
+                Md5AsString(this[c]) if isinstance(self._schema.get(c), Text)
+                else NormalizeAsString(this[c])
+                for c in self.relevant_columns
+            ]
         return [NormalizeAsString(this[c]) for c in self.relevant_columns]
 
+    def with_md5_text_columns(self) -> "TableSegment":
+        """Return a copy of this segment that MD5-hashes Text columns to avoid concatenation length errors."""
+        return self.new(_md5_text_columns=True)
+
     def count(self) -> int:
         """Count how many rows are in the segment, in one pass."""
         return self.database.query(self.make_select().select(Count()), int)
@@ -330,7 +344,7 @@ def count_and_checksum(self) -> Tuple[int, int]:
         return (0, None)
 
     def __getattr__(self, attr):
-        assert attr in ("database", "key_columns", "key_types", "relevant_columns", "_schema")
+        assert attr in ("database", "key_columns", "key_types", "relevant_columns", "_schema", "_md5_text_columns")
         return getattr(self._table_segment, attr)
 
     @property
@@ -357,5 +371,8 @@ def make_select(self):
         # XXX shouldn't be called
         return self._table_segment.make_select()
 
+    def with_md5_text_columns(self) -> "EmptyTableSegment":
+        return self
+
     def get_values(self) -> list:
         return []

sqeleton-project/pyproject.toml

@@ -0,0 +1,114 @@
+[tool.poetry]
+name = "sqeleton"
+version = "0.1.8"
+description = "Python library for querying SQL databases"
+authors = ["Erez Shinan <erezshin@gmail.com>"]
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/erezsh/sqeleton"
+documentation = "https://sqeleton.readthedocs.io/en/latest/"
+classifiers = [
+    "Intended Audience :: Developers",
+    "Intended Audience :: Information Technology",
+    "Intended Audience :: System Administrators",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Topic :: Database :: Database Engines/Servers",
+    "Typing :: Typed"
+]
+packages = [{ include = "sqeleton" }]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+runtype = ">=0.5.0"
+dsnparse = "*"
+click = ">=8.1"
+rich = "*"
+toml = ">=0.10.2"
+mysql-connector-python = {version=">=8.0.29", optional=true}
+# psycopg2 = {version="*", optional=true}
+psycopg2-binary = {version="*", optional=true}
+snowflake-connector-python = {version=">=2.7.2", optional=true}
+cryptography = {version="*", optional=true}
+trino = {version=">=0.314.0", optional=true}
+presto-python-client = {version="*", optional=true}
+clickhouse-driver = {version="*", optional=true}
+duckdb = {version=">=0.7.0", optional=true}
+textual = {version=">=0.9.1", optional=true}
+textual-select = {version="*", optional=true}
+pygments = {version=">=2.13.0", optional=true}
+prompt-toolkit = {version=">=3.0.36", optional=true}
+pyarrow = [
+    {version=">=18.0.0", markers="python_version >= '3.9'", optional=true},
+    {version="<18.0.0", markers="python_version < '3.9'", optional=true},
+]
+pandas = [
+    {version=">=2.1", markers="python_version >= '3.9'", optional=true},
+    {version="*", markers="python_version < '3.9'", optional=true},
+]
+numpy = [
+    {version=">=1.26.4", markers="python_version >= '3.9'", optional=true},
+    {version=">=1.24.4", markers="python_version < '3.9'", optional=true},
+]
+# sqlalchemy-dremio 3.0.4 is missing a type conversion for datetime[ms] and is not compatible with Python 3.8. Created a
+# fork with a fix and opening a PR.
+sqlalchemy-dremio = {git = "https://github.com/scottkwalker/sqlalchemy_dremio", rev="308e686b166787941180850ce5813c313d3a9985", optional=true}
+
+[tool.poetry.dev-dependencies]
+parameterized = "*"
+unittest-parallel = "*"
+
+duckdb = "<1.2"
+mysql-connector-python = "*"
+# psycopg2 = "*"
+psycopg2-binary = "*"
+snowflake-connector-python = ">=2.7.2"
+cryptography = "*"
+trino = ">=0.314.0"
+presto-python-client = "*"
+clickhouse-driver = "*"
+vertica-python = "*"
+pyarrow = [
+    {version=">=18.0.0", markers="python_version >= '3.9'"},
+    {version="<18.0.0", markers="python_version < '3.9'"},
+]
+pandas = [
+    {version=">=2.1", markers="python_version >= '3.9'"},
+    {version="*", markers="python_version < '3.9'"},
+]
+numpy = [
+    {version=">=1.26.4", markers="python_version >= '3.9'"},
+    {version=">=1.24.4", markers="python_version < '3.9'"},
+]
+# sqlalchemy-dremio 3.0.4 is missing a type conversion for datetime[ms] and is not compatible with Python 3.8. Created a
+# fork with a fix and opening a PR.
+sqlalchemy-dremio = {git = "https://github.com/scottkwalker/sqlalchemy_dremio", rev="308e686b166787941180850ce5813c313d3a9985"}
+
+
+[tool.poetry.extras]
+mysql = ["mysql-connector-python"]
+postgresql = ["psycopg2-binary"]
+snowflake = ["snowflake-connector-python", "cryptography"]
+presto = ["presto-python-client"]
+oracle = ["cx_Oracle"]
+databricks = ["databricks-sql-connector"]
+trino = ["trino"]
+clickhouse = ["clickhouse-driver"]
+vertica = ["vertica-python"]
+duckdb = ["duckdb"]
+tui = ["textual", "textual-select", "pygments", "prompt-toolkit"]
+dremio = ["sqlalchemy-dremio", "pandas", "pyarrow"]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry.scripts]
+sqeleton = 'sqeleton.__main__:main'
+
+[tool.ruff]
+line-length = 120
+target-version = "py38"

sqeleton/__init__.py

@@ -0,0 +1,4 @@
+from .databases import connect
+from .queries import table, this, SKIP, code, commit
+
+__version__ = "0.1.8"

sqeleton/__main__.py

@@ -0,0 +1,28 @@
+import click
+from .repl import repl as repl_main
+
+
+@click.group(no_args_is_help=True)
+def main():
+    pass
+
+
+@main.command(no_args_is_help=True)
+@click.argument("database", required=True)
+def repl(database):
+    return repl_main(database)
+
+
+CONN_EDITOR_HELP = """CONFIG_PATH - Path to a TOML config file of db connections, new or existing."""
+
+
+@main.command(no_args_is_help=True, help=CONN_EDITOR_HELP)
+@click.argument("config_path", required=True)
+def conn_editor(config_path):
+    from .conn_editor import main
+
+    return main(config_path)
+
+
+if __name__ == "__main__":
+    main()

sqeleton/abcs/__init__.py

@@ -0,0 +1,15 @@
+from .database_types import (
+    AbstractDatabase,
+    AbstractDialect,
+    DbKey,
+    DbPath,
+    DbTime,
+    IKey,
+    ColType_UUID,
+    NumericType,
+    PrecisionType,
+    StringType,
+    Boolean,
+    Text,
+)
+from .compiler import AbstractCompiler, Compilable