diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 74b29f09..ffc7b8ec 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,24 +26,24 @@ jobs: fail-fast: false matrix: os: ['ubuntu-22.04'] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] - cratedb-version: ['4.8.4', '5.9.2'] + python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] + cratedb-version: ['4.8.4', '5.10.16', '6.3.3'] include: # A single slot for testing CrateDB nightly. - os: 'ubuntu-latest' - python-version: '3.13' + python-version: '3.14' cratedb-version: 'nightly' # A single slot for testing macOS. - os: 'macos-latest' - python-version: '3.13' - cratedb-version: '5.9.2' + python-version: '3.14' + cratedb-version: '6.3.3' # A single slot for testing Windows. - os: 'windows-latest' - python-version: '3.13' - cratedb-version: '5.9.2' + python-version: '3.14' + cratedb-version: '6.3.3' env: OS: ${{ matrix.os }} diff --git a/DEVELOP.rst b/DEVELOP.rst index 6af6552a..40d545e4 100644 --- a/DEVELOP.rst +++ b/DEVELOP.rst @@ -50,6 +50,7 @@ Running Tests The tests are run using the `unittest`_ module:: + export TESTCONTAINERS_RYUK_DISABLED=true python -m unittest -v In order to adjust the CrateDB version used for running the tests, amend the diff --git a/devtools/ci.sh b/devtools/ci.sh index a8e59ea8..d038193f 100755 --- a/devtools/ci.sh +++ b/devtools/ci.sh @@ -4,4 +4,5 @@ set -e -x isort --check --diff src/crate/ tests/ setup.py flake8 src/crate/crash +export TESTCONTAINERS_RYUK_DISABLED=true coverage run -m unittest -v diff --git a/setup.py b/setup.py index 39f17ba0..67329444 100644 --- a/setup.py +++ b/setup.py @@ -77,8 +77,8 @@ def read(path): extras_require=dict( test=[ 'crate[test]>=1.0.0.dev2', - 'cratedb-toolkit[testing]', 'sqlalchemy-cratedb', + 'testcontainers[cratedb] @ git+https://github.com/testcontainers/testcontainers-python.git@main', 'zc.customdoctests<2', ], devel=[ diff --git a/tests/test_integration.py b/tests/test_integration.py index 033c4781..afed2406 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -8,8 +8,6 @@ from unittest import SkipTest, TestCase from unittest.mock import Mock, patch -from cratedb_toolkit.testing.testcontainers.cratedb import CrateDBTestAdapter -from cratedb_toolkit.util.common import setup_logging from urllib3.exceptions import LocationParseError from crate.client.exceptions import ProgrammingError @@ -26,6 +24,7 @@ from crate.crash.outputs import _val_len as val_len from crate.crash.printer import ColorPrinter from tests import ftouch +from tests.util import CrateDBTestAdapter, setup_logging logger = logging.getLogger(__name__) @@ -50,32 +49,17 @@ def _skip_tests_in_ci() -> bool: raise SkipTest("Platform is not supported") -class EntrypointOpts: - version = os.getenv("CRATEDB_VERSION", "5.4.5") - psql_port = 45441 - http_port = 44209 - transport_port = 44309 - settings = { - "cluster.name": "Testing44209", - "node.name": "crate", - "lang.js.enabled": True, - "psql.port": psql_port, - "http.port": http_port, - "transport.tcp.port": transport_port, - } - - -node = CrateDBTestAdapter(crate_version=EntrypointOpts.version) +node = CrateDBTestAdapter(crate_version=os.getenv("CRATEDB_VERSION", "6.3.3")) def setUpModule(): - node.start( - cmd_opts=EntrypointOpts.settings, - # all ports inside container to be bound to the randomly generated ports on the host - ports={EntrypointOpts.http_port: None, - EntrypointOpts.psql_port: None, - EntrypointOpts.transport_port: None}) - node.reset() + options = { + "cluster.name": "Testing0815", + "node.name": "crate", + "lang.js.enabled": True, + } + node.start(cmd_opts=list(options.items())) + node.reset(schemas=["test"]) def tearDownModule(): @@ -121,7 +105,7 @@ def test_connect(self): class CommandTest(TestCase): def setUp(self): - node.reset() + node.reset(schemas=["test"]) def _output_format(self, format, func, query="select name from sys.cluster"): orig_argv = sys.argv[:] @@ -154,8 +138,8 @@ def assert_func(self, e, output, err): exception_code = e.code self.assertEqual(exception_code, 0) output = output.getvalue() - self.assertIn('| name |', output) - self.assertIn('| Testing44209 |', output) + self.assertIn('| name |', output) + self.assertIn('| Testing0815 |', output) self._output_format('tabular', assert_func) def test_json_output(self): @@ -163,7 +147,7 @@ def assert_func(self, e, output, err): exception_code = e.code self.assertEqual(exception_code, 0) output = output.getvalue() - self.assertIn('"name": "Testing44209"', output) + self.assertIn('"name": "Testing0815"', output) self._output_format('json', assert_func) def test_json_row_output(self): @@ -183,7 +167,7 @@ def assert_func(self, e, output, err): exception_code = e.code self.assertEqual(exception_code, 0) output = output.getvalue() - self.assertIn("""crate,'{"http": 44209, "psql": 45441, "transport": 44309}'""", output) + self.assertIn("""crate,'{"http": 4200, "psql": 5432, "transport": 4300}'""", output) self._output_format('csv', assert_func, query) @@ -214,7 +198,7 @@ def assert_func(self, e, output, err): exception_code = e.code self.assertEqual(exception_code, 0) output = output.getvalue() - self.assertIn("name | Testing44209", output) + self.assertIn("name | Testing0815", output) self._output_format('mixed', assert_func) def test_pprint_duplicate_keys(self): @@ -732,7 +716,7 @@ def test_command_timeout(self): # Get randomly generated host port bound to the predefined HTTP Interface port inside container node.cratedb._container.reload() - host_port = node.cratedb._container.ports.get("{}/tcp".format(EntrypointOpts.http_port), [])[0].get("HostPort") + host_port = node.cratedb._container.ports["4200/tcp"][0].get("HostPort") crash.logger.warn.assert_any_call( "No more Servers available, exception from last server: " @@ -819,7 +803,7 @@ def test_connect_info(self): schema='test') as crash: self.assertEqual(crash.connect_info.user, "crate") self.assertEqual(crash.connect_info.schema, "test") - self.assertEqual(crash.connect_info.cluster, "Testing44209") + self.assertEqual(crash.connect_info.cluster, "Testing0815") with patch.object( crash.cursor, @@ -832,7 +816,7 @@ def test_connect_info(self): crash._fetch_session_info() self.assertEqual(crash.connect_info.user, None) self.assertEqual(crash.connect_info.schema, "test") - self.assertEqual(crash.connect_info.cluster, "Testing44209") + self.assertEqual(crash.connect_info.cluster, "Testing0815") with patch.object( crash.cursor, diff --git a/tests/util.py b/tests/util.py index 06a50b70..c8aa8c66 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,5 +1,13 @@ +import logging +import os +import sys +import warnings +from typing import Optional from unittest.mock import Mock +from testcontainers.community.cratedb import CrateDBContainer +from verlib2 import Version + from crate.client.cursor import Cursor @@ -21,3 +29,78 @@ def fake_cursor(): that just works if you do not care about results. """ return mocked_cursor(description=[('undef',)], records=[('undef', None)]) + + +def setup_logging(level=logging.INFO, verbose: bool = False): + log_format = "%(asctime)-15s [%(name)-26s] %(levelname)-8s: %(message)s" + logging.basicConfig(format=log_format, stream=sys.stderr, level=level) + + +class CrateDBTestAdapter: + """ + A little helper wrapping Testcontainer's `CrateDBContainer`. + """ + + def __init__(self, crate_version: str = "nightly", **kwargs) -> None: + self.cratedb: Optional[CrateDBContainer] = None + self.image: str = "crate/crate:{}".format(crate_version) + + def start(self, **kwargs) -> None: + """ + Start container, used for tests set up + """ + self.cratedb = CrateDBContainer(image=self.image, **kwargs) + self.cratedb.start() + + def stop(self) -> None: + """ + Stop container, used for tests tear down + """ + if self.cratedb: + self.cratedb.stop() + + def reset(self, tables: Optional[list] = None, schemas: Optional[list] = None) -> None: + """ + Drop tables from the given list, used for tests set up or tear down + """ + import sqlalchemy as sa + engine = sa.create_engine(self.cratedb.get_connection_url()) + with engine.begin() as connection: + if schemas: + has_drop_schema_cascade = True + if "CRATEDB_VERSION" in os.environ: + cratedb_version = os.environ["CRATEDB_VERSION"] + if cratedb_version != "nightly" and Version(cratedb_version) < Version("6.2"): + warnings.warn("CrateDB earlier than 6.2 does not support DROP SCHEMA ... CASCADE") + has_drop_schema_cascade = False + if has_drop_schema_cascade: + for reset_schema in schemas: + connection.exec_driver_sql( + f'DROP SCHEMA IF EXISTS {reset_schema} CASCADE;' + ) + if tables: + for reset_table in tables: + connection.exec_driver_sql( + f"DROP TABLE IF EXISTS {reset_table};" + ) + + def get_connection_url(self, *args, **kwargs) -> str: + """ + Return a URL for SQLAlchemy DB engine + """ + return self.cratedb.get_connection_url(*args, **kwargs) + + def get_http_url(self, **kwargs) -> str: + """ + Return a URL for CrateDB's HTTP endpoint + """ + return self.get_connection_url(**kwargs).replace("crate://", "http://") + + @property + def http_url(self) -> str: + """ + Return a URL for CrateDB's HTTP endpoint. + + Used to stay backward compatible with the downstream code. + """ + return self.get_http_url()